From 2c9d94e5fae49f7523cc55f9fa86512dd686bed9 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 11:03:54 -0400
Subject: [PATCH 01/40] Start plumbing

---
 src/transformers/__init__.py           |   1 +
 src/transformers/modeling_tf_auto.py   |   9 +-
 src/transformers/modeling_tf_marian.py | 150 +++++++++++++++++
 tests/test_modeling_marian.py          |   2 +-
 tests/test_modeling_tf_marian.py       | 223 +++++++++++++++++++++++++
 5 files changed, 383 insertions(+), 2 deletions(-)
 create mode 100644 src/transformers/modeling_tf_marian.py
 create mode 100644 tests/test_modeling_tf_marian.py

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 26a1bfd0bff1..c1c6c6c4fe8d 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -592,6 +592,7 @@
         XLNetPreTrainedModel,
         load_tf_weights_in_xlnet,
     )
+    from .modeling_tf_marian import TFMarianMTModel
 
     # Optimization
     from .optimization import (
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index 06e77754538b..a8ca3ecfa291 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -106,6 +106,12 @@
     TFFunnelForTokenClassification,
     TFFunnelModel,
 )
+from .modeling_tf_marian import TFMarianMTModel
+from .configuration_marian import MarianConfig
+from .configuration_pegasus import PegasusConfig
+from .configuration_mbart import MBartConfig
+from .configuration_blenderbot import BlenderbotConfig
+
 from .modeling_tf_gpt2 import TFGPT2LMHeadModel, TFGPT2Model
 from .modeling_tf_longformer import TFLongformerForMaskedLM, TFLongformerForQuestionAnswering, TFLongformerModel
 from .modeling_tf_mobilebert import (
@@ -260,7 +266,8 @@
 )
 
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
-    [(T5Config, TFT5ForConditionalGeneration), (BartConfig, TFBartForConditionalGeneration)]
+    [(T5Config, TFT5ForConditionalGeneration), (BartConfig, TFBartForConditionalGeneration),
+     (MarianConfig, TFMarianMTModel)]
 )
 
 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
new file mode 100644
index 000000000000..fd14f8a5f56a
--- /dev/null
+++ b/src/transformers/modeling_tf_marian.py
@@ -0,0 +1,150 @@
+# coding=utf-8
+# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TF BART model, ported from the fairseq repo."""
+
+import math
+import random
+import warnings
+from typing import Dict, Optional, Tuple
+
+import tensorflow as tf
+from tensorflow import Tensor
+from tensorflow.keras.layers import Dense, LayerNormalization
+
+from .activations_tf import ACT2FN
+from .configuration_bart import BartConfig
+from .file_utils import add_start_docstrings, add_start_docstrings_to_callable, replace_return_docstrings
+from .modeling_tf_outputs import TFBaseModelOutput, TFBaseModelOutputWithPast, TFSeq2SeqLMOutput, TFSeq2SeqModelOutput
+from .modeling_tf_bart import TFBartForConditionalGeneration
+# Public API
+from .modeling_tf_utils import (
+    DUMMY_INPUTS,
+    TFPreTrainedModel,
+    TFSharedEmbeddings,
+    cast_bool_to_primitive,
+    keras_serializable,
+    shape_list,
+)
+from .tokenization_utils_base import BatchEncoding
+from .utils import logging
+
+
+_CONFIG_FOR_DOC = "BartConfig"
+_TOKENIZER_FOR_DOC = "BartTokenizer"
+
+BART_START_DOCSTRING = r"""
+
+    This model inherits from :class:`~transformers.TFPreTrainedModel`. Check the superclass documentation for the
+    generic methods the library implements for all its model (such as downloading or saving, resizing the input
+    embeddings, pruning heads etc.)
+
+    This model is also a `tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ subclass.
+    Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general
+    usage and behavior.
+
+    .. note::
+
+        TF 2.0 models accepts two formats as inputs:
+
+        - having all inputs as keyword arguments (like PyTorch models), or
+        - having all inputs as a list, tuple or dict in the first positional arguments.
+
+        This second option is useful when using :meth:`tf.keras.Model.fit` method which currently requires having
+        all the tensors in the first argument of the model call function: :obj:`model(inputs)`.
+
+        If you choose this second option, there are three possibilities you can use to gather all the input Tensors
+        in the first positional argument :
+
+        - a single Tensor with :obj:`input_ids` only and nothing else: :obj:`model(inputs_ids)`
+        - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
+          :obj:`model([input_ids, attention_mask])` or :obj:`model([input_ids, attention_mask, token_type_ids])`
+        - a dictionary with one or several input Tensors associated to the input names given in the docstring:
+          :obj:`model({"input_ids": input_ids, "token_type_ids": token_type_ids})`
+
+    Args:
+        config (:class:`~transformers.BartConfig`): Model configuration class with all the parameters of the model.
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the :meth:`~transformers.TFPreTrainedModel.from_pretrained` method to load the model weights.
+"""
+
+
+BART_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (:obj:`tf.Tensor` of shape :obj:`({0})`):
+            Indices of input sequence tokens in the vocabulary.
+
+            Indices can be obtained using :class:`~transformers.BertTokenizer`.
+            See :meth:`transformers.PreTrainedTokenizer.encode` and
+            :meth:`transformers.PreTrainedTokenizer.__call__` for details.
+
+            `What are input IDs? <../glossary.html#input-ids>`__
+        attention_mask (:obj:`tf.Tensor` of shape :obj:`({0})`, `optional`):
+            Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            `What are attention masks? <../glossary.html#attention-mask>`__
+        decoder_input_ids (:obj:`tf.Tensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
+            Provide for translation and summarization training. By default, the model will create this tensor by
+            shifting the input_ids right, following the paper.
+        decoder_attention_mask (:obj:`tf.Tensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`):
+            will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
+        encoder_outputs (:obj:`tf.FloatTensor`, `optional`):
+            hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
+            of shape :obj:`(batch_size, sequence_length, hidden_size)` is a sequence of
+        past_key_values (:obj:`Tuple[Dict[str: tf.Tensor]]` of length :obj:`config.n_layers`)
+            contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`). Set to :obj:`False` during training, :obj:`True` during generation
+        output_attentions (:obj:`bool`, `optional`):
+            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
+            tensors for more detail.
+        output_hidden_states (:obj:`bool`, `optional`):
+            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
+            more detail.
+        return_dict (:obj:`bool`, `optional`):
+            Whether or not to return a :class:`~transformers.file_utils.TFModelOutput` instead of a plain tuple.
+        training (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether or not to use the model in training mode (some modules like dropout modules have different
+            behaviors between training and evaluation).
+"""
+LARGE_NEGATIVE = -1e8
+
+
+logger = logging.get_logger(__name__)
+@add_start_docstrings(
+    "The BART Model with a language modeling head. Can be used for summarization.",
+    BART_START_DOCSTRING,
+)
+class TFMarianMTModel(TFBartForConditionalGeneration):
+    base_model_prefix = "model"
+    authorized_missing_keys = [
+        r"encoder\.version",
+        r"decoder\.version",
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
+    ]
+    def adjust_logits_during_generation(self, logits, cur_len, max_length):
+        # THIS WILL BREAK IN TF!
+        logits[:, self.config.pad_token_id] = float("-inf")  # never predict pad token.
+        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
+            logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
+        return logits
diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py
index 852f6e0d577e..fa2ede4c3518 100644
--- a/tests/test_modeling_marian.py
+++ b/tests/test_modeling_marian.py
@@ -37,7 +37,7 @@
     from transformers.pipelines import TranslationPipeline
 
 
-@require_torch
+
 class ModelTester:
     def __init__(self, parent):
         self.config = MarianConfig(
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
new file mode 100644
index 000000000000..8bd34ebfe75e
--- /dev/null
+++ b/tests/test_modeling_tf_marian.py
@@ -0,0 +1,223 @@
+# coding=utf-8
+# Copyright 2020 HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from transformers import  AutoTokenizer, MarianConfig, MarianTokenizer, is_tf_available, TranslationPipeline
+from transformers.file_utils import cached_property
+from transformers.hf_api import HfApi
+from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
+from .test_modeling_marian import ModelTester
+
+from .test_modeling_common import ModelTesterMixin
+
+
+if is_tf_available():
+
+    from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
+
+
+class ModelTester:
+    def __init__(self, parent):
+        self.config = MarianConfig(
+            vocab_size=99,
+            d_model=24,
+            encoder_layers=2,
+            decoder_layers=2,
+            encoder_attention_heads=2,
+            decoder_attention_heads=2,
+            encoder_ffn_dim=32,
+            decoder_ffn_dim=32,
+            max_position_embeddings=48,
+            add_final_layer_norm=True,
+            return_dict=True,
+        )
+
+    def prepare_config_and_inputs_for_common(self):
+        return self.config, {}
+
+
+@require_torch
+@require_sentencepiece
+@require_tokenizers
+class MarianIntegrationTest(unittest.TestCase):
+    src = "en"
+    tgt = "de"
+    src_text = [
+        "I am a small frog.",
+        "Now I can forget the 100 words of german that I know.",
+        "Tom asked his teacher for advice.",
+        "That's how I would do it.",
+        "Tom really admired Mary's courage.",
+        "Turn around and close your eyes.",
+    ]
+    expected_text = [
+        "Ich bin ein kleiner Frosch.",
+        "Jetzt kann ich die 100 Wörter des Deutschen vergessen, die ich kenne.",
+        "Tom bat seinen Lehrer um Rat.",
+        "So würde ich das machen.",
+        "Tom bewunderte Marias Mut wirklich.",
+        "Drehen Sie sich um und schließen Sie die Augen.",
+    ]
+    # ^^ actual C++ output differs slightly: (1) des Deutschen removed, (2) ""-> "O", (3) tun -> machen
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.model_name = f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
+        return cls
+
+    @cached_property
+    def tokenizer(self) -> MarianTokenizer:
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @property
+    def eos_token_id(self) -> int:
+        return self.tokenizer.eos_token_id
+
+    @cached_property
+    def model(self):
+        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True).to(torch_device)
+        c = model.config
+        self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
+        self.assertEqual(c.max_length, 512)
+        self.assertEqual(c.decoder_start_token_id, c.pad_token_id)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        self.assertListEqual(self.expected_text, generated_words)
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(src_texts=self.src_text, **tokenizer_kwargs, return_tensors='tf')
+        self.assertEqual(self.model.device, model_inputs.input_ids.device)
+        generated_ids = self.model.generate(
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+        return generated_words
+
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_EN_FR(MarianIntegrationTest):
+    src = "en"
+    tgt = "fr"
+    src_text = [
+        "I am a small frog.",
+        "Now I can forget the 100 words of german that I know.",
+    ]
+    expected_text = [
+        "Je suis une petite grenouille.",
+        "Maintenant, je peux oublier les 100 mots d'allemand que je connais.",
+    ]
+
+    @slow
+    def test_batch_generation_en_fr(self):
+        self._assert_generated_batch_equal_expected()
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_FR_EN(MarianIntegrationTest):
+    src = "fr"
+    tgt = "en"
+    src_text = [
+        "Donnez moi le micro.",
+        "Tom et Mary étaient assis à une table.",  # Accents
+    ]
+    expected_text = [
+        "Give me the microphone.",
+        "Tom and Mary were sitting at a table.",
+    ]
+
+    @slow
+    def test_batch_generation_fr_en(self):
+        self._assert_generated_batch_equal_expected()
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_RU_FR(MarianIntegrationTest):
+    src = "ru"
+    tgt = "fr"
+    src_text = ["Он показал мне рукопись своей новой пьесы."]
+    expected_text = ["Il m'a montré le manuscrit de sa nouvelle pièce."]
+
+    @slow
+    def test_batch_generation_ru_fr(self):
+        self._assert_generated_batch_equal_expected()
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_MT_EN(MarianIntegrationTest):
+    src = "mt"
+    tgt = "en"
+    src_text = ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
+    expected_text = ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]
+
+    @slow
+    def test_batch_generation_mt_en(self):
+        self._assert_generated_batch_equal_expected()
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_en_zh(MarianIntegrationTest):
+    src = "en"
+    tgt = "zh"
+    src_text = ["My name is Wolfgang and I live in Berlin"]
+    expected_text = ["我叫沃尔夫冈 我住在柏林"]
+
+    @slow
+    def test_batch_generation_eng_zho(self):
+        self._assert_generated_batch_equal_expected()
+
+
+@require_sentencepiece
+@require_tokenizers
+class TestMarian_en_ROMANCE(MarianIntegrationTest):
+    """Multilingual on target side."""
+
+    src = "en"
+    tgt = "ROMANCE"
+    src_text = [
+        ">>fr<< Don't spend so much time watching TV.",
+        ">>pt<< Your message has been sent.",
+        ">>es<< He's two years older than me.",
+    ]
+    expected_text = [
+        "Ne passez pas autant de temps à regarder la télé.",
+        "A sua mensagem foi enviada.",
+        "Es dos años más viejo que yo.",
+    ]
+
+    @slow
+    def test_batch_generation_en_ROMANCE_multi(self):
+        self._assert_generated_batch_equal_expected()
+
+    def test_tokenizer_handles_empty(self):
+        normalized = self.tokenizer.normalize("")
+        self.assertIsInstance(normalized, str)
+        with self.assertRaises(ValueError):
+            self.tokenizer.prepare_seq2seq_batch([""])
+
+    def test_pipeline(self):
+        device = 0 if torch_device == "cuda" else -1
+        pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf", device=device)
+        output = pipeline(self.src_text)
+        self.assertEqual(self.expected_text, [x["translation_text"] for x in output])

From c2b3194aabf0826eed4c755cbb26e142c62ceec4 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 13:06:06 -0400
Subject: [PATCH 02/40] Marian close

---
 src/transformers/modeling_bart.py      |   4 +-
 src/transformers/modeling_tf_auto.py   |   1 +
 src/transformers/modeling_tf_bart.py   | 108 +++++++++++++++++----
 src/transformers/modeling_tf_marian.py | 125 ++-----------------------
 tests/test_modeling_tf_marian.py       |   5 +-
 5 files changed, 104 insertions(+), 139 deletions(-)

diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 8b811577eacf..12bb81e83ac0 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -1324,7 +1324,7 @@ def forward(
 class SinusoidalPositionalEmbedding(nn.Embedding):
     """This module produces sinusoidal positional embeddings of any length."""
 
-    def __init__(self, num_positions, embedding_dim, padding_idx=None):
+    def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
         super().__init__(num_positions, embedding_dim)
         if embedding_dim % 2 != 0:
             raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
@@ -1339,7 +1339,7 @@ def _init_weight(out: nn.Parameter):
         position_enc = np.array(
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
-        out[:, 0 : dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
+        out[:, dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
         out[:, dim // 2 :] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
         out.detach_()
         out.requires_grad = False
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index a8ca3ecfa291..707a39feb8fc 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -214,6 +214,7 @@
         (T5Config, TFT5ForConditionalGeneration),
         (DistilBertConfig, TFDistilBertForMaskedLM),
         (AlbertConfig, TFAlbertForMaskedLM),
+        (MarianConfig, TFMarianMTModel),
         (BartConfig, TFBartForConditionalGeneration),
         (CamembertConfig, TFCamembertForMaskedLM),
         (XLMRobertaConfig, TFXLMRobertaForMaskedLM),
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 600044444d1a..71b8280c12d7 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -21,7 +21,7 @@
 
 import tensorflow as tf
 from tensorflow import Tensor
-from tensorflow.keras.layers import Dense, LayerNormalization
+from tensorflow.keras.layers import Dense, LayerNormalization, Dropout, Layer
 
 from .activations_tf import ACT2FN
 from .configuration_bart import BartConfig
@@ -42,7 +42,6 @@
 
 
 _CONFIG_FOR_DOC = "BartConfig"
-_TOKENIZER_FOR_DOC = "BartTokenizer"
 
 BART_START_DOCSTRING = r"""
 
@@ -254,10 +253,10 @@ def __init__(self, config: BartConfig, **kwargs):
             self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"
         )
 
-        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
-        self.dropout_wt = tf.keras.layers.Dropout(config.dropout)
+        self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
+        self.dropout_wt = Dropout(config.dropout)
         self.activation_fn = ACT2FN[config.activation_function]
-        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)
+        self.activation_dropout = Dropout(config.activation_dropout)
         self.fc1 = Dense(config.encoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
         self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
@@ -323,17 +322,16 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
         self.max_source_positions = config.max_position_embeddings
 
         self.embed_tokens = embed_tokens
-        self.embed_positions = TFLearnedPositionalEmbedding(
-            config.max_position_embeddings,
-            embed_tokens.hidden_size,
-            self.padding_idx,
-            config.extra_pos_embeddings,
-            name="embed_positions",
-        )
+        if config.static_position_embeddings:
+            self.embed_positions = TFSinusoidalPositionalEmbedding(config.max_position_embeddings, config.d_model, self.padding_idx, name="embed_positions",)
+        else:
+            self.embed_positions = TFLearnedPositionalEmbedding(
+                config.max_position_embeddings, config.d_model, self.padding_idx, config.extra_pos_embeddings, config.extra_pos_embeddings,
+                name="embed_positions")
         self.layers = [TFEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
-        self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
+        self.layernorm_embedding = LayerNormalization(epsilon=1e-5, name="layernorm_embedding")if config.add_final_layer_norm else tf.keras.layers.Layer()
         self.layer_norm = (
-            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")
+            LayerNormalization(epsilon=1e-5, name="layer_norm")
             if config.add_final_layer_norm
             else None
         )
@@ -520,7 +518,13 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
         self.max_target_positions = config.max_position_embeddings
         self.embed_tokens = embed_tokens
         self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
-        self.embed_positions = TFLearnedPositionalEmbedding(
+        if config.static_position_embeddings:
+            self.embed_positions = TFSinusoidalPositionalEmbedding(
+                config.max_position_embeddings, config.d_model, self.padding_idx,
+                name="embed_positions",
+            )
+        else:
+            self.embed_positions = TFLearnedPositionalEmbedding(
             config.max_position_embeddings,
             config.d_model,
             self.padding_idx,
@@ -531,7 +535,7 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
         self.layernorm_embedding = (
             tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
             if config.normalize_embedding
-            else tf.identity
+            else Layer()
         )
         self.layer_norm = (
             tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")
@@ -809,6 +813,67 @@ def call(self, input_ids: tf.Tensor, use_cache=False):
             positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
         return super().call(positions + self.offset)  # super object is not callable for some reason
 
+import numpy as np
+
+
+class TFSinusoidalPositionalEmbedding(TFSharedEmbeddings):
+    """This module produces sinusoidal positional embeddings of any length."""
+
+    def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
+
+        if embedding_dim % 2 != 0:
+            raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
+        super().__init__(num_positions, embedding_dim, **kwargs)
+        #self.weight = self._init_weight(*self.weight.shape)
+    def build(self, input_shape):
+        """Build shared token embedding layer
+        Shared weights logic adapted from
+            https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
+        """
+        super().build(input_shape)
+        #self.weight = self._init_weight(*self.weight.shape)
+
+
+    @staticmethod
+    def _init_weight( n_pos, dim):
+        """Identical to the XLM create_sinusoidal_embeddings except features are not interleaved.
+        The cos features are in the 2nd half of the vector. [dim // 2:]
+        """
+        position_enc = np.array(
+            [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
+        )
+        # index 0 is all zero
+        position_enc[:, :dim // 2] = np.sin(position_enc[:, 0::2])
+        position_enc[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
+        # convert to tensor
+        table = tf.convert_to_tensor(position_enc, dtype=tf.float32)
+        tf.stop_gradient(table)
+        return table
+
+
+    def call(self, input_ids, use_cache=False):
+        """Input is expected to be of size [bsz x seqlen]."""
+        bsz, seq_len = input_ids.shape[:2]
+        if use_cache:
+            positions = tf.fill((1, 1), seq_len - 1)
+        else:
+            # starts at 0, ends at 1-seq_len
+            positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
+        return super().call(positions)
+
+
+class BiasLayer(tf.keras.layers.Layer):
+    def __init__(self, *args, **kwargs):
+        super(BiasLayer, self).__init__(*args, **kwargs)
+
+    def build(self, input_shape):
+        self.bias = self.add_weight('bias',
+                                    shape=input_shape[1:],
+                                    initializer='zeros',
+                                    trainable=True)
+    def call(self, x):
+        return x + self.bias
+
 
 # Public API
 
@@ -986,6 +1051,7 @@ def __init__(self, config: BartConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         self.model = TFBartModel(config, name="model")
         self.use_cache = config.use_cache
+        self.final_logits_bias = self.add_weight(name="/final_logits_bias", shape=[1, config.vocab_size], initializer='zeros', trainable=True)
 
     @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
     @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
@@ -1076,6 +1142,7 @@ def call(
             return_dict=True,  # TODO(SS): this may need to change to support compilation
         )
         logits = self.model.shared(outputs.last_hidden_state, mode="linear")
+        logits = logits + self.final_logits_bias
         loss = None if labels is None else self.compute_loss(labels, logits)
 
         past = outputs.past_key_values if cast_bool_to_primitive(use_cache, self.config.use_cache) else None
@@ -1149,6 +1216,7 @@ def _reorder_cache(past, beam_idx):
         return past
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
+
         if cur_len == 1 and self.config.force_bos_token_to_be_generated:
             logits = self._force_token_id_to_be_generated(logits, self.config.bos_token_id)
         elif cur_len == max_length - 1 and self.config.eos_token_id is not None:
@@ -1156,15 +1224,17 @@ def adjust_logits_during_generation(self, logits, cur_len, max_length):
         return logits
 
     @staticmethod
-    def _force_token_id_to_be_generated(scores, token_id) -> None:
+    def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
         output_list = []
 
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
         bs, vocab_size = scores.shape
+        inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
         for x in range(vocab_size):
-            if x != token_id:
-                output_list.append(tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype))
+            do_inf = (x==token_id) if inverted else (x != token_id)
+            if do_inf:
+                output_list.append(inf_tensor)
             else:
                 output_list.append(scores[:, x])
         scores = tf.stack(output_list, axis=1, name="scores")
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index fd14f8a5f56a..2f5e4367080e 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -13,127 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """TF BART model, ported from the fairseq repo."""
+from .file_utils import add_start_docstrings
+from .modeling_tf_bart import TFBartForConditionalGeneration, BART_START_DOCSTRING
 
-import math
-import random
-import warnings
-from typing import Dict, Optional, Tuple
-
-import tensorflow as tf
-from tensorflow import Tensor
-from tensorflow.keras.layers import Dense, LayerNormalization
-
-from .activations_tf import ACT2FN
-from .configuration_bart import BartConfig
-from .file_utils import add_start_docstrings, add_start_docstrings_to_callable, replace_return_docstrings
-from .modeling_tf_outputs import TFBaseModelOutput, TFBaseModelOutputWithPast, TFSeq2SeqLMOutput, TFSeq2SeqModelOutput
-from .modeling_tf_bart import TFBartForConditionalGeneration
-# Public API
-from .modeling_tf_utils import (
-    DUMMY_INPUTS,
-    TFPreTrainedModel,
-    TFSharedEmbeddings,
-    cast_bool_to_primitive,
-    keras_serializable,
-    shape_list,
-)
-from .tokenization_utils_base import BatchEncoding
 from .utils import logging
 
 
-_CONFIG_FOR_DOC = "BartConfig"
-_TOKENIZER_FOR_DOC = "BartTokenizer"
-
-BART_START_DOCSTRING = r"""
-
-    This model inherits from :class:`~transformers.TFPreTrainedModel`. Check the superclass documentation for the
-    generic methods the library implements for all its model (such as downloading or saving, resizing the input
-    embeddings, pruning heads etc.)
-
-    This model is also a `tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ subclass.
-    Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general
-    usage and behavior.
-
-    .. note::
-
-        TF 2.0 models accepts two formats as inputs:
-
-        - having all inputs as keyword arguments (like PyTorch models), or
-        - having all inputs as a list, tuple or dict in the first positional arguments.
-
-        This second option is useful when using :meth:`tf.keras.Model.fit` method which currently requires having
-        all the tensors in the first argument of the model call function: :obj:`model(inputs)`.
-
-        If you choose this second option, there are three possibilities you can use to gather all the input Tensors
-        in the first positional argument :
-
-        - a single Tensor with :obj:`input_ids` only and nothing else: :obj:`model(inputs_ids)`
-        - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
-          :obj:`model([input_ids, attention_mask])` or :obj:`model([input_ids, attention_mask, token_type_ids])`
-        - a dictionary with one or several input Tensors associated to the input names given in the docstring:
-          :obj:`model({"input_ids": input_ids, "token_type_ids": token_type_ids})`
-
-    Args:
-        config (:class:`~transformers.BartConfig`): Model configuration class with all the parameters of the model.
-            Initializing with a config file does not load the weights associated with the model, only the configuration.
-            Check out the :meth:`~transformers.TFPreTrainedModel.from_pretrained` method to load the model weights.
-"""
-
-
-BART_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`tf.Tensor` of shape :obj:`({0})`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using :class:`~transformers.BertTokenizer`.
-            See :meth:`transformers.PreTrainedTokenizer.encode` and
-            :meth:`transformers.PreTrainedTokenizer.__call__` for details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        attention_mask (:obj:`tf.Tensor` of shape :obj:`({0})`, `optional`):
-            Mask to avoid performing attention on padding token indices.
-            Mask values selected in ``[0, 1]``:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+_CONFIG_FOR_DOC = "MarianConfig"
 
-            `What are attention masks? <../glossary.html#attention-mask>`__
-        decoder_input_ids (:obj:`tf.Tensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
-            Provide for translation and summarization training. By default, the model will create this tensor by
-            shifting the input_ids right, following the paper.
-        decoder_attention_mask (:obj:`tf.Tensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`):
-            will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
-        encoder_outputs (:obj:`tf.FloatTensor`, `optional`):
-            hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
-            of shape :obj:`(batch_size, sequence_length, hidden_size)` is a sequence of
-        past_key_values (:obj:`Tuple[Dict[str: tf.Tensor]]` of length :obj:`config.n_layers`)
-            contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
-            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
-            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
-            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
-        use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
-            decoding (see :obj:`past_key_values`). Set to :obj:`False` during training, :obj:`True` during generation
-        output_attentions (:obj:`bool`, `optional`):
-            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
-            tensors for more detail.
-        output_hidden_states (:obj:`bool`, `optional`):
-            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
-            more detail.
-        return_dict (:obj:`bool`, `optional`):
-            Whether or not to return a :class:`~transformers.file_utils.TFModelOutput` instead of a plain tuple.
-        training (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether or not to use the model in training mode (some modules like dropout modules have different
-            behaviors between training and evaluation).
-"""
-LARGE_NEGATIVE = -1e8
+START_DOCSTRING = BART_START_DOCSTRING.replace(
+    'inherits from :class:`~transformers.TFPreTrainedModel`',
+    'inherits from :class:`~transformers.TFBartForConditionalGeneration`'
+).replace('BartConfig', _CONFIG_FOR_DOC)
 
 
 logger = logging.get_logger(__name__)
-@add_start_docstrings(
-    "The BART Model with a language modeling head. Can be used for summarization.",
-    BART_START_DOCSTRING,
-)
+@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
     base_model_prefix = "model"
     authorized_missing_keys = [
@@ -143,8 +38,8 @@ class TFMarianMTModel(TFBartForConditionalGeneration):
         "model.decoder.embed_tokens.weight",
     ]
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        # THIS WILL BREAK IN TF!
-        logits[:, self.config.pad_token_id] = float("-inf")  # never predict pad token.
+        print(f'Gen Step: {cur_len}')
+        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
         if cur_len == max_length - 1 and self.config.eos_token_id is not None:
             logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
         return logits
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 8bd34ebfe75e..75122f4a0a88 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -89,7 +89,7 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
-        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True).to(torch_device)
+        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
         c = model.config
         self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
         self.assertEqual(c.max_length, 512)
@@ -217,7 +217,6 @@ def test_tokenizer_handles_empty(self):
             self.tokenizer.prepare_seq2seq_batch([""])
 
     def test_pipeline(self):
-        device = 0 if torch_device == "cuda" else -1
-        pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf", device=device)
+        pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf")
         output = pipeline(self.src_text)
         self.assertEqual(self.expected_text, [x["translation_text"] for x in output])

From 376bf6d726d133c6dc74faf0264ce9f3259f7e81 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 13:20:09 -0400
Subject: [PATCH 03/40] Small stubs for all children

---
 src/transformers/__init__.py            |  2 +-
 src/transformers/generation_tf_utils.py |  1 +
 src/transformers/modeling_tf_auto.py    | 18 ++++---
 src/transformers/modeling_tf_bart.py    | 66 +++++++++++++++----------
 src/transformers/modeling_tf_marian.py  | 18 ++++---
 tests/test_modeling_marian.py           |  1 -
 tests/test_modeling_tf_marian.py        |  9 ++--
 7 files changed, 66 insertions(+), 49 deletions(-)

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index c1c6c6c4fe8d..1fa9f5173fe0 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -542,6 +542,7 @@
         T5PreTrainedModel,
         load_tf_weights_in_t5,
     )
+    from .modeling_tf_marian import TFMarianMTModel
     from .modeling_transfo_xl import (
         TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
         AdaptiveEmbedding,
@@ -592,7 +593,6 @@
         XLNetPreTrainedModel,
         load_tf_weights_in_xlnet,
     )
-    from .modeling_tf_marian import TFMarianMTModel
 
     # Optimization
     from .optimization import (
diff --git a/src/transformers/generation_tf_utils.py b/src/transformers/generation_tf_utils.py
index 6d2e056b667b..e4be6e61d785 100644
--- a/src/transformers/generation_tf_utils.py
+++ b/src/transformers/generation_tf_utils.py
@@ -619,6 +619,7 @@ def _generate_beam_search(
         done = [False for _ in range(batch_size)]
 
         while cur_len < max_length:
+            print(f"cur_len: {cur_len}, generated: {input_ids}")
             model_inputs = self.prepare_inputs_for_generation(
                 input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache
             )
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index 707a39feb8fc..4f343e5ee0c3 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -41,6 +41,10 @@
     XLNetConfig,
     replace_list_option_in_docstrings,
 )
+from .configuration_blenderbot import BlenderbotConfig
+from .configuration_marian import MarianConfig
+from .configuration_mbart import MBartConfig
+from .configuration_pegasus import PegasusConfig
 from .configuration_utils import PretrainedConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_albert import (
@@ -106,14 +110,9 @@
     TFFunnelForTokenClassification,
     TFFunnelModel,
 )
-from .modeling_tf_marian import TFMarianMTModel
-from .configuration_marian import MarianConfig
-from .configuration_pegasus import PegasusConfig
-from .configuration_mbart import MBartConfig
-from .configuration_blenderbot import BlenderbotConfig
-
 from .modeling_tf_gpt2 import TFGPT2LMHeadModel, TFGPT2Model
 from .modeling_tf_longformer import TFLongformerForMaskedLM, TFLongformerForQuestionAnswering, TFLongformerModel
+from .modeling_tf_marian import TFMarianMTModel
 from .modeling_tf_mobilebert import (
     TFMobileBertForMaskedLM,
     TFMobileBertForMultipleChoice,
@@ -267,8 +266,11 @@
 )
 
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
-    [(T5Config, TFT5ForConditionalGeneration), (BartConfig, TFBartForConditionalGeneration),
-     (MarianConfig, TFMarianMTModel)]
+    [
+        (T5Config, TFT5ForConditionalGeneration),
+        (BartConfig, TFBartForConditionalGeneration),
+        (MarianConfig, TFMarianMTModel),
+    ]
 )
 
 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 71b8280c12d7..5749c12e7560 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -21,7 +21,7 @@
 
 import tensorflow as tf
 from tensorflow import Tensor
-from tensorflow.keras.layers import Dense, LayerNormalization, Dropout, Layer
+from tensorflow.keras.layers import Dense, Dropout, Layer, LayerNormalization
 
 from .activations_tf import ACT2FN
 from .configuration_bart import BartConfig
@@ -323,18 +323,28 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
 
         self.embed_tokens = embed_tokens
         if config.static_position_embeddings:
-            self.embed_positions = TFSinusoidalPositionalEmbedding(config.max_position_embeddings, config.d_model, self.padding_idx, name="embed_positions",)
+            self.embed_positions = TFSinusoidalPositionalEmbedding(
+                config.max_position_embeddings,
+                config.d_model,
+                self.padding_idx,
+                name="embed_positions",
+            )
         else:
             self.embed_positions = TFLearnedPositionalEmbedding(
-                config.max_position_embeddings, config.d_model, self.padding_idx, config.extra_pos_embeddings, config.extra_pos_embeddings,
-                name="embed_positions")
+                config.max_position_embeddings,
+                config.d_model,
+                self.padding_idx,
+                config.extra_pos_embeddings,
+                config.extra_pos_embeddings,
+                name="embed_positions",
+            )
         self.layers = [TFEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
-        self.layernorm_embedding = LayerNormalization(epsilon=1e-5, name="layernorm_embedding")if config.add_final_layer_norm else tf.keras.layers.Layer()
-        self.layer_norm = (
-            LayerNormalization(epsilon=1e-5, name="layer_norm")
+        self.layernorm_embedding = (
+            LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
             if config.add_final_layer_norm
-            else None
+            else tf.keras.layers.Layer()
         )
+        self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
         self.return_dict = config.return_dict
 
     def call(
@@ -520,17 +530,19 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
         self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
         if config.static_position_embeddings:
             self.embed_positions = TFSinusoidalPositionalEmbedding(
-                config.max_position_embeddings, config.d_model, self.padding_idx,
+                config.max_position_embeddings,
+                config.d_model,
+                self.padding_idx,
                 name="embed_positions",
             )
         else:
             self.embed_positions = TFLearnedPositionalEmbedding(
-            config.max_position_embeddings,
-            config.d_model,
-            self.padding_idx,
-            config.extra_pos_embeddings,
-            name="embed_positions",
-        )
+                config.max_position_embeddings,
+                config.d_model,
+                self.padding_idx,
+                config.extra_pos_embeddings,
+                name="embed_positions",
+            )
         self.layers = [TFDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
         self.layernorm_embedding = (
             tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
@@ -813,6 +825,7 @@ def call(self, input_ids: tf.Tensor, use_cache=False):
             positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
         return super().call(positions + self.offset)  # super object is not callable for some reason
 
+
 import numpy as np
 
 
@@ -824,18 +837,18 @@ def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
         if embedding_dim % 2 != 0:
             raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
         super().__init__(num_positions, embedding_dim, **kwargs)
-        #self.weight = self._init_weight(*self.weight.shape)
+        # self.weight = self._init_weight(*self.weight.shape)
+
     def build(self, input_shape):
         """Build shared token embedding layer
         Shared weights logic adapted from
             https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         super().build(input_shape)
-        #self.weight = self._init_weight(*self.weight.shape)
-
+        # self.weight = self._init_weight(*self.weight.shape)
 
     @staticmethod
-    def _init_weight( n_pos, dim):
+    def _init_weight(n_pos, dim):
         """Identical to the XLM create_sinusoidal_embeddings except features are not interleaved.
         The cos features are in the 2nd half of the vector. [dim // 2:]
         """
@@ -843,14 +856,13 @@ def _init_weight( n_pos, dim):
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
         # index 0 is all zero
-        position_enc[:, :dim // 2] = np.sin(position_enc[:, 0::2])
+        position_enc[:, : dim // 2] = np.sin(position_enc[:, 0::2])
         position_enc[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
         # convert to tensor
         table = tf.convert_to_tensor(position_enc, dtype=tf.float32)
         tf.stop_gradient(table)
         return table
 
-
     def call(self, input_ids, use_cache=False):
         """Input is expected to be of size [bsz x seqlen]."""
         bsz, seq_len = input_ids.shape[:2]
@@ -867,10 +879,8 @@ def __init__(self, *args, **kwargs):
         super(BiasLayer, self).__init__(*args, **kwargs)
 
     def build(self, input_shape):
-        self.bias = self.add_weight('bias',
-                                    shape=input_shape[1:],
-                                    initializer='zeros',
-                                    trainable=True)
+        self.bias = self.add_weight("bias", shape=input_shape[1:], initializer="zeros", trainable=True)
+
     def call(self, x):
         return x + self.bias
 
@@ -1051,7 +1061,9 @@ def __init__(self, config: BartConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         self.model = TFBartModel(config, name="model")
         self.use_cache = config.use_cache
-        self.final_logits_bias = self.add_weight(name="/final_logits_bias", shape=[1, config.vocab_size], initializer='zeros', trainable=True)
+        self.final_logits_bias = self.add_weight(
+            name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=True
+        )
 
     @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
     @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
@@ -1232,7 +1244,7 @@ def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         bs, vocab_size = scores.shape
         inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
         for x in range(vocab_size):
-            do_inf = (x==token_id) if inverted else (x != token_id)
+            do_inf = (x == token_id) if inverted else (x != token_id)
             if do_inf:
                 output_list.append(inf_tensor)
             else:
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 2f5e4367080e..3deafbff25ae 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -13,32 +13,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """TF BART model, ported from the fairseq repo."""
+from .configuration_marian import MarianConfig
 from .file_utils import add_start_docstrings
-from .modeling_tf_bart import TFBartForConditionalGeneration, BART_START_DOCSTRING
-
+from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
 from .utils import logging
 
 
 _CONFIG_FOR_DOC = "MarianConfig"
 
 START_DOCSTRING = BART_START_DOCSTRING.replace(
-    'inherits from :class:`~transformers.TFPreTrainedModel`',
-    'inherits from :class:`~transformers.TFBartForConditionalGeneration`'
-).replace('BartConfig', _CONFIG_FOR_DOC)
+    "inherits from :class:`~transformers.TFPreTrainedModel`",
+    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
+).replace("BartConfig", _CONFIG_FOR_DOC)
 
 
 logger = logging.get_logger(__name__)
+
+
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
     base_model_prefix = "model"
     authorized_missing_keys = [
-        r"encoder\.version",
-        r"decoder\.version",
         "model.encoder.embed_tokens.weight",
         "model.decoder.embed_tokens.weight",
     ]
+    config_class = MarianConfig
+
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        print(f'Gen Step: {cur_len}')
+        print(f"Gen Step: {cur_len}")
         self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
         if cur_len == max_length - 1 and self.config.eos_token_id is not None:
             logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py
index fa2ede4c3518..f8c30a35fe2f 100644
--- a/tests/test_modeling_marian.py
+++ b/tests/test_modeling_marian.py
@@ -37,7 +37,6 @@
     from transformers.pipelines import TranslationPipeline
 
 
-
 class ModelTester:
     def __init__(self, parent):
         self.config = MarianConfig(
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 75122f4a0a88..38ac65e07ab1 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -16,13 +16,13 @@
 
 import unittest
 
-from transformers import  AutoTokenizer, MarianConfig, MarianTokenizer, is_tf_available, TranslationPipeline
+from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.hf_api import HfApi
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
-from .test_modeling_marian import ModelTester
 
 from .test_modeling_common import ModelTesterMixin
+from .test_modeling_marian import ModelTester
 
 
 if is_tf_available():
@@ -101,7 +101,9 @@ def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
         self.assertListEqual(self.expected_text, generated_words)
 
     def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(src_texts=self.src_text, **tokenizer_kwargs, return_tensors='tf')
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
         self.assertEqual(self.model.device, model_inputs.input_ids.device)
         generated_ids = self.model.generate(
             model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
@@ -110,7 +112,6 @@ def translate_src_text(self, **tokenizer_kwargs):
         return generated_words
 
 
-
 @require_sentencepiece
 @require_tokenizers
 class TestMarian_EN_FR(MarianIntegrationTest):

From 418599cfd92d32c4821dc1e183c31a5a9f2e4125 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 13:49:46 -0400
Subject: [PATCH 04/40] Fixed bart

---
 src/transformers/modeling_bart.py      | 18 ++++++++-
 src/transformers/modeling_tf_bart.py   | 51 +++++++++++++-------------
 src/transformers/modeling_tf_marian.py |  5 ---
 3 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 12bb81e83ac0..063d5f123f16 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -385,6 +385,7 @@ def forward(
 
         if not return_dict:
             return tuple(v for v in [x, encoder_states, all_attentions] if v is not None)
+        print_tensor('encoder_out', x)
         return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions)
 
 
@@ -479,6 +480,21 @@ def forward(
             layer_state,
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
+def print_tensor(msg, t):  # DELEMETME
+    # assert t.shape
+    if t is None:
+        print(f"{msg}: {t}")
+        return
+    ndim = len(t.shape)
+    if ndim == 1:
+        slice = t[:3]
+    elif ndim == 2:
+        slice = t[:3, :3]
+    elif ndim == 3:
+        slice = t[:3, :3, :3]
+    elif ndim == 4:
+        slice = t[:3, :3, :3, :3]
+    print(f"{msg}: {slice}")
 
 class BartDecoder(nn.Module):
     """
@@ -1339,7 +1355,7 @@ def _init_weight(out: nn.Parameter):
         position_enc = np.array(
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
-        out[:, dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
+        out[:, 0:dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
         out[:, dim // 2 :] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
         out.detach_()
         out.requires_grad = False
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 5749c12e7560..1aa03c418a1c 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -332,16 +332,15 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
         else:
             self.embed_positions = TFLearnedPositionalEmbedding(
                 config.max_position_embeddings,
-                config.d_model,
+                embed_tokens.hidden_size,
                 self.padding_idx,
                 config.extra_pos_embeddings,
-                config.extra_pos_embeddings,
                 name="embed_positions",
             )
         self.layers = [TFEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
         self.layernorm_embedding = (
             LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
-            if config.add_final_layer_norm
+            if config.normalize_embedding
             else tf.keras.layers.Layer()
         )
         self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
@@ -511,6 +510,21 @@ def call(
             layer_state,
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
+def print_tensor(msg, t):  # DELEMETME
+    # assert t.shape
+    if t is None:
+        print(f"{msg}: {t}")
+        return
+    ndim = len(t.shape)
+    if ndim == 1:
+        slice = t[:3]
+    elif ndim == 2:
+        slice = t[:3, :3]
+    elif ndim == 3:
+        slice = t[:3, :3, :3]
+    elif ndim == 4:
+        slice = t[:3, :3, :3, :3]
+    print(f"{msg}: {slice}")
 
 class TFBartDecoder(tf.keras.layers.Layer):
     """
@@ -544,16 +558,8 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
                 name="embed_positions",
             )
         self.layers = [TFDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
-        self.layernorm_embedding = (
-            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
-            if config.normalize_embedding
-            else Layer()
-        )
-        self.layer_norm = (
-            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")
-            if config.add_final_layer_norm
-            else None
-        )
+        self.layernorm_embedding = (LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer())
+        self.layer_norm = (LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None)
 
         self.dropout = tf.keras.layers.Dropout(config.dropout)
         self.output_hidden_states = config.output_hidden_states
@@ -588,16 +594,19 @@ def call(
 
         # embed positions
         positions = self.embed_positions(input_ids, use_cache=use_cache)
+        print_tensor('pos_emb', positions)
 
         if use_cache:
             input_ids = input_ids[:, -1:]
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
+        #print_tensor('tok_emb', x)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x) + positions
         else:
             x = self.layernorm_embedding(x + positions)
+        print_tensor('x1', x)
         x = self.dropout(x)
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
@@ -856,7 +865,7 @@ def _init_weight(n_pos, dim):
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
         # index 0 is all zero
-        position_enc[:, : dim // 2] = np.sin(position_enc[:, 0::2])
+        position_enc[:, 0: dim // 2] = np.sin(position_enc[:, 0::2])
         position_enc[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
         # convert to tensor
         table = tf.convert_to_tensor(position_enc, dtype=tf.float32)
@@ -871,20 +880,10 @@ def call(self, input_ids, use_cache=False):
         else:
             # starts at 0, ends at 1-seq_len
             positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
+        print(f'positions: {positions}')
         return super().call(positions)
 
 
-class BiasLayer(tf.keras.layers.Layer):
-    def __init__(self, *args, **kwargs):
-        super(BiasLayer, self).__init__(*args, **kwargs)
-
-    def build(self, input_shape):
-        self.bias = self.add_weight("bias", shape=input_shape[1:], initializer="zeros", trainable=True)
-
-    def call(self, x):
-        return x + self.bias
-
-
 # Public API
 
 
@@ -1184,9 +1183,11 @@ def call(
 
     def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, use_cache=True, **kwargs) -> Dict:
         assert past is not None and len(past) in {1, 2}, f"past has to be an iterable of length 1,2 got {past}"
+
         if len(past) == 1:
             assert isinstance(past[0], tf.Tensor)
             encoder_outputs = TFBaseModelOutput(last_hidden_state=past[0])
+            print_tensor('encoder_out', past[0])
             decoder_cached_states = None
         else:
             assert len(past) == 2
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 3deafbff25ae..305e4b855021 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -32,11 +32,6 @@
 
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
-    base_model_prefix = "model"
-    authorized_missing_keys = [
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
-    ]
     config_class = MarianConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):

From 6cba94f447ff76c840b28fa18c8c34a63a1fc17f Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 14:27:14 -0400
Subject: [PATCH 05/40] marian working

---
 src/transformers/modeling_bart.py    |  6 +++++-
 src/transformers/modeling_tf_bart.py | 14 ++++++++++++--
 tests/test_modeling_tf_marian.py     |  9 +++++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 063d5f123f16..b060a5050dd5 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -348,9 +348,13 @@ def forward(
         # check attention mask and invert
         if attention_mask is not None:
             attention_mask = invert_mask(attention_mask)
-
+        print_tensor('weight', self.embed_tokens.weight)
+        print_tensor('input_ids', input_ids)
+        print(f'embed_scale: {self.embed_scale}')
         inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
+        print_tensor('embedded_tok', inputs_embeds)
         embed_pos = self.embed_positions(input_ids)
+        print_tensor('embedded_pos', embed_pos)
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
         x = F.dropout(x, p=self.dropout, training=self.training)
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 1aa03c418a1c..bdd3c54d21b1 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -316,7 +316,7 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
 
-        embed_dim = embed_tokens.vocab_size
+        embed_dim = embed_tokens.hidden_size
         self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
         self.padding_idx = config.pad_token_id
         self.max_source_positions = config.max_position_embeddings
@@ -382,8 +382,16 @@ def call(
             ), f"expected attention_mask._rank() to be a 2D tensor got {attention_mask._rank()}"
             attention_mask = tf.cast(attention_mask, dtype=tf.float32)
             attention_mask = (1.0 - attention_mask) * LARGE_NEGATIVE
-        inputs_embeds = self.embed_tokens(input_ids)
+
+        #print_tensor('weight', self.embed_tokens._layer.weight)
+        print_tensor('input_ids', input_ids)
+        print(f'embed_scale: {self.embed_scale}')
+        inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
+
+        print_tensor('embedded_tok', inputs_embeds)
         embed_pos = self.embed_positions(input_ids)
+        print_tensor('embedded_pos', embed_pos)
+
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
@@ -1183,11 +1191,13 @@ def call(
 
     def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, use_cache=True, **kwargs) -> Dict:
         assert past is not None and len(past) in {1, 2}, f"past has to be an iterable of length 1,2 got {past}"
+        #print_tensor('encoder_out', past[0])
 
         if len(past) == 1:
             assert isinstance(past[0], tf.Tensor)
             encoder_outputs = TFBaseModelOutput(last_hidden_state=past[0])
             print_tensor('encoder_out', past[0])
+            import ipdb; ipdb.set_trace()
             decoder_cached_states = None
         else:
             assert len(past) == 2
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 38ac65e07ab1..2042d0bf3684 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -89,9 +89,11 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
-        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        model: TFMarianMTModel = TFMarianMTModel.from_pretrained(self.model_name, from_pt=True)
         c = model.config
         self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
+
+        #self.assertEqual(c.model.shared.weight ==)
         self.assertEqual(c.max_length, 512)
         self.assertEqual(c.decoder_start_token_id, c.pad_token_id)
         return model
@@ -104,7 +106,6 @@ def translate_src_text(self, **tokenizer_kwargs):
         model_inputs = self.tokenizer.prepare_seq2seq_batch(
             src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
         )
-        self.assertEqual(self.model.device, model_inputs.input_ids.device)
         generated_ids = self.model.generate(
             model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
         )
@@ -207,7 +208,7 @@ class TestMarian_en_ROMANCE(MarianIntegrationTest):
         "Es dos años más viejo que yo.",
     ]
 
-    @slow
+    #@slow
     def test_batch_generation_en_ROMANCE_multi(self):
         self._assert_generated_batch_equal_expected()
 
@@ -216,7 +217,7 @@ def test_tokenizer_handles_empty(self):
         self.assertIsInstance(normalized, str)
         with self.assertRaises(ValueError):
             self.tokenizer.prepare_seq2seq_batch([""])
-
+    @slow
     def test_pipeline(self):
         pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf")
         output = pipeline(self.src_text)

From 06efa16fd665cdbd98505c106a0ba39b11525591 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 16:12:27 -0400
Subject: [PATCH 06/40] pegasus test is good, but failing

---
 src/transformers/__init__.py         |  3 +++
 src/transformers/modeling_bart.py    | 12 ++++++---
 src/transformers/modeling_tf_auto.py | 14 +++++++++-
 src/transformers/modeling_tf_bart.py | 40 +++++++++++++++++++---------
 tests/test_modeling_mbart.py         | 28 +------------------
 tests/test_modeling_pegasus.py       | 12 ++++-----
 6 files changed, 60 insertions(+), 49 deletions(-)

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 1fa9f5173fe0..2ce7191dd5fa 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -653,6 +653,9 @@
         TFAutoModelForTokenClassification,
         TFAutoModelWithLMHead,
     )
+    from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
+    from .modeling_tf_mbart import TFMBartForConditionalGeneration
+    from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
     from .modeling_tf_bart import TFBartForConditionalGeneration, TFBartModel
     from .modeling_tf_bert import (
         TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index b060a5050dd5..112b90dfdde4 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -431,7 +431,6 @@ def forward(
         output_attentions=False,
     ):
         residual = x
-
         if layer_state is None:
             layer_state = {}
         if self.normalize_before:
@@ -451,7 +450,7 @@ def forward(
         if not self.normalize_before:
             x = self.self_attn_layer_norm(x)
 
-        # Cross attention
+        # Cross-Attention Block
         residual = x
         assert self.encoder_attn.cache_key != self.self_attn.cache_key
         if self.normalize_before:
@@ -593,12 +592,17 @@ def forward(
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
+        print_tensor('input_ids', input_ids)
+        print(f'embed_scale: {self.embed_scale}')
+        print_tensor('embedded_tok', x)
+        print_tensor('embedded_pos', positions)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x)
             x += positions
         else:
             x += positions
             x = self.layernorm_embedding(x)
+        print_tensor('x1', x)
 
         x = F.dropout(x, p=self.dropout, training=self.training)
 
@@ -636,6 +640,8 @@ def forward(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
+            print_tensor(f'decoder layer {idx} output', x)
+
         if self.layer_norm:  # if config.add_final_layer_norm (mBART)
             x = self.layer_norm(x)
 
@@ -643,10 +649,10 @@ def forward(
         if output_hidden_states:
             all_hidden_states = tuple(hidden_state.transpose(0, 1) for hidden_state in all_hidden_states)
         x = x.transpose(0, 1)
+        print_tensor(f'decoder output', x)
         encoder_hidden_states = encoder_hidden_states.transpose(0, 1)
 
         next_cache = next_decoder_cache if use_cache else None
-
         if not return_dict:
             return tuple(v for v in [x, next_cache, all_hidden_states, all_self_attns] if v is not None)
         return BaseModelOutputWithPast(
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index 4f343e5ee0c3..d648dba54fe9 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -264,12 +264,24 @@
         (FunnelConfig, TFFunnelForMaskedLM),
     ]
 )
+from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
+from .modeling_tf_mbart import TFMBartForConditionalGeneration
+from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
+from .configuration_pegasus import PegasusConfig
+from .configuration_mbart import MBartConfig
+from .configuration_blenderbot import BlenderbotConfig
+
 
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
     [
         (T5Config, TFT5ForConditionalGeneration),
-        (BartConfig, TFBartForConditionalGeneration),
         (MarianConfig, TFMarianMTModel),
+        (MBartConfig, TFMBartForConditionalGeneration),
+        (PegasusConfig, TFPegasusForConditionalGeneration),
+        (BlenderbotConfig, TFMBartForConditionalGeneration),
+        (BartConfig, TFBartForConditionalGeneration),
+
+
     ]
 )
 
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index bdd3c54d21b1..85911120464a 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -440,8 +440,9 @@ def __init__(self, config: BartConfig, **kwargs):
         self.dropout = config.dropout
         self.activation_fn = ACT2FN[config.activation_function]
         self.activation_dropout = config.activation_dropout
+        self.normalize_before = config.normalize_before
 
-        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
+        self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
         self.encoder_attn = TFAttention(
             self.embed_dim,
             config.decoder_attention_heads,
@@ -449,10 +450,10 @@ def __init__(self, config: BartConfig, **kwargs):
             encoder_decoder_attention=True,
             name="encoder_attn",
         )
-        self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
+        self.encoder_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
         self.fc1 = Dense(config.decoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
-        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")
+        self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
 
     def call(
         self,
@@ -477,10 +478,12 @@ def call(
 
             Tuple containing, encoded output of shape `(seq_len, batch, embed_dim)`, self_attn_weights, layer_state
         """
+        residual = x  # Make a copy of the input tensor to add later.
         if layer_state is None:
             layer_state = {}
+        if self.normalize_before:
+            x = self.self_attn_layer_norm(x)
 
-        residual = x  # Make a copy of the input tensor to add later.
         # next line mutates layer state and we need a copy of it
         x, self_attn_weights = self.self_attn(
             query=x,
@@ -491,9 +494,12 @@ def call(
         )
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-        x = self.self_attn_layer_norm(x)
+        if not self.normalize_before:
+            x = self.self_attn_layer_norm(x)
+        # Cross-Attention Block
         residual = x
-        # Cross-Attention
+        if self.normalize_before:
+            x = self.encoder_attn_layer_norm(x)
         x, _ = self.encoder_attn(
             query=x,
             key=encoder_hidden_states,
@@ -502,16 +508,19 @@ def call(
         )
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-
-        x = self.encoder_attn_layer_norm(x)
-
+        if not self.normalize_before:
+            x = self.encoder_attn_layer_norm(x)
+        # Fully Connected
         residual = x
+        if self.normalize_before:
+            x = self.final_layer_norm(x)
         x = self.activation_fn(self.fc1(x))
         x = tf.nn.dropout(x, rate=self.activation_dropout if training else 0)
         x = self.fc2(x)
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-        x = self.final_layer_norm(x)
+        if not self.normalize_before:
+            x = self.final_layer_norm(x)
         return (
             x,
             self_attn_weights,
@@ -567,7 +576,7 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
             )
         self.layers = [TFDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
         self.layernorm_embedding = (LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer())
-        self.layer_norm = (LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None)
+        self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
 
         self.dropout = tf.keras.layers.Dropout(config.dropout)
         self.output_hidden_states = config.output_hidden_states
@@ -609,6 +618,10 @@ def call(
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
+        print_tensor('input_ids', input_ids)
+        print(f'embed_scale: {self.embed_scale}')
+        print_tensor('embedded_tok', x)
+        print_tensor('embedded_pos', positions)
         #print_tensor('tok_emb', x)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x) + positions
@@ -651,6 +664,8 @@ def call(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
+            print_tensor(f'decoder layer {idx} output', x)
+
         if self.layer_norm is not None:  # same as if config.add_final_layer_norm
             x = self.layer_norm(x)
 
@@ -664,6 +679,7 @@ def call(
         all_self_attns = list(all_self_attns) if output_attentions else None
 
         x = tf.transpose(x, perm=(1, 0, 2))
+        print_tensor(f'decoder output', x)
         encoder_hidden_states = tf.transpose(encoder_hidden_states, perm=(1, 0, 2))  # could maybe be avoided.
 
         next_cache = (encoder_hidden_states, next_decoder_cache) if use_cache else None
@@ -1197,7 +1213,7 @@ def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask,
             assert isinstance(past[0], tf.Tensor)
             encoder_outputs = TFBaseModelOutput(last_hidden_state=past[0])
             print_tensor('encoder_out', past[0])
-            import ipdb; ipdb.set_trace()
+
             decoder_cached_states = None
         else:
             assert len(past) == 2
diff --git a/tests/test_modeling_mbart.py b/tests/test_modeling_mbart.py
index 29dac21562a1..27b4c1ce35f2 100644
--- a/tests/test_modeling_mbart.py
+++ b/tests/test_modeling_mbart.py
@@ -91,32 +91,6 @@ class MBartEnroIntegrationTest(AbstractSeq2SeqIntegrationTest):
     ]
     expected_src_tokens = [8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2, EN_CODE]
 
-    @slow
-    @unittest.skip("This has been failing since June 20th at least.")
-    def test_enro_forward(self):
-        model = self.model
-        net_input = {
-            "input_ids": _long_tensor(
-                [
-                    [3493, 3060, 621, 104064, 1810, 100, 142, 566, 13158, 6889, 5, 2, 250004],
-                    [64511, 7, 765, 2837, 45188, 297, 4049, 237, 10, 122122, 5, 2, 250004],
-                ]
-            ),
-            "decoder_input_ids": _long_tensor(
-                [
-                    [250020, 31952, 144, 9019, 242307, 21980, 55749, 11, 5, 2, 1, 1],
-                    [250020, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2],
-                ]
-            ),
-        }
-        net_input["attention_mask"] = net_input["input_ids"].ne(1)
-        with torch.no_grad():
-            logits, *other_stuff = model(**net_input)
-
-        expected_slice = torch.tensor([9.0078, 10.1113, 14.4787], device=logits.device, dtype=logits.dtype)
-        result_slice = logits[0, 0, :3]
-        assert_tensors_close(expected_slice, result_slice, atol=TOLERANCE)
-
     @slow
     def test_enro_generate_one(self):
         batch: BatchEncoding = self.tokenizer.prepare_seq2seq_batch(
@@ -128,7 +102,7 @@ def test_enro_generate_one(self):
         # self.assertEqual(self.tgt_text[1], decoded[1])
 
     @slow
-    def test_enro_generate(self):
+    def test_enro_generate_batch(self):
         batch: BatchEncoding = self.tokenizer.prepare_seq2seq_batch(self.src_text).to(torch_device)
         translated_tokens = self.model.generate(**batch)
         decoded = self.tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)
diff --git a/tests/test_modeling_pegasus.py b/tests/test_modeling_pegasus.py
index 6896976fb7a8..33a1d0242eb2 100644
--- a/tests/test_modeling_pegasus.py
+++ b/tests/test_modeling_pegasus.py
@@ -15,6 +15,10 @@
     from transformers import AutoModelForSeq2SeqLM, PegasusConfig, PegasusForConditionalGeneration
 
 XSUM_ENTRY_LONGER = """ The London trio are up for best UK act and best album, as well as getting two nominations in the best song category."We got told like this morning 'Oh I think you're nominated'", said Dappy."And I was like 'Oh yeah, which one?' And now we've got nominated for four awards. I mean, wow!"Bandmate Fazer added: "We thought it's best of us to come down and mingle with everyone and say hello to the cameras. And now we find we've got four nominations."The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn't be too disappointed if they didn't win this time around."At the end of the day we're grateful to be where we are in our careers."If it don't happen then it don't happen - live to fight another day and keep on making albums and hits for the fans."Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers' All These Things That I've Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year's Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border."We just done Edinburgh the other day," said Dappy."We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!" """
+EXPECTED_SUMMARIES = [
+        "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
+        "Pop group N-Dubz have revealed they were surprised to get four nominations for this year's Mobo Awards."
+    ]
 
 set_verbosity(ERROR)
 
@@ -56,11 +60,7 @@ def setUp(self):
 class PegasusXSUMIntegrationTest(AbstractSeq2SeqIntegrationTest):
     checkpoint_name = "google/pegasus-xsum"
     src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
-    tgt_text = [
-        "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
-        "N-Dubz have said they were surprised to get four nominations for this year's Mobo Awards.",
-    ]
-
+    tgt_text = EXPECTED_SUMMARIES
     @cached_property
     def model(self):
         return AutoModelForSeq2SeqLM.from_pretrained(self.checkpoint_name).to(torch_device)
@@ -72,7 +72,7 @@ def test_pegasus_xsum_summary(self):
             torch_device
         )
         assert inputs.input_ids.shape == (2, 421)
-        translated_tokens = self.model.generate(**inputs)
+        translated_tokens = self.model.generate(**inputs, num_beams=2)
         decoded = self.tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)
         assert self.tgt_text == decoded
 

From f8b9cb42541862599c93722942cf323e2bb775aa Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 16:19:27 -0400
Subject: [PATCH 07/40] Checkin tests

---
 src/transformers/modeling_bart.py    |  2 +-
 src/transformers/modeling_tf_bart.py |  2 +-
 tests/test_modeling_tf_blenderbot.py | 81 ++++++++++++++++++++++++++++
 tests/test_modeling_tf_mbart.py      | 67 +++++++++++++++++++++++
 tests/test_modeling_tf_pegasus.py    | 60 +++++++++++++++++++++
 5 files changed, 210 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_modeling_tf_blenderbot.py
 create mode 100644 tests/test_modeling_tf_mbart.py
 create mode 100644 tests/test_modeling_tf_pegasus.py

diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 112b90dfdde4..2caa19804dba 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -640,7 +640,7 @@ def forward(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
-            print_tensor(f'decoder layer {idx} output', x)
+            # print_tensor(f'decoder layer {idx} output', x)
 
         if self.layer_norm:  # if config.add_final_layer_norm (mBART)
             x = self.layer_norm(x)
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 85911120464a..8b905a1437ed 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -664,7 +664,7 @@ def call(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
-            print_tensor(f'decoder layer {idx} output', x)
+            #print_tensor(f'decoder layer {idx} output', x)
 
         if self.layer_norm is not None:  # same as if config.add_final_layer_norm
             x = self.layer_norm(x)
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
new file mode 100644
index 000000000000..6f15ca13f234
--- /dev/null
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -0,0 +1,81 @@
+# coding=utf-8
+# Copyright 2020 HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
+from transformers.file_utils import cached_property
+from transformers.hf_api import HfApi
+from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
+
+from .test_modeling_common import ModelTesterMixin
+from .test_modeling_marian import ModelTester
+
+
+if is_tf_available():
+
+    from transformers import TFAutoModelForSeq2SeqLM
+
+@require_torch
+@require_sentencepiece
+@require_tokenizers
+class TestMbartEnRO(unittest.TestCase):
+    src = "en"
+    tgt = "ro"
+    src_text = [
+        " UN Chief Says There Is No Military Solution in Syria",
+        #""" Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+    ]
+    expected_text = [
+        "Şeful ONU declară că nu există o soluţie militară în Siria",
+        #'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
+    ]
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.model_name = f"facebook/mbart-large-en-ro"
+        return cls
+
+    @cached_property
+    def tokenizer(self) -> MarianTokenizer:
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @property
+    def eos_token_id(self) -> int:
+        return self.tokenizer.eos_token_id
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        self.assertListEqual(self.expected_text, generated_words)
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
+        generated_ids = self.model.generate(
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+        return generated_words
+
+    @slow
+    def test_batch_generation_en_ro(self):
+        self._assert_generated_batch_equal_expected()
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
new file mode 100644
index 000000000000..cd618523c75b
--- /dev/null
+++ b/tests/test_modeling_tf_mbart.py
@@ -0,0 +1,67 @@
+# coding=utf-8
+# Copyright 2020 HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from transformers import AutoTokenizer, is_tf_available
+from transformers.file_utils import cached_property
+from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow
+
+
+if is_tf_available():
+
+    from transformers import TFAutoModelForSeq2SeqLM
+
+@require_tf
+@require_sentencepiece
+@require_tokenizers
+class TestMbartEnRO(unittest.TestCase):
+    src_text = [
+        " UN Chief Says There Is No Military Solution in Syria",
+        #""" Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+    ]
+    expected_text = [
+        "Şeful ONU declară că nu există o soluţie militară în Siria",
+        #'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
+    ]
+    model_name = f"facebook/mbart-large-en-ro"
+
+    @cached_property
+    def tokenizer(self):
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        self.assertListEqual(self.expected_text, generated_words)
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
+        generated_ids = self.model.generate(
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+        return generated_words
+
+    @slow
+    def test_batch_generation_en_ro(self):
+        self._assert_generated_batch_equal_expected()
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
new file mode 100644
index 000000000000..8de846ef8d0a
--- /dev/null
+++ b/tests/test_modeling_tf_pegasus.py
@@ -0,0 +1,60 @@
+# coding=utf-8
+# Copyright 2020 HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from transformers import AutoTokenizer, is_tf_available
+from transformers.file_utils import cached_property
+from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow
+from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER, EXPECTED_SUMMARIES
+
+if is_tf_available():
+    from transformers import TFAutoModelForSeq2SeqLM
+
+@require_tf
+@require_sentencepiece
+@require_tokenizers
+class TFPegasusIntegrationTests(unittest.TestCase):
+    src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
+    expected_text = EXPECTED_SUMMARIES
+    model_name = f"google/pegasus-xsum"
+
+    @cached_property
+    def tokenizer(self):
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        self.assertListEqual(self.expected_text, generated_words)
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
+        generated_ids = self.model.generate(
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2, use_cache=True,
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
+        return generated_words
+
+    @slow
+    def test_batch_generation(self):
+        self._assert_generated_batch_equal_expected()

From 8e692cacae552b896624d6b20144876eccfae3f5 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 16:20:00 -0400
Subject: [PATCH 08/40] More model files

---
 src/transformers/modeling_tf_blenderbot.py | 40 ++++++++++++++++++++++
 src/transformers/modeling_tf_mbart.py      | 35 +++++++++++++++++++
 src/transformers/modeling_tf_pegasus.py    | 36 +++++++++++++++++++
 3 files changed, 111 insertions(+)
 create mode 100644 src/transformers/modeling_tf_blenderbot.py
 create mode 100644 src/transformers/modeling_tf_mbart.py
 create mode 100644 src/transformers/modeling_tf_pegasus.py

diff --git a/src/transformers/modeling_tf_blenderbot.py b/src/transformers/modeling_tf_blenderbot.py
new file mode 100644
index 000000000000..2fce8c7fed7c
--- /dev/null
+++ b/src/transformers/modeling_tf_blenderbot.py
@@ -0,0 +1,40 @@
+# coding=utf-8
+# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TF BART model, ported from the fairseq repo."""
+from .configuration_blenderbot import BlenderbotConfig
+from .file_utils import add_start_docstrings
+from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
+from .utils import logging
+
+_CONFIG_FOR_DOC = "BlenderbotConfig"
+
+START_DOCSTRING = BART_START_DOCSTRING.replace(
+    "inherits from :class:`~transformers.TFPreTrainedModel`",
+    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
+).replace("BartConfig", _CONFIG_FOR_DOC)
+
+
+logger = logging.get_logger(__name__)
+
+
+@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
+class TFBlenderbotForConditionalGeneration(TFBartForConditionalGeneration):
+    config_class = BlenderbotConfig
+
+    def adjust_logits_during_generation(self, logits, cur_len, max_length):
+        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
+        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
+            self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
+        return logits
diff --git a/src/transformers/modeling_tf_mbart.py b/src/transformers/modeling_tf_mbart.py
new file mode 100644
index 000000000000..606146a050d2
--- /dev/null
+++ b/src/transformers/modeling_tf_mbart.py
@@ -0,0 +1,35 @@
+# coding=utf-8
+# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TF BART model, ported from the fairseq repo."""
+from .configuration_mbart import MBartConfig
+from .file_utils import add_start_docstrings
+from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
+from .utils import logging
+
+
+_CONFIG_FOR_DOC = "MBartConfig"
+
+START_DOCSTRING = BART_START_DOCSTRING.replace(
+    "inherits from :class:`~transformers.TFPreTrainedModel`",
+    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
+).replace("BartConfig", _CONFIG_FOR_DOC)
+
+
+logger = logging.get_logger(__name__)
+
+
+@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
+class TFMBartForConditionalGeneration(TFBartForConditionalGeneration):
+    config_class = MBartConfig
diff --git a/src/transformers/modeling_tf_pegasus.py b/src/transformers/modeling_tf_pegasus.py
new file mode 100644
index 000000000000..7cf03801f635
--- /dev/null
+++ b/src/transformers/modeling_tf_pegasus.py
@@ -0,0 +1,36 @@
+# coding=utf-8
+# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TF BART model, ported from the fairseq repo."""
+from .file_utils import add_start_docstrings
+from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
+from .configuration_pegasus import PegasusConfig
+from .utils import logging
+
+
+_CONFIG_FOR_DOC = "PegasusConfig"
+
+START_DOCSTRING = BART_START_DOCSTRING.replace(
+    "inherits from :class:`~transformers.TFPreTrainedModel`",
+    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
+).replace("BartConfig", _CONFIG_FOR_DOC)
+
+
+logger = logging.get_logger(__name__)
+
+
+@add_start_docstrings("Pegasus model for summarization", START_DOCSTRING)
+class TFPegasusForConditionalGeneration(TFBartForConditionalGeneration):
+    config_class = PegasusConfig
+    # All the code is in src/transformers/modeling_tf_bart.py

From 67fa62262192c3d4e85dce2ba23730f4d79cbcfe Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 21 Oct 2020 22:29:28 -0400
Subject: [PATCH 09/40] Subtle marian, pegasus integration test failures

---
 src/transformers/__init__.py               |   6 +-
 src/transformers/modeling_bart.py          |  34 ++++---
 src/transformers/modeling_tf_auto.py       |  12 +--
 src/transformers/modeling_tf_bart.py       |  64 +++++++------
 src/transformers/modeling_tf_blenderbot.py |   1 +
 src/transformers/modeling_tf_pegasus.py    |   2 +-
 tests/test_modeling_bart.py                | 102 ++++++++++-----------
 tests/test_modeling_pegasus.py             |   7 +-
 tests/test_modeling_tf_bart.py             |  33 +++++++
 tests/test_modeling_tf_blenderbot.py       |   3 +-
 tests/test_modeling_tf_marian.py           |   5 +-
 tests/test_modeling_tf_mbart.py            |   5 +-
 tests/test_modeling_tf_pegasus.py          |  12 ++-
 13 files changed, 171 insertions(+), 115 deletions(-)

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 2ce7191dd5fa..80766d1ef843 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -653,9 +653,6 @@
         TFAutoModelForTokenClassification,
         TFAutoModelWithLMHead,
     )
-    from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
-    from .modeling_tf_mbart import TFMBartForConditionalGeneration
-    from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
     from .modeling_tf_bart import TFBartForConditionalGeneration, TFBartModel
     from .modeling_tf_bert import (
         TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
@@ -672,6 +669,7 @@
         TFBertModel,
         TFBertPreTrainedModel,
     )
+    from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
     from .modeling_tf_camembert import (
         TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCamembertForMaskedLM,
@@ -752,6 +750,7 @@
         TFLxmertPreTrainedModel,
         TFLxmertVisualFeatureEncoder,
     )
+    from .modeling_tf_mbart import TFMBartForConditionalGeneration
     from .modeling_tf_mobilebert import (
         TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFMobileBertForMaskedLM,
@@ -773,6 +772,7 @@
         TFOpenAIGPTModel,
         TFOpenAIGPTPreTrainedModel,
     )
+    from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
     from .modeling_tf_roberta import (
         TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFRobertaForMaskedLM,
diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 2caa19804dba..7a74a5c5beb1 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -348,13 +348,13 @@ def forward(
         # check attention mask and invert
         if attention_mask is not None:
             attention_mask = invert_mask(attention_mask)
-        print_tensor('weight', self.embed_tokens.weight)
-        print_tensor('input_ids', input_ids)
-        print(f'embed_scale: {self.embed_scale}')
+        print_tensor("weight", self.embed_tokens.weight)
+        print_tensor("input_ids", input_ids)
+        print(f"embed_scale: {self.embed_scale}")
         inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor('embedded_tok', inputs_embeds)
+        print_tensor("embedded_tok", inputs_embeds)
         embed_pos = self.embed_positions(input_ids)
-        print_tensor('embedded_pos', embed_pos)
+        print_tensor("embedded_pos", embed_pos)
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
         x = F.dropout(x, p=self.dropout, training=self.training)
@@ -364,7 +364,8 @@ def forward(
 
         encoder_states = [] if output_hidden_states else None
         all_attentions = () if output_attentions else None
-        for encoder_layer in self.layers:
+        for i, encoder_layer in enumerate(self.layers):
+            print_tensor(f"encoder layer {i} input", x)
             if output_hidden_states:
                 encoder_states.append(x)
             # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
@@ -389,7 +390,7 @@ def forward(
 
         if not return_dict:
             return tuple(v for v in [x, encoder_states, all_attentions] if v is not None)
-        print_tensor('encoder_out', x)
+        print_tensor("encoder_out", x)
         return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions)
 
 
@@ -483,6 +484,7 @@ def forward(
             layer_state,
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
+
 def print_tensor(msg, t):  # DELEMETME
     # assert t.shape
     if t is None:
@@ -499,6 +501,7 @@ def print_tensor(msg, t):  # DELEMETME
         slice = t[:3, :3, :3, :3]
     print(f"{msg}: {slice}")
 
+
 class BartDecoder(nn.Module):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer
@@ -592,17 +595,17 @@ def forward(
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor('input_ids', input_ids)
-        print(f'embed_scale: {self.embed_scale}')
-        print_tensor('embedded_tok', x)
-        print_tensor('embedded_pos', positions)
+        print_tensor("input_ids", input_ids)
+        print(f"embed_scale: {self.embed_scale}")
+        print_tensor("embedded_tok", x)
+        print_tensor("embedded_pos", positions)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x)
             x += positions
         else:
             x += positions
             x = self.layernorm_embedding(x)
-        print_tensor('x1', x)
+        print_tensor("x1", x)
 
         x = F.dropout(x, p=self.dropout, training=self.training)
 
@@ -649,7 +652,7 @@ def forward(
         if output_hidden_states:
             all_hidden_states = tuple(hidden_state.transpose(0, 1) for hidden_state in all_hidden_states)
         x = x.transpose(0, 1)
-        print_tensor(f'decoder output', x)
+        print_tensor(f"decoder output", x)
         encoder_hidden_states = encoder_hidden_states.transpose(0, 1)
 
         next_cache = next_decoder_cache if use_cache else None
@@ -1114,6 +1117,9 @@ def forward(
     def prepare_inputs_for_generation(
         self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs
     ):
+        import ipdb
+
+        ipdb.set_trace()
         return {
             "input_ids": None,  # encoder_outputs is defined. input_ids not needed
             "encoder_outputs": encoder_outputs,
@@ -1365,7 +1371,7 @@ def _init_weight(out: nn.Parameter):
         position_enc = np.array(
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
-        out[:, 0:dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
+        out[:, 0 : dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
         out[:, dim // 2 :] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
         out.detach_()
         out.requires_grad = False
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index d648dba54fe9..7db08c043bd4 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -264,12 +264,12 @@
         (FunnelConfig, TFFunnelForMaskedLM),
     ]
 )
-from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
-from .modeling_tf_mbart import TFMBartForConditionalGeneration
-from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
-from .configuration_pegasus import PegasusConfig
-from .configuration_mbart import MBartConfig
 from .configuration_blenderbot import BlenderbotConfig
+from .configuration_mbart import MBartConfig
+from .configuration_pegasus import PegasusConfig
+from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
+from .modeling_tf_mbart import TFMBartForConditionalGeneration
+from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
 
 
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
@@ -280,8 +280,6 @@
         (PegasusConfig, TFPegasusForConditionalGeneration),
         (BlenderbotConfig, TFMBartForConditionalGeneration),
         (BartConfig, TFBartForConditionalGeneration),
-
-
     ]
 )
 
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 8b905a1437ed..2ac5b0bf4bd6 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -252,15 +252,15 @@ def __init__(self, config: BartConfig, **kwargs):
         self.self_attn = TFAttention(
             self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"
         )
-
+        self.normalize_before = config.normalize_before
         self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
-        self.dropout_wt = Dropout(config.dropout)
+        self.dropout = Dropout(config.dropout)
         self.activation_fn = ACT2FN[config.activation_function]
         self.activation_dropout = Dropout(config.activation_dropout)
         self.fc1 = Dense(config.encoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
         self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
-        self.normalize_before = config.normalize_before
+
 
     def call(self, x, encoder_padding_mask, training=False):
         """
@@ -279,7 +279,7 @@ def call(self, x, encoder_padding_mask, training=False):
             x = self.self_attn_layer_norm(x)
         x, self_attn_weights = self.self_attn(query=x, key=x, key_padding_mask=encoder_padding_mask)
         assert x.shape == residual.shape, f"Self attn modified the shape of query {residual.shape} to {x.shape}"
-        x = self.dropout_wt(x, training=training)
+        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
         if not self.normalize_before:
             x = self.self_attn_layer_norm(x)
@@ -290,7 +290,7 @@ def call(self, x, encoder_padding_mask, training=False):
         x = self.activation_fn(self.fc1(x))
         x = self.activation_dropout(x, training=training)
         x = self.fc2(x)
-        x = self.dropout_wt(x, training=training)
+        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
         if not self.normalize_before:
             x = self.final_layer_norm(x)
@@ -383,14 +383,14 @@ def call(
             attention_mask = tf.cast(attention_mask, dtype=tf.float32)
             attention_mask = (1.0 - attention_mask) * LARGE_NEGATIVE
 
-        #print_tensor('weight', self.embed_tokens._layer.weight)
-        print_tensor('input_ids', input_ids)
-        print(f'embed_scale: {self.embed_scale}')
+        # print_tensor('weight', self.embed_tokens._layer.weight)
+        print_tensor("input_ids", input_ids)
+        print(f"embed_scale: {self.embed_scale}")
         inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
 
-        print_tensor('embedded_tok', inputs_embeds)
+        print_tensor("embedded_tok", inputs_embeds)
         embed_pos = self.embed_positions(input_ids)
-        print_tensor('embedded_pos', embed_pos)
+        print_tensor("embedded_pos", embed_pos)
 
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
@@ -403,8 +403,8 @@ def call(
         all_attentions = () if output_attentions else None
 
         # encoder layers
-        for encoder_layer in self.layers:
-
+        for i, encoder_layer in enumerate(self.layers):
+            print_tensor(f"encoder layer {i} input", x)
             if output_hidden_states:
                 encoder_states.append(x)
             # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
@@ -527,6 +527,7 @@ def call(
             layer_state,
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
+
 def print_tensor(msg, t):  # DELEMETME
     # assert t.shape
     if t is None:
@@ -543,6 +544,7 @@ def print_tensor(msg, t):  # DELEMETME
         slice = t[:3, :3, :3, :3]
     print(f"{msg}: {slice}")
 
+
 class TFBartDecoder(tf.keras.layers.Layer):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer
@@ -575,7 +577,9 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
                 name="embed_positions",
             )
         self.layers = [TFDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
-        self.layernorm_embedding = (LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer())
+        self.layernorm_embedding = (
+            LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer()
+        )
         self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
 
         self.dropout = tf.keras.layers.Dropout(config.dropout)
@@ -611,23 +615,23 @@ def call(
 
         # embed positions
         positions = self.embed_positions(input_ids, use_cache=use_cache)
-        print_tensor('pos_emb', positions)
+        print_tensor("pos_emb", positions)
 
         if use_cache:
             input_ids = input_ids[:, -1:]
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor('input_ids', input_ids)
-        print(f'embed_scale: {self.embed_scale}')
-        print_tensor('embedded_tok', x)
-        print_tensor('embedded_pos', positions)
-        #print_tensor('tok_emb', x)
+        print_tensor("input_ids", input_ids)
+        print(f"embed_scale: {self.embed_scale}")
+        print_tensor("embedded_tok", x)
+        print_tensor("embedded_pos", positions)
+        # print_tensor('tok_emb', x)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x) + positions
         else:
             x = self.layernorm_embedding(x + positions)
-        print_tensor('x1', x)
+        print_tensor("x1", x)
         x = self.dropout(x)
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
@@ -664,7 +668,7 @@ def call(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
-            #print_tensor(f'decoder layer {idx} output', x)
+            # print_tensor(f'decoder layer {idx} output', x)
 
         if self.layer_norm is not None:  # same as if config.add_final_layer_norm
             x = self.layer_norm(x)
@@ -679,7 +683,7 @@ def call(
         all_self_attns = list(all_self_attns) if output_attentions else None
 
         x = tf.transpose(x, perm=(1, 0, 2))
-        print_tensor(f'decoder output', x)
+        print_tensor(f"decoder output", x)
         encoder_hidden_states = tf.transpose(encoder_hidden_states, perm=(1, 0, 2))  # could maybe be avoided.
 
         next_cache = (encoder_hidden_states, next_decoder_cache) if use_cache else None
@@ -869,7 +873,9 @@ def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
 
         if embedding_dim % 2 != 0:
             raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
-        super().__init__(num_positions, embedding_dim, **kwargs)
+        super().__init__(num_positions, embedding_dim,
+                         #embeddings_initializer="zeros",
+                         **kwargs)
         # self.weight = self._init_weight(*self.weight.shape)
 
     def build(self, input_shape):
@@ -878,7 +884,8 @@ def build(self, input_shape):
             https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         super().build(input_shape)
-        # self.weight = self._init_weight(*self.weight.shape)
+
+        self.weight = self._init_weight(self.vocab_size, self.hidden_size)
 
     @staticmethod
     def _init_weight(n_pos, dim):
@@ -889,7 +896,7 @@ def _init_weight(n_pos, dim):
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
         )
         # index 0 is all zero
-        position_enc[:, 0: dim // 2] = np.sin(position_enc[:, 0::2])
+        position_enc[:, 0 : dim // 2] = np.sin(position_enc[:, 0::2])
         position_enc[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
         # convert to tensor
         table = tf.convert_to_tensor(position_enc, dtype=tf.float32)
@@ -904,7 +911,7 @@ def call(self, input_ids, use_cache=False):
         else:
             # starts at 0, ends at 1-seq_len
             positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
-        print(f'positions: {positions}')
+        print(f"positions: {positions}")
         return super().call(positions)
 
 
@@ -1207,12 +1214,12 @@ def call(
 
     def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, use_cache=True, **kwargs) -> Dict:
         assert past is not None and len(past) in {1, 2}, f"past has to be an iterable of length 1,2 got {past}"
-        #print_tensor('encoder_out', past[0])
+        # print_tensor('encoder_out', past[0])
 
         if len(past) == 1:
             assert isinstance(past[0], tf.Tensor)
             encoder_outputs = TFBaseModelOutput(last_hidden_state=past[0])
-            print_tensor('encoder_out', past[0])
+            print_tensor("encoder_out", past[0])
 
             decoder_cached_states = None
         else:
@@ -1229,6 +1236,7 @@ def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask,
         assert isinstance(
             encoder_outputs, TFBaseModelOutput
         ), "encoder_outputs should be a TFBaseModelOutput, Instead got "
+        # import ipdb; ipdb.set_trace()
         return {
             "inputs": None,  # encoder_outputs is defined. input_ids not needed
             "encoder_outputs": encoder_outputs,
diff --git a/src/transformers/modeling_tf_blenderbot.py b/src/transformers/modeling_tf_blenderbot.py
index 2fce8c7fed7c..ab4942dee3c2 100644
--- a/src/transformers/modeling_tf_blenderbot.py
+++ b/src/transformers/modeling_tf_blenderbot.py
@@ -18,6 +18,7 @@
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
 from .utils import logging
 
+
 _CONFIG_FOR_DOC = "BlenderbotConfig"
 
 START_DOCSTRING = BART_START_DOCSTRING.replace(
diff --git a/src/transformers/modeling_tf_pegasus.py b/src/transformers/modeling_tf_pegasus.py
index 7cf03801f635..db5c434d9a47 100644
--- a/src/transformers/modeling_tf_pegasus.py
+++ b/src/transformers/modeling_tf_pegasus.py
@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """TF BART model, ported from the fairseq repo."""
+from .configuration_pegasus import PegasusConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
-from .configuration_pegasus import PegasusConfig
 from .utils import logging
 
 
diff --git a/tests/test_modeling_bart.py b/tests/test_modeling_bart.py
index 2f085eb4981c..4c72a76a2b8c 100644
--- a/tests/test_modeling_bart.py
+++ b/tests/test_modeling_bart.py
@@ -602,57 +602,6 @@ def test_cnn_summarization_same_as_fairseq(self):
         assert generated_summaries == EXPECTED
 
 
-@require_torch
-class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
-    desired_weights = [
-        [0, 0, 0, 0, 0],
-        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
-        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
-    ]
-
-    def test_positional_emb_cache_logic(self):
-        pad = 1
-        input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device)
-        emb1 = SinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad).to(torch_device)
-        no_cache = emb1(input_ids, use_cache=False)
-        yes_cache = emb1(input_ids, use_cache=True)
-        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
-        self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())
-
-    def test_odd_embed_dim(self):
-        with self.assertRaises(NotImplementedError):
-            SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
-
-        # odd num_positions is allowed
-        SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
-
-    def test_positional_emb_weights_against_marian(self):
-        pad = 1
-        emb1 = SinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(torch_device)
-        weights = emb1.weight.data[:3, :5].tolist()
-        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
-            for j in range(5):
-                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
-
-        # test that forward pass is just a lookup, there is no ignore padding logic
-        input_ids = torch.tensor([[4, 10, pad, pad, pad]], dtype=torch.long, device=torch_device)
-        no_cache_pad_zero = emb1(input_ids)
-        self.assertTrue(
-            torch.allclose(
-                torch.tensor(self.desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3
-            )
-        )
-
-    def test_child_config_equivalence(self):
-        """Test that configs associated with children of BartForConditionalGeneration are identical."""
-        child_classes = [BlenderbotConfig, MBartConfig, MarianConfig, PegasusConfig]
-        parent_keys = BartConfig().to_dict().keys()
-        for c in child_classes:
-            assert c().to_dict().keys() == parent_keys  # traceback is very nice on it's own
-        # check that test is not stupid
-        assert BertConfig().to_dict().keys() != parent_keys
-
-
 @require_torch
 @slow
 class FastIntegrationTests(unittest.TestCase):
@@ -715,3 +664,54 @@ def test_encoder_equiv(self):
         features = self.xsum_1_1_model.get_encoder()(**batch, return_dict=True).last_hidden_state
         expected = [[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]]
         assert_tensors_close(features[0, :3, :3], torch.tensor(expected), atol=1e-3)
+
+
+@require_torch
+class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
+    desired_weights = [
+        [0, 0, 0, 0, 0],
+        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
+        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
+    ]
+
+    def test_positional_emb_cache_logic(self):
+        pad = 1
+        input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device)
+        emb1 = SinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad).to(torch_device)
+        no_cache = emb1(input_ids, use_cache=False)
+        yes_cache = emb1(input_ids, use_cache=True)
+        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
+        self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())
+
+    def test_odd_embed_dim(self):
+        with self.assertRaises(NotImplementedError):
+            SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
+
+        # odd num_positions is allowed
+        SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
+
+    def test_positional_emb_weights_against_marian(self):
+        pad = 1
+        emb1 = SinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(torch_device)
+        weights = emb1.weight.data[:3, :5].tolist()
+        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
+            for j in range(5):
+                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
+
+        # test that forward pass is just a lookup, there is no ignore padding logic
+        input_ids = torch.tensor([[4, 10, pad, pad, pad]], dtype=torch.long, device=torch_device)
+        no_cache_pad_zero = emb1(input_ids)
+        self.assertTrue(
+            torch.allclose(
+                torch.tensor(self.desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3
+            )
+        )
+
+    def test_child_config_equivalence(self):
+        """Test that configs associated with children of BartForConditionalGeneration are identical."""
+        child_classes = [BlenderbotConfig, MBartConfig, MarianConfig, PegasusConfig]
+        parent_keys = BartConfig().to_dict().keys()
+        for c in child_classes:
+            assert c().to_dict().keys() == parent_keys  # traceback is very nice on it's own
+        # check that test is not stupid
+        assert BertConfig().to_dict().keys() != parent_keys
diff --git a/tests/test_modeling_pegasus.py b/tests/test_modeling_pegasus.py
index 33a1d0242eb2..cbc044fa833e 100644
--- a/tests/test_modeling_pegasus.py
+++ b/tests/test_modeling_pegasus.py
@@ -16,9 +16,9 @@
 
 XSUM_ENTRY_LONGER = """ The London trio are up for best UK act and best album, as well as getting two nominations in the best song category."We got told like this morning 'Oh I think you're nominated'", said Dappy."And I was like 'Oh yeah, which one?' And now we've got nominated for four awards. I mean, wow!"Bandmate Fazer added: "We thought it's best of us to come down and mingle with everyone and say hello to the cameras. And now we find we've got four nominations."The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn't be too disappointed if they didn't win this time around."At the end of the day we're grateful to be where we are in our careers."If it don't happen then it don't happen - live to fight another day and keep on making albums and hits for the fans."Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers' All These Things That I've Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year's Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border."We just done Edinburgh the other day," said Dappy."We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!" """
 EXPECTED_SUMMARIES = [
-        "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
-        "Pop group N-Dubz have revealed they were surprised to get four nominations for this year's Mobo Awards."
-    ]
+    "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
+    "Pop group N-Dubz have revealed they were surprised to get four nominations for this year's Mobo Awards.",
+]
 
 set_verbosity(ERROR)
 
@@ -61,6 +61,7 @@ class PegasusXSUMIntegrationTest(AbstractSeq2SeqIntegrationTest):
     checkpoint_name = "google/pegasus-xsum"
     src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
     tgt_text = EXPECTED_SUMMARIES
+
     @cached_property
     def model(self):
         return AutoModelForSeq2SeqLM.from_pretrained(self.checkpoint_name).to(torch_device)
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 91dfbfb48c7a..35a43b279169 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -76,6 +76,7 @@ def prepare_config_and_inputs_for_common(self):
             bos_token_id=self.bos_token_id,
             pad_token_id=self.pad_token_id,
             decoder_start_token_id=self.pad_token_id,
+            static_position_embeddings=True,
         )
         inputs_dict = prepare_bart_inputs_dict(config, input_ids)
         return config, inputs_dict
@@ -355,3 +356,35 @@ def test_encoder_equiv(self):
 
         expected = np.array([[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]])
         assert np.allclose(features[0, :3, :3].numpy(), expected, atol=1e-3)
+
+
+from transformers.modeling_tf_bart import TFSinusoidalPositionalEmbedding
+
+
+@require_tf
+class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
+    desired_weights = [
+        [0, 0, 0, 0, 0],
+        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
+        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
+    ]
+
+    def test_positional_emb_cache_logic(self):
+        pad = 1
+        input_ids = _long_tensor([[4, 10]])
+        emb1 = TFSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad)
+        no_cache = emb1(input_ids, use_cache=False)
+        yes_cache = emb1(input_ids, use_cache=True)
+        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
+        import numpy as np
+
+        np.testing.assert_almost_equal(no_cache[-1].numpy(), yes_cache[0][0].numpy())
+
+    def test_positional_emb_weights_against_marian(self):
+        pad = 1
+        emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad)
+        emb1.build(None)
+        weights = emb1.embeddings.numpy()
+        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
+            for j in range(5):
+                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 6f15ca13f234..7105c3cc67e5 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -29,6 +29,7 @@
 
     from transformers import TFAutoModelForSeq2SeqLM
 
+
 @require_torch
 @require_sentencepiece
 @require_tokenizers
@@ -37,7 +38,7 @@ class TestMbartEnRO(unittest.TestCase):
     tgt = "ro"
     src_text = [
         " UN Chief Says There Is No Military Solution in Syria",
-        #""" Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
     ]
     expected_text = [
         "Şeful ONU declară că nu există o soluţie militară în Siria",
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 2042d0bf3684..420ab798979c 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -93,7 +93,7 @@ def model(self):
         c = model.config
         self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
 
-        #self.assertEqual(c.model.shared.weight ==)
+        # self.assertEqual(c.model.shared.weight ==)
         self.assertEqual(c.max_length, 512)
         self.assertEqual(c.decoder_start_token_id, c.pad_token_id)
         return model
@@ -208,7 +208,7 @@ class TestMarian_en_ROMANCE(MarianIntegrationTest):
         "Es dos años más viejo que yo.",
     ]
 
-    #@slow
+    # @slow
     def test_batch_generation_en_ROMANCE_multi(self):
         self._assert_generated_batch_equal_expected()
 
@@ -217,6 +217,7 @@ def test_tokenizer_handles_empty(self):
         self.assertIsInstance(normalized, str)
         with self.assertRaises(ValueError):
             self.tokenizer.prepare_seq2seq_batch([""])
+
     @slow
     def test_pipeline(self):
         pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf")
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index cd618523c75b..7124f51c7455 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -18,20 +18,21 @@
 
 from transformers import AutoTokenizer, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow
+from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
 
 if is_tf_available():
 
     from transformers import TFAutoModelForSeq2SeqLM
 
+
 @require_tf
 @require_sentencepiece
 @require_tokenizers
 class TestMbartEnRO(unittest.TestCase):
     src_text = [
         " UN Chief Says There Is No Military Solution in Syria",
-        #""" Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
     ]
     expected_text = [
         "Şeful ONU declară că nu există o soluţie militară în Siria",
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index 8de846ef8d0a..a3360eb0d63d 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -18,12 +18,15 @@
 
 from transformers import AutoTokenizer, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow
-from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER, EXPECTED_SUMMARIES
+from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
+
+from .test_modeling_pegasus import EXPECTED_SUMMARIES, PGE_ARTICLE, XSUM_ENTRY_LONGER
+
 
 if is_tf_available():
     from transformers import TFAutoModelForSeq2SeqLM
 
+
 @require_tf
 @require_sentencepiece
 @require_tokenizers
@@ -50,7 +53,10 @@ def translate_src_text(self, **tokenizer_kwargs):
             src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
         )
         generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2, use_cache=True,
+            model_inputs.input_ids,
+            attention_mask=model_inputs.attention_mask,
+            num_beams=2,
+            use_cache=True,
         )
         generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
         return generated_words

From f90307731bcfc5531eed68ff536da0a00a817bca Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 14:40:32 -0400
Subject: [PATCH 10/40] Works well

---
 tests/test_modeling_tf_marian.py | 26 ++++----------------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 420ab798979c..837dbda43f03 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -19,7 +19,7 @@
 from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.hf_api import HfApi
-from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
+from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow, torch_device
 
 from .test_modeling_common import ModelTesterMixin
 from .test_modeling_marian import ModelTester
@@ -30,27 +30,8 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
 
 
-class ModelTester:
-    def __init__(self, parent):
-        self.config = MarianConfig(
-            vocab_size=99,
-            d_model=24,
-            encoder_layers=2,
-            decoder_layers=2,
-            encoder_attention_heads=2,
-            decoder_attention_heads=2,
-            encoder_ffn_dim=32,
-            decoder_ffn_dim=32,
-            max_position_embeddings=48,
-            add_final_layer_norm=True,
-            return_dict=True,
-        )
-
-    def prepare_config_and_inputs_for_common(self):
-        return self.config, {}
-
 
-@require_torch
+@require_tf
 @require_sentencepiece
 @require_tokenizers
 class MarianIntegrationTest(unittest.TestCase):
@@ -89,7 +70,8 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
-        model: TFMarianMTModel = TFMarianMTModel.from_pretrained(self.model_name, from_pt=True)
+        # FIXME: TFAutoModelForSeq2SeqLM
+        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
         c = model.config
         self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
 

From 34e2297ec705bf6a8c6654e51bf80c9230466dc6 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 15:19:55 -0400
Subject: [PATCH 11/40] rm print

---
 docs/source/model_doc/blenderbot.rst |  8 ++++
 docs/source/model_doc/marian.rst     |  6 +++
 docs/source/model_doc/mbart.rst      |  9 +++-
 docs/source/model_doc/pegasus.rst    |  6 +++
 src/transformers/modeling_bart.py    | 40 ++--------------
 src/transformers/modeling_tf_auto.py | 11 ++---
 src/transformers/modeling_tf_bart.py | 70 ++++++---------------------
 tests/test_modeling_tf_bart.py       |  6 +--
 tests/test_modeling_tf_blenderbot.py | 71 ++++++++++------------------
 tests/test_modeling_tf_marian.py     | 17 ++-----
 tests/test_modeling_tf_mbart.py      |  6 +--
 tests/test_modeling_tf_pegasus.py    | 11 +++--
 12 files changed, 91 insertions(+), 170 deletions(-)

diff --git a/docs/source/model_doc/blenderbot.rst b/docs/source/model_doc/blenderbot.rst
index 94988443f04d..eea993ba4928 100644
--- a/docs/source/model_doc/blenderbot.rst
+++ b/docs/source/model_doc/blenderbot.rst
@@ -73,3 +73,11 @@ See :obj:`transformers.BartForConditionalGeneration` for arguments to `forward`
 
 .. autoclass:: transformers.BlenderbotForConditionalGeneration
     :members:
+
+
+TFBlenderbotForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+See :obj:`transformers.TFBartForConditionalGeneration` for arguments to `forward` and `generate`
+
+.. autoclass:: transformers.TFBlenderbotForConditionalGeneration
+    :members:
diff --git a/docs/source/model_doc/marian.rst b/docs/source/model_doc/marian.rst
index 447cefeb16a5..5f80191aeb42 100644
--- a/docs/source/model_doc/marian.rst
+++ b/docs/source/model_doc/marian.rst
@@ -127,3 +127,9 @@ MarianMTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.MarianMTModel
+
+
+TFMarianMTModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MarianMTModel
diff --git a/docs/source/model_doc/mbart.rst b/docs/source/model_doc/mbart.rst
index 63852466afab..95e4910bf09b 100644
--- a/docs/source/model_doc/mbart.rst
+++ b/docs/source/model_doc/mbart.rst
@@ -77,4 +77,11 @@ MBartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.MBartForConditionalGeneration
-    :members: forward
+    :members:
+
+
+TFMBartForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMBartForConditionalGeneration
+    :members:
diff --git a/docs/source/model_doc/pegasus.rst b/docs/source/model_doc/pegasus.rst
index ec679c81d63a..95226a210b0b 100644
--- a/docs/source/model_doc/pegasus.rst
+++ b/docs/source/model_doc/pegasus.rst
@@ -94,3 +94,9 @@ PegasusForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.PegasusForConditionalGeneration
+
+
+TFPegasusForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFPegasusForConditionalGeneration
diff --git a/src/transformers/modeling_bart.py b/src/transformers/modeling_bart.py
index 7a74a5c5beb1..2c63b4ce9557 100644
--- a/src/transformers/modeling_bart.py
+++ b/src/transformers/modeling_bart.py
@@ -348,13 +348,9 @@ def forward(
         # check attention mask and invert
         if attention_mask is not None:
             attention_mask = invert_mask(attention_mask)
-        print_tensor("weight", self.embed_tokens.weight)
-        print_tensor("input_ids", input_ids)
-        print(f"embed_scale: {self.embed_scale}")
+
         inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor("embedded_tok", inputs_embeds)
         embed_pos = self.embed_positions(input_ids)
-        print_tensor("embedded_pos", embed_pos)
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
         x = F.dropout(x, p=self.dropout, training=self.training)
@@ -364,8 +360,7 @@ def forward(
 
         encoder_states = [] if output_hidden_states else None
         all_attentions = () if output_attentions else None
-        for i, encoder_layer in enumerate(self.layers):
-            print_tensor(f"encoder layer {i} input", x)
+        for encoder_layer in self.layers:
             if output_hidden_states:
                 encoder_states.append(x)
             # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
@@ -390,7 +385,6 @@ def forward(
 
         if not return_dict:
             return tuple(v for v in [x, encoder_states, all_attentions] if v is not None)
-        print_tensor("encoder_out", x)
         return BaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions)
 
 
@@ -485,23 +479,6 @@ def forward(
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
 
-def print_tensor(msg, t):  # DELEMETME
-    # assert t.shape
-    if t is None:
-        print(f"{msg}: {t}")
-        return
-    ndim = len(t.shape)
-    if ndim == 1:
-        slice = t[:3]
-    elif ndim == 2:
-        slice = t[:3, :3]
-    elif ndim == 3:
-        slice = t[:3, :3, :3]
-    elif ndim == 4:
-        slice = t[:3, :3, :3, :3]
-    print(f"{msg}: {slice}")
-
-
 class BartDecoder(nn.Module):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer
@@ -595,17 +572,12 @@ def forward(
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor("input_ids", input_ids)
-        print(f"embed_scale: {self.embed_scale}")
-        print_tensor("embedded_tok", x)
-        print_tensor("embedded_pos", positions)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x)
             x += positions
         else:
             x += positions
             x = self.layernorm_embedding(x)
-        print_tensor("x1", x)
 
         x = F.dropout(x, p=self.dropout, training=self.training)
 
@@ -643,8 +615,6 @@ def forward(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
-            # print_tensor(f'decoder layer {idx} output', x)
-
         if self.layer_norm:  # if config.add_final_layer_norm (mBART)
             x = self.layer_norm(x)
 
@@ -652,7 +622,6 @@ def forward(
         if output_hidden_states:
             all_hidden_states = tuple(hidden_state.transpose(0, 1) for hidden_state in all_hidden_states)
         x = x.transpose(0, 1)
-        print_tensor(f"decoder output", x)
         encoder_hidden_states = encoder_hidden_states.transpose(0, 1)
 
         next_cache = next_decoder_cache if use_cache else None
@@ -1117,9 +1086,6 @@ def forward(
     def prepare_inputs_for_generation(
         self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs
     ):
-        import ipdb
-
-        ipdb.set_trace()
         return {
             "input_ids": None,  # encoder_outputs is defined. input_ids not needed
             "encoder_outputs": encoder_outputs,
@@ -1356,7 +1322,7 @@ def forward(
 class SinusoidalPositionalEmbedding(nn.Embedding):
     """This module produces sinusoidal positional embeddings of any length."""
 
-    def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
+    def __init__(self, num_positions, embedding_dim, padding_idx=None):
         super().__init__(num_positions, embedding_dim)
         if embedding_dim % 2 != 0:
             raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
diff --git a/src/transformers/modeling_tf_auto.py b/src/transformers/modeling_tf_auto.py
index 7db08c043bd4..ea6ae19ea4fd 100644
--- a/src/transformers/modeling_tf_auto.py
+++ b/src/transformers/modeling_tf_auto.py
@@ -67,6 +67,7 @@
     TFBertLMHeadModel,
     TFBertModel,
 )
+from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
 from .modeling_tf_camembert import (
     TFCamembertForMaskedLM,
     TFCamembertForMultipleChoice,
@@ -113,6 +114,7 @@
 from .modeling_tf_gpt2 import TFGPT2LMHeadModel, TFGPT2Model
 from .modeling_tf_longformer import TFLongformerForMaskedLM, TFLongformerForQuestionAnswering, TFLongformerModel
 from .modeling_tf_marian import TFMarianMTModel
+from .modeling_tf_mbart import TFMBartForConditionalGeneration
 from .modeling_tf_mobilebert import (
     TFMobileBertForMaskedLM,
     TFMobileBertForMultipleChoice,
@@ -123,6 +125,7 @@
     TFMobileBertModel,
 )
 from .modeling_tf_openai import TFOpenAIGPTLMHeadModel, TFOpenAIGPTModel
+from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
 from .modeling_tf_roberta import (
     TFRobertaForMaskedLM,
     TFRobertaForMultipleChoice,
@@ -264,12 +267,6 @@
         (FunnelConfig, TFFunnelForMaskedLM),
     ]
 )
-from .configuration_blenderbot import BlenderbotConfig
-from .configuration_mbart import MBartConfig
-from .configuration_pegasus import PegasusConfig
-from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
-from .modeling_tf_mbart import TFMBartForConditionalGeneration
-from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
 
 
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
@@ -278,7 +275,7 @@
         (MarianConfig, TFMarianMTModel),
         (MBartConfig, TFMBartForConditionalGeneration),
         (PegasusConfig, TFPegasusForConditionalGeneration),
-        (BlenderbotConfig, TFMBartForConditionalGeneration),
+        (BlenderbotConfig, TFBlenderbotForConditionalGeneration),
         (BartConfig, TFBartForConditionalGeneration),
     ]
 )
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 2ac5b0bf4bd6..f58fcfa6d8d5 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -261,7 +261,6 @@ def __init__(self, config: BartConfig, **kwargs):
         self.fc2 = Dense(self.embed_dim, name="fc2")
         self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
 
-
     def call(self, x, encoder_padding_mask, training=False):
         """
         Args:
@@ -316,8 +315,7 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
 
-        embed_dim = embed_tokens.hidden_size
-        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
+        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
         self.padding_idx = config.pad_token_id
         self.max_source_positions = config.max_position_embeddings
 
@@ -326,22 +324,19 @@ def __init__(self, config: BartConfig, embed_tokens: TFSharedEmbeddings, **kwarg
             self.embed_positions = TFSinusoidalPositionalEmbedding(
                 config.max_position_embeddings,
                 config.d_model,
-                self.padding_idx,
                 name="embed_positions",
             )
         else:
             self.embed_positions = TFLearnedPositionalEmbedding(
                 config.max_position_embeddings,
-                embed_tokens.hidden_size,
+                config.d_model,
                 self.padding_idx,
                 config.extra_pos_embeddings,
                 name="embed_positions",
             )
         self.layers = [TFEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
         self.layernorm_embedding = (
-            LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
-            if config.normalize_embedding
-            else tf.keras.layers.Layer()
+            LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer()
         )
         self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
         self.return_dict = config.return_dict
@@ -382,16 +377,8 @@ def call(
             ), f"expected attention_mask._rank() to be a 2D tensor got {attention_mask._rank()}"
             attention_mask = tf.cast(attention_mask, dtype=tf.float32)
             attention_mask = (1.0 - attention_mask) * LARGE_NEGATIVE
-
-        # print_tensor('weight', self.embed_tokens._layer.weight)
-        print_tensor("input_ids", input_ids)
-        print(f"embed_scale: {self.embed_scale}")
-        inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
-
-        print_tensor("embedded_tok", inputs_embeds)
+        inputs_embeds = self.embed_tokens(input_ids)
         embed_pos = self.embed_positions(input_ids)
-        print_tensor("embedded_pos", embed_pos)
-
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
@@ -403,8 +390,8 @@ def call(
         all_attentions = () if output_attentions else None
 
         # encoder layers
-        for i, encoder_layer in enumerate(self.layers):
-            print_tensor(f"encoder layer {i} input", x)
+        for encoder_layer in self.layers:
+
             if output_hidden_states:
                 encoder_states.append(x)
             # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
@@ -528,23 +515,6 @@ def call(
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
 
-def print_tensor(msg, t):  # DELEMETME
-    # assert t.shape
-    if t is None:
-        print(f"{msg}: {t}")
-        return
-    ndim = len(t.shape)
-    if ndim == 1:
-        slice = t[:3]
-    elif ndim == 2:
-        slice = t[:3, :3]
-    elif ndim == 3:
-        slice = t[:3, :3, :3]
-    elif ndim == 4:
-        slice = t[:3, :3, :3, :3]
-    print(f"{msg}: {slice}")
-
-
 class TFBartDecoder(tf.keras.layers.Layer):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer
@@ -565,7 +535,6 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
             self.embed_positions = TFSinusoidalPositionalEmbedding(
                 config.max_position_embeddings,
                 config.d_model,
-                self.padding_idx,
                 name="embed_positions",
             )
         else:
@@ -615,23 +584,16 @@ def call(
 
         # embed positions
         positions = self.embed_positions(input_ids, use_cache=use_cache)
-        print_tensor("pos_emb", positions)
 
         if use_cache:
             input_ids = input_ids[:, -1:]
             positions = positions[:, -1:]
 
         x = self.embed_tokens(input_ids) * self.embed_scale
-        print_tensor("input_ids", input_ids)
-        print(f"embed_scale: {self.embed_scale}")
-        print_tensor("embedded_tok", x)
-        print_tensor("embedded_pos", positions)
-        # print_tensor('tok_emb', x)
         if self.do_blenderbot_90_layernorm:
             x = self.layernorm_embedding(x) + positions
         else:
             x = self.layernorm_embedding(x + positions)
-        print_tensor("x1", x)
         x = self.dropout(x)
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
@@ -668,8 +630,6 @@ def call(
             if output_attentions:
                 all_self_attns += (layer_self_attn,)
 
-            # print_tensor(f'decoder layer {idx} output', x)
-
         if self.layer_norm is not None:  # same as if config.add_final_layer_norm
             x = self.layer_norm(x)
 
@@ -683,7 +643,6 @@ def call(
         all_self_attns = list(all_self_attns) if output_attentions else None
 
         x = tf.transpose(x, perm=(1, 0, 2))
-        print_tensor(f"decoder output", x)
         encoder_hidden_states = tf.transpose(encoder_hidden_states, perm=(1, 0, 2))  # could maybe be avoided.
 
         next_cache = (encoder_hidden_states, next_decoder_cache) if use_cache else None
@@ -869,13 +828,15 @@ def call(self, input_ids: tf.Tensor, use_cache=False):
 class TFSinusoidalPositionalEmbedding(TFSharedEmbeddings):
     """This module produces sinusoidal positional embeddings of any length."""
 
-    def __init__(self, num_positions, embedding_dim, padding_idx=None, **kwargs):
+    def __init__(self, num_positions, embedding_dim, **kwargs):
 
         if embedding_dim % 2 != 0:
             raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
-        super().__init__(num_positions, embedding_dim,
-                         #embeddings_initializer="zeros",
-                         **kwargs)
+        super().__init__(
+            num_positions,
+            embedding_dim,
+            **kwargs,
+        )
         # self.weight = self._init_weight(*self.weight.shape)
 
     def build(self, input_shape):
@@ -883,9 +844,8 @@ def build(self, input_shape):
         Shared weights logic adapted from
             https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
-        super().build(input_shape)
-
-        self.weight = self._init_weight(self.vocab_size, self.hidden_size)
+        super().build(input_shape)  # Instantiates self.weight so it can be loaded
+        self.weight = self._init_weight(self.vocab_size, self.hidden_size)  # overwrite with good defaults
 
     @staticmethod
     def _init_weight(n_pos, dim):
@@ -1263,7 +1223,6 @@ def _reorder_cache(past, beam_idx):
         return past
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
-
         if cur_len == 1 and self.config.force_bos_token_to_be_generated:
             logits = self._force_token_id_to_be_generated(logits, self.config.bos_token_id)
         elif cur_len == max_length - 1 and self.config.eos_token_id is not None:
@@ -1274,7 +1233,6 @@ def adjust_logits_during_generation(self, logits, cur_len, max_length):
     def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
         output_list = []
-
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
         bs, vocab_size = scores.shape
         inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 35a43b279169..77f489beeaac 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -362,7 +362,7 @@ def test_encoder_equiv(self):
 
 
 @require_tf
-class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
+class TestTFSinusoidalPositionalEmbeddings(unittest.TestCase):
     desired_weights = [
         [0, 0, 0, 0, 0],
         [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
@@ -372,7 +372,7 @@ class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
     def test_positional_emb_cache_logic(self):
         pad = 1
         input_ids = _long_tensor([[4, 10]])
-        emb1 = TFSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad)
+        emb1 = TFSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6)
         no_cache = emb1(input_ids, use_cache=False)
         yes_cache = emb1(input_ids, use_cache=True)
         self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
@@ -382,7 +382,7 @@ def test_positional_emb_cache_logic(self):
 
     def test_positional_emb_weights_against_marian(self):
         pad = 1
-        emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad)
+        emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512)
         emb1.build(None)
         weights = emb1.embeddings.numpy()
         for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 7105c3cc67e5..0c39effe373f 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -16,67 +16,44 @@
 
 import unittest
 
-from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
+from transformers import AutoTokenizer, BlenderbotSmallTokenizer, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.hf_api import HfApi
-from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
+from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
-from .test_modeling_common import ModelTesterMixin
-from .test_modeling_marian import ModelTester
 
-
-if is_tf_available():
-
-    from transformers import TFAutoModelForSeq2SeqLM
-
-
-@require_torch
-@require_sentencepiece
+@require_tf
 @require_tokenizers
-class TestMbartEnRO(unittest.TestCase):
-    src = "en"
-    tgt = "ro"
+class TFBlenderbot90MIntegrationTests(unittest.TestCase):
     src_text = [
-        " UN Chief Says There Is No Military Solution in Syria",
-        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
-    ]
-    expected_text = [
-        "Şeful ONU declară că nu există o soluţie militară în Siria",
-        #'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
+        "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like   i'm going to throw up.\nand why is that?"
     ]
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        cls.model_name = f"facebook/mbart-large-en-ro"
-        return cls
+    model_name = "facebook/blenderbot-90M"
 
     @cached_property
-    def tokenizer(self) -> MarianTokenizer:
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @property
-    def eos_token_id(self) -> int:
-        return self.tokenizer.eos_token_id
+    def tokenizer(self):
+        return BlenderbotSmallTokenizer.from_pretrained(self.model_name)
 
     @cached_property
     def model(self):
         model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
         return model
 
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        self.assertListEqual(self.expected_text, generated_words)
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
+    @slow
+    def test_90_generation_from_long_input(self):
+        model_inputs = self.tokenizer(self.src_text, return_tensors="tf")
         generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+            model_inputs.input_ids,
+            attention_mask=model_inputs.attention_mask,
+            num_beams=2,
+            use_cache=True,
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)[0]
+        assert generated_words in (
+            "i don't know. i just feel like i'm going to throw up. it's not fun.",
+            "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
+            "i'm not sure. i just feel like i've been in a bad situation.",
         )
-        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
-        return generated_words
 
-    @slow
-    def test_batch_generation_en_ro(self):
-        self._assert_generated_batch_equal_expected()
+
+if is_tf_available():
+    from transformers import TFAutoModelForSeq2SeqLM
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 837dbda43f03..2f3b43616499 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -18,11 +18,7 @@
 
 from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.hf_api import HfApi
-from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_tf, slow, torch_device
-
-from .test_modeling_common import ModelTesterMixin
-from .test_modeling_marian import ModelTester
+from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
 
 if is_tf_available():
@@ -30,7 +26,6 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
 
 
-
 @require_tf
 @require_sentencepiece
 @require_tokenizers
@@ -70,12 +65,10 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
-        # FIXME: TFAutoModelForSeq2SeqLM
         model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        assert isinstance(model, TFMarianMTModel)
         c = model.config
         self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
-
-        # self.assertEqual(c.model.shared.weight ==)
         self.assertEqual(c.max_length, 512)
         self.assertEqual(c.decoder_start_token_id, c.pad_token_id)
         return model
@@ -168,7 +161,7 @@ class TestMarian_en_zh(MarianIntegrationTest):
     expected_text = ["我叫沃尔夫冈 我住在柏林"]
 
     @slow
-    def test_batch_generation_eng_zho(self):
+    def test_batch_generation_en_zh(self):
         self._assert_generated_batch_equal_expected()
 
 
@@ -185,9 +178,9 @@ class TestMarian_en_ROMANCE(MarianIntegrationTest):
         ">>es<< He's two years older than me.",
     ]
     expected_text = [
-        "Ne passez pas autant de temps à regarder la télé.",
+        "Ne regardez pas tant de temps à la télé.",
         "A sua mensagem foi enviada.",
-        "Es dos años más viejo que yo.",
+        "Tiene dos años más que yo.",
     ]
 
     # @slow
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index 7124f51c7455..7a5164932631 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -29,16 +29,16 @@
 @require_tf
 @require_sentencepiece
 @require_tokenizers
-class TestMbartEnRO(unittest.TestCase):
+class TestMBartEnRO(unittest.TestCase):
     src_text = [
         " UN Chief Says There Is No Military Solution in Syria",
         # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
     ]
     expected_text = [
         "Şeful ONU declară că nu există o soluţie militară în Siria",
-        #'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
+        # 'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
     ]
-    model_name = f"facebook/mbart-large-en-ro"
+    model_name = "facebook/mbart-large-en-ro"
 
     @cached_property
     def tokenizer(self):
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index a3360eb0d63d..ffec76629d17 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -20,7 +20,7 @@
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
-from .test_modeling_pegasus import EXPECTED_SUMMARIES, PGE_ARTICLE, XSUM_ENTRY_LONGER
+from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER
 
 
 if is_tf_available():
@@ -32,8 +32,11 @@
 @require_tokenizers
 class TFPegasusIntegrationTests(unittest.TestCase):
     src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
-    expected_text = EXPECTED_SUMMARIES
-    model_name = f"google/pegasus-xsum"
+    expected_text = [
+        "California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to reduce the risk of wildfires.",
+        'N-Dubz have revealed they\'re "grateful" to have been nominated for four Mobo Awards.',
+    ]  # differs slightly from pytorch, likely due to numerical differences in linear layers
+    model_name = "google/pegasus-xsum"
 
     @cached_property
     def tokenizer(self):
@@ -46,7 +49,7 @@ def model(self):
 
     def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
         generated_words = self.translate_src_text(**tokenizer_kwargs)
-        self.assertListEqual(self.expected_text, generated_words)
+        assert self.expected_text == generated_words
 
     def translate_src_text(self, **tokenizer_kwargs):
         model_inputs = self.tokenizer.prepare_seq2seq_batch(

From 2e6377ec783d6e7bf9331f9f3d674fc1a30dab19 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 15:53:34 -0400
Subject: [PATCH 12/40] boom boom

---
 .github/workflows/self-scheduled.yml       |  8 +-
 src/transformers/__init__.py               |  2 +-
 src/transformers/modeling_tf_bart.py       | 12 +--
 src/transformers/utils/dummy_tf_objects.py | 36 ++++++++
 tests/test_modeling_mbart.py               |  1 -
 tests/test_modeling_tf_bart.py             | 12 +--
 tests/test_modeling_tf_blenderbot.py       |  4 +-
 tests/test_modeling_tf_marian.py           | 99 ++--------------------
 8 files changed, 57 insertions(+), 117 deletions(-)

diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
index 3d94a77cac69..4b56dfeadb3f 100644
--- a/.github/workflows/self-scheduled.yml
+++ b/.github/workflows/self-scheduled.yml
@@ -59,7 +59,7 @@ jobs:
         RUN_SLOW: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
+        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50
 
     - name: Run examples tests on GPU
       env:
@@ -69,7 +69,7 @@ jobs:
       run: |
         source .env/bin/activate
         pip install -r examples/requirements.txt
-        python -m pytest -n 1 --dist=loadfile -s examples --durations=0
+        python -m pytest -n 1 --dist=loadfile -s examples --durations=50
 
   run_all_tests_torch_and_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
@@ -120,7 +120,7 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50
 
       - name: Run examples tests on GPU
         env:
@@ -130,4 +130,4 @@ jobs:
         run: |
           source .env/bin/activate
           pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s examples --durations=0
+          python -m pytest -n 1 --dist=loadfile -s examples --durations=50
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 03cee316b794..6662b2011e26 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -544,7 +544,6 @@
         T5PreTrainedModel,
         load_tf_weights_in_t5,
     )
-    from .modeling_tf_marian import TFMarianMTModel
     from .modeling_transfo_xl import (
         TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
         AdaptiveEmbedding,
@@ -752,6 +751,7 @@
         TFLxmertPreTrainedModel,
         TFLxmertVisualFeatureEncoder,
     )
+    from .modeling_tf_marian import TFMarianMTModel
     from .modeling_tf_mbart import TFMBartForConditionalGeneration
     from .modeling_tf_mobilebert import (
         TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index f58fcfa6d8d5..f255d6bdcf0a 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -19,6 +19,7 @@
 import warnings
 from typing import Dict, Optional, Tuple
 
+import numpy as np
 import tensorflow as tf
 from tensorflow import Tensor
 from tensorflow.keras.layers import Dense, Dropout, Layer, LayerNormalization
@@ -822,9 +823,6 @@ def call(self, input_ids: tf.Tensor, use_cache=False):
         return super().call(positions + self.offset)  # super object is not callable for some reason
 
 
-import numpy as np
-
-
 class TFSinusoidalPositionalEmbedding(TFSharedEmbeddings):
     """This module produces sinusoidal positional embeddings of any length."""
 
@@ -1174,13 +1172,9 @@ def call(
 
     def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, use_cache=True, **kwargs) -> Dict:
         assert past is not None and len(past) in {1, 2}, f"past has to be an iterable of length 1,2 got {past}"
-        # print_tensor('encoder_out', past[0])
-
         if len(past) == 1:
             assert isinstance(past[0], tf.Tensor)
             encoder_outputs = TFBaseModelOutput(last_hidden_state=past[0])
-            print_tensor("encoder_out", past[0])
-
             decoder_cached_states = None
         else:
             assert len(past) == 2
@@ -1195,8 +1189,7 @@ def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask,
             ), f"decoder cached states must be truthy. got {decoder_cached_states} from the 2nd element of past"
         assert isinstance(
             encoder_outputs, TFBaseModelOutput
-        ), "encoder_outputs should be a TFBaseModelOutput, Instead got "
-        # import ipdb; ipdb.set_trace()
+        ), f"encoder_outputs should be a TFBaseModelOutput, Instead got {type(encoder_outputs)}."
         return {
             "inputs": None,  # encoder_outputs is defined. input_ids not needed
             "encoder_outputs": encoder_outputs,
@@ -1232,6 +1225,7 @@ def adjust_logits_during_generation(self, logits, cur_len, max_length):
     @staticmethod
     def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
+        # See
         output_list = []
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
         bs, vocab_size = scores.shape
diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py
index fb1268a51ab7..21a85e436346 100644
--- a/src/transformers/utils/dummy_tf_objects.py
+++ b/src/transformers/utils/dummy_tf_objects.py
@@ -325,6 +325,15 @@ def from_pretrained(self, *args, **kwargs):
         requires_tf(self)
 
 
+class TFBlenderbotForConditionalGeneration:
+    def __init__(self, *args, **kwargs):
+        requires_tf(self)
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_tf(self)
+
+
 TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 
@@ -797,6 +806,24 @@ def __init__(self, *args, **kwargs):
         requires_tf(self)
 
 
+class TFMarianMTModel:
+    def __init__(self, *args, **kwargs):
+        requires_tf(self)
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_tf(self)
+
+
+class TFMBartForConditionalGeneration:
+    def __init__(self, *args, **kwargs):
+        requires_tf(self)
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_tf(self)
+
+
 TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 
@@ -922,6 +949,15 @@ def from_pretrained(self, *args, **kwargs):
         requires_tf(self)
 
 
+class TFPegasusForConditionalGeneration:
+    def __init__(self, *args, **kwargs):
+        requires_tf(self)
+
+    @classmethod
+    def from_pretrained(self, *args, **kwargs):
+        requires_tf(self)
+
+
 TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 
diff --git a/tests/test_modeling_mbart.py b/tests/test_modeling_mbart.py
index 27b4c1ce35f2..ced627907c83 100644
--- a/tests/test_modeling_mbart.py
+++ b/tests/test_modeling_mbart.py
@@ -4,7 +4,6 @@
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from .test_modeling_bart import TOLERANCE, _long_tensor, assert_tensors_close
 from .test_modeling_common import ModelTesterMixin
 
 
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 77f489beeaac..7954facceb3e 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -17,6 +17,8 @@
 import tempfile
 import unittest
 
+import numpy as np
+
 from transformers import is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_tf, require_torch, slow
@@ -29,6 +31,7 @@
     import tensorflow as tf
 
     from transformers import BartConfig, TFBartForConditionalGeneration, TFBartModel
+    from transformers.modeling_tf_bart import TFSinusoidalPositionalEmbedding
     from transformers.tokenization_bart import BartTokenizer
 
 
@@ -76,7 +79,6 @@ def prepare_config_and_inputs_for_common(self):
             bos_token_id=self.bos_token_id,
             pad_token_id=self.pad_token_id,
             decoder_start_token_id=self.pad_token_id,
-            static_position_embeddings=True,
         )
         inputs_dict = prepare_bart_inputs_dict(config, input_ids)
         return config, inputs_dict
@@ -358,9 +360,6 @@ def test_encoder_equiv(self):
         assert np.allclose(features[0, :3, :3].numpy(), expected, atol=1e-3)
 
 
-from transformers.modeling_tf_bart import TFSinusoidalPositionalEmbedding
-
-
 @require_tf
 class TestTFSinusoidalPositionalEmbeddings(unittest.TestCase):
     desired_weights = [
@@ -370,21 +369,18 @@ class TestTFSinusoidalPositionalEmbeddings(unittest.TestCase):
     ]
 
     def test_positional_emb_cache_logic(self):
-        pad = 1
         input_ids = _long_tensor([[4, 10]])
         emb1 = TFSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6)
         no_cache = emb1(input_ids, use_cache=False)
         yes_cache = emb1(input_ids, use_cache=True)
         self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
-        import numpy as np
 
         np.testing.assert_almost_equal(no_cache[-1].numpy(), yes_cache[0][0].numpy())
 
     def test_positional_emb_weights_against_marian(self):
-        pad = 1
         emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512)
         emb1.build(None)
-        weights = emb1.embeddings.numpy()
+        weights = emb1.weight.numpy()
         for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
             for j in range(5):
                 self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 0c39effe373f..a8119c27098b 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -16,9 +16,9 @@
 
 import unittest
 
-from transformers import AutoTokenizer, BlenderbotSmallTokenizer, is_tf_available
+from transformers import BlenderbotSmallTokenizer, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
+from transformers.testing_utils import require_tf, require_tokenizers, slow
 
 
 @require_tf
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 2f3b43616499..021c708e1137 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -16,7 +16,7 @@
 
 import unittest
 
-from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
+from transformers import AutoTokenizer, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
@@ -26,30 +26,7 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
 
 
-@require_tf
-@require_sentencepiece
-@require_tokenizers
-class MarianIntegrationTest(unittest.TestCase):
-    src = "en"
-    tgt = "de"
-    src_text = [
-        "I am a small frog.",
-        "Now I can forget the 100 words of german that I know.",
-        "Tom asked his teacher for advice.",
-        "That's how I would do it.",
-        "Tom really admired Mary's courage.",
-        "Turn around and close your eyes.",
-    ]
-    expected_text = [
-        "Ich bin ein kleiner Frosch.",
-        "Jetzt kann ich die 100 Wörter des Deutschen vergessen, die ich kenne.",
-        "Tom bat seinen Lehrer um Rat.",
-        "So würde ich das machen.",
-        "Tom bewunderte Marias Mut wirklich.",
-        "Drehen Sie sich um und schließen Sie die Augen.",
-    ]
-    # ^^ actual C++ output differs slightly: (1) des Deutschen removed, (2) ""-> "O", (3) tun -> machen
-
+class AbstractMarianIntegrationTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.model_name = f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
@@ -88,73 +65,10 @@ def translate_src_text(self, **tokenizer_kwargs):
         return generated_words
 
 
+@require_tf
 @require_sentencepiece
 @require_tokenizers
-class TestMarian_EN_FR(MarianIntegrationTest):
-    src = "en"
-    tgt = "fr"
-    src_text = [
-        "I am a small frog.",
-        "Now I can forget the 100 words of german that I know.",
-    ]
-    expected_text = [
-        "Je suis une petite grenouille.",
-        "Maintenant, je peux oublier les 100 mots d'allemand que je connais.",
-    ]
-
-    @slow
-    def test_batch_generation_en_fr(self):
-        self._assert_generated_batch_equal_expected()
-
-
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_FR_EN(MarianIntegrationTest):
-    src = "fr"
-    tgt = "en"
-    src_text = [
-        "Donnez moi le micro.",
-        "Tom et Mary étaient assis à une table.",  # Accents
-    ]
-    expected_text = [
-        "Give me the microphone.",
-        "Tom and Mary were sitting at a table.",
-    ]
-
-    @slow
-    def test_batch_generation_fr_en(self):
-        self._assert_generated_batch_equal_expected()
-
-
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_RU_FR(MarianIntegrationTest):
-    src = "ru"
-    tgt = "fr"
-    src_text = ["Он показал мне рукопись своей новой пьесы."]
-    expected_text = ["Il m'a montré le manuscrit de sa nouvelle pièce."]
-
-    @slow
-    def test_batch_generation_ru_fr(self):
-        self._assert_generated_batch_equal_expected()
-
-
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_MT_EN(MarianIntegrationTest):
-    src = "mt"
-    tgt = "en"
-    src_text = ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
-    expected_text = ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]
-
-    @slow
-    def test_batch_generation_mt_en(self):
-        self._assert_generated_batch_equal_expected()
-
-
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_en_zh(MarianIntegrationTest):
+class TestMarian_en_zh(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "zh"
     src_text = ["My name is Wolfgang and I live in Berlin"]
@@ -165,9 +79,10 @@ def test_batch_generation_en_zh(self):
         self._assert_generated_batch_equal_expected()
 
 
+@require_tf
 @require_sentencepiece
 @require_tokenizers
-class TestMarian_en_ROMANCE(MarianIntegrationTest):
+class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
     """Multilingual on target side."""
 
     src = "en"
@@ -183,7 +98,7 @@ class TestMarian_en_ROMANCE(MarianIntegrationTest):
         "Tiene dos años más que yo.",
     ]
 
-    # @slow
+    @slow
     def test_batch_generation_en_ROMANCE_multi(self):
         self._assert_generated_batch_equal_expected()
 

From 966917481dfc01badf35d4e0dc8edccb07fd5fea Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 16:43:40 -0400
Subject: [PATCH 13/40] Still failing model2doc

---
 docs/source/model_doc/marian.rst     |  2 +-
 src/transformers/modeling_tf_bart.py |  1 -
 tests/test_modeling_tf_bart.py       | 27 ++++++----
 tests/test_modeling_tf_blenderbot.py | 80 ++++++++++++++++++++++++++--
 tests/test_modeling_tf_marian.py     | 71 +++++++++++++++++++++++-
 tests/test_modeling_tf_mbart.py      | 73 +++++++++++++++++++++++--
 tests/test_modeling_tf_pegasus.py    | 75 ++++++++++++++++++++++++--
 7 files changed, 304 insertions(+), 25 deletions(-)

diff --git a/docs/source/model_doc/marian.rst b/docs/source/model_doc/marian.rst
index 5f80191aeb42..16591232bed2 100644
--- a/docs/source/model_doc/marian.rst
+++ b/docs/source/model_doc/marian.rst
@@ -132,4 +132,4 @@ MarianMTModel
 TFMarianMTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.MarianMTModel
+.. autoclass:: transformers.TFMarianMTModel
diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index f255d6bdcf0a..82f247136cce 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -869,7 +869,6 @@ def call(self, input_ids, use_cache=False):
         else:
             # starts at 0, ends at 1-seq_len
             positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
-        print(f"positions: {positions}")
         return super().call(positions)
 
 
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 7954facceb3e..3a66ac6a5e1e 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -37,6 +37,9 @@
 
 @require_tf
 class ModelTester:
+    kwargs = {}
+    hidden_act = "gelu"
+
     def __init__(self, parent):
         self.parent = parent
         self.batch_size = 13
@@ -48,7 +51,7 @@ def __init__(self, parent):
         self.num_hidden_layers = 5
         self.num_attention_heads = 4
         self.intermediate_size = 37
-        self.hidden_act = "gelu"
+
         self.hidden_dropout_prob = 0.1
         self.attention_probs_dropout_prob = 0.1
         self.max_position_embeddings = 20
@@ -79,6 +82,7 @@ def prepare_config_and_inputs_for_common(self):
             bos_token_id=self.bos_token_id,
             pad_token_id=self.pad_token_id,
             decoder_start_token_id=self.pad_token_id,
+            **self.kwargs,
         )
         inputs_dict = prepare_bart_inputs_dict(config, input_ids)
         return config, inputs_dict
@@ -104,9 +108,10 @@ class TestTFBart(TFModelTesterMixin, unittest.TestCase):
     all_generative_model_classes = (TFBartForConditionalGeneration,) if is_tf_available() else ()
     is_encoder_decoder = True
     test_pruning = False
+    model_tester_cls = ModelTester
 
     def setUp(self):
-        self.model_tester = ModelTester(self)
+        self.model_tester = self.model_tester_cls(self)
         self.config_tester = ConfigTester(self, config_class=BartConfig)
 
     def test_config(self):
@@ -116,6 +121,14 @@ def test_inputs_embeds(self):
         # inputs_embeds not supported
         pass
 
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
     def test_compile_tf_model(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
 
@@ -123,7 +136,7 @@ def test_compile_tf_model(self):
         loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
         metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
 
-        model_class = TFBartForConditionalGeneration
+        model_class = self.all_generative_model_classes[0]
         input_ids = {
             "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
             "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
@@ -147,14 +160,6 @@ def test_compile_tf_model(self):
         extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
 
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
 
 @require_tf
 class TFBartHeadTests(unittest.TestCase):
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index a8119c27098b..b8a20e273c77 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -12,15 +12,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
+import tempfile
 import unittest
 
-from transformers import BlenderbotSmallTokenizer, is_tf_available
+import tensorflow as tf
+
+from tests.test_configuration_common import ConfigTester
+from tests.test_modeling_tf_bart import ModelTester
+from tests.test_modeling_tf_common import TFModelTesterMixin
+from transformers import BlenderbotConfig, BlenderbotSmallTokenizer, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_tf, require_tokenizers, slow
 
 
+if is_tf_available():
+    from transformers import TFBlenderbotForConditionalGeneration
+
+
 @require_tf
 @require_tokenizers
 class TFBlenderbot90MIntegrationTests(unittest.TestCase):
@@ -57,3 +65,69 @@ def test_90_generation_from_long_input(self):
 
 if is_tf_available():
     from transformers import TFAutoModelForSeq2SeqLM
+
+
+class BlenderbotModelTester(ModelTester):
+    kwargs = dict(
+        normalize_before=True,
+        static_position_embeddings=True,
+        do_blenderbot_90_layernorm=True,
+        normalize_embeddings=True,
+    )
+
+
+@require_tf
+class TestTFBlenderbotCommon(TFModelTesterMixin, unittest.TestCase):
+    all_model_classes = (TFBlenderbotForConditionalGeneration,)
+    all_generative_model_classes = (TFBlenderbotForConditionalGeneration,)
+    model_tester_cls = BlenderbotModelTester
+
+    def setUp(self):
+        self.model_tester = self.model_tester_cls(self)
+        self.config_tester = ConfigTester(self, config_class=BlenderbotConfig)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_inputs_embeds(self):
+        # inputs_embeds not supported
+        pass
+
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_compile_tf_model(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
+        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
+
+        model_class = self.all_generative_model_classes[0]
+        input_ids = {
+            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
+            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
+        }
+
+        # Prepare our model
+        model = model_class(config)
+        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
+        # Let's load it from the disk to be sure we can use pretrained weights
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            model.save_pretrained(tmpdirname)
+            model = model_class.from_pretrained(tmpdirname)
+
+        outputs_dict = model(input_ids)
+        hidden_states = outputs_dict[0]
+
+        # Add a dense layer on top to test integration with other keras modules
+        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
+
+        # Compile extended model
+        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
+        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 021c708e1137..85ed026e7bb0 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import tempfile
 import unittest
 
 from transformers import AutoTokenizer, MarianTokenizer, TranslationPipeline, is_tf_available
@@ -23,7 +23,74 @@
 
 if is_tf_available():
 
-    from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
+    import tensorflow as tf
+
+    from transformers import MarianConfig, TFAutoModelForSeq2SeqLM, TFMarianMTModel
+
+    from .test_configuration_common import ConfigTester
+    from .test_modeling_tf_bart import ModelTester
+    from .test_modeling_tf_common import TFModelTesterMixin
+
+
+class MarianModelTester(ModelTester):
+    kwargs = dict(static_position_embeddings=True, add_bias_logits=True)
+
+
+@require_tf
+class TestTFMarianCommon(TFModelTesterMixin, unittest.TestCase):
+    all_model_classes = (TFMarianMTModel,)
+    all_generative_model_classes = (TFMarianMTModel,)
+    model_tester_cls = MarianModelTester
+
+    def setUp(self):
+        self.model_tester = self.model_tester_cls(self)
+        self.config_tester = ConfigTester(self, config_class=MarianConfig)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_inputs_embeds(self):
+        # inputs_embeds not supported
+        pass
+
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_compile_tf_model(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
+        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
+
+        model_class = self.all_generative_model_classes[0]
+        input_ids = {
+            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
+            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
+        }
+
+        # Prepare our model
+        model = model_class(config)
+        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
+        # Let's load it from the disk to be sure we can use pretrained weights
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            model.save_pretrained(tmpdirname)
+            model = model_class.from_pretrained(tmpdirname)
+
+        outputs_dict = model(input_ids)
+        hidden_states = outputs_dict[0]
+
+        # Add a dense layer on top to test integration with other keras modules
+        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
+
+        # Compile extended model
+        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
+        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
 
 
 class AbstractMarianIntegrationTest(unittest.TestCase):
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index 7a5164932631..40a702cb0034 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -12,18 +12,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
+import tempfile
 import unittest
 
-from transformers import AutoTokenizer, is_tf_available
+import tensorflow as tf
+
+from tests.test_configuration_common import ConfigTester
+from tests.test_modeling_tf_bart import ModelTester
+from tests.test_modeling_tf_common import TFModelTesterMixin
+from transformers import AutoTokenizer, MBartConfig, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
 
 if is_tf_available():
 
-    from transformers import TFAutoModelForSeq2SeqLM
+    from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
 
 
 @require_tf
@@ -66,3 +70,64 @@ def translate_src_text(self, **tokenizer_kwargs):
     @slow
     def test_batch_generation_en_ro(self):
         self._assert_generated_batch_equal_expected()
+
+
+class MbartModelTester(ModelTester):
+    kwargs = dict(normalize_before=True, add_final_layer_norm=True)
+
+
+@require_tf
+class TestTFMBartCommon(TFModelTesterMixin, unittest.TestCase):
+    all_model_classes = (TFMBartForConditionalGeneration,)
+    all_generative_model_classes = (TFMBartForConditionalGeneration,)
+    model_tester_cls = MbartModelTester
+
+    def setUp(self):
+        self.model_tester = self.model_tester_cls(self)
+        self.config_tester = ConfigTester(self, config_class=MBartConfig)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_inputs_embeds(self):
+        # inputs_embeds not supported
+        pass
+
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_compile_tf_model(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
+        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
+
+        model_class = self.all_generative_model_classes[0]
+        input_ids = {
+            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
+            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
+        }
+
+        # Prepare our model
+        model = model_class(config)
+        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
+        # Let's load it from the disk to be sure we can use pretrained weights
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            model.save_pretrained(tmpdirname)
+            model = model_class.from_pretrained(tmpdirname)
+
+        outputs_dict = model(input_ids)
+        hidden_states = outputs_dict[0]
+
+        # Add a dense layer on top to test integration with other keras modules
+        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
+
+        # Compile extended model
+        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
+        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index ffec76629d17..d1cedb80b144 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -12,19 +12,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
+import tempfile
 import unittest
 
+import tensorflow as tf
+
 from transformers import AutoTokenizer, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
+from .test_configuration_common import ConfigTester
 from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER
+from .test_modeling_tf_bart import ModelTester
+from .test_modeling_tf_common import TFModelTesterMixin
 
 
 if is_tf_available():
-    from transformers import TFAutoModelForSeq2SeqLM
+    from transformers import PegasusConfig, TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
 
 
 @require_tf
@@ -67,3 +71,68 @@ def translate_src_text(self, **tokenizer_kwargs):
     @slow
     def test_batch_generation(self):
         self._assert_generated_batch_equal_expected()
+
+
+class PegasusModelTester(ModelTester):
+    kwargs = dict(
+        normalize_before=True,
+        static_position_embeddings=True,
+    )
+    hidden_act = "relu"
+
+
+@require_tf
+class TestTFPegasusCommon(TFModelTesterMixin, unittest.TestCase):
+    all_model_classes = (TFPegasusForConditionalGeneration,)
+    all_generative_model_classes = (TFPegasusForConditionalGeneration,)
+    model_tester_cls = PegasusModelTester
+
+    def setUp(self):
+        self.model_tester = self.model_tester_cls(self)
+        self.config_tester = ConfigTester(self, config_class=PegasusConfig)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_inputs_embeds(self):
+        # inputs_embeds not supported
+        pass
+
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_compile_tf_model(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+
+        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
+        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
+
+        model_class = self.all_generative_model_classes[0]
+        input_ids = {
+            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
+            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
+        }
+
+        # Prepare our model
+        model = model_class(config)
+        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
+        # Let's load it from the disk to be sure we can use pretrained weights
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            model.save_pretrained(tmpdirname)
+            model = model_class.from_pretrained(tmpdirname)
+
+        outputs_dict = model(input_ids)
+        hidden_states = outputs_dict[0]
+
+        # Add a dense layer on top to test integration with other keras modules
+        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
+
+        # Compile extended model
+        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
+        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])

From 92fc836125ae8a5dd453a88e4db2bc379e70ab58 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 16:44:37 -0400
Subject: [PATCH 14/40] merge master

---
 .github/workflows/self-scheduled.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
index 4b56dfeadb3f..3d94a77cac69 100644
--- a/.github/workflows/self-scheduled.yml
+++ b/.github/workflows/self-scheduled.yml
@@ -59,7 +59,7 @@ jobs:
         RUN_SLOW: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50
+        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
 
     - name: Run examples tests on GPU
       env:
@@ -69,7 +69,7 @@ jobs:
       run: |
         source .env/bin/activate
         pip install -r examples/requirements.txt
-        python -m pytest -n 1 --dist=loadfile -s examples --durations=50
+        python -m pytest -n 1 --dist=loadfile -s examples --durations=0
 
   run_all_tests_torch_and_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
@@ -120,7 +120,7 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=50
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
 
       - name: Run examples tests on GPU
         env:
@@ -130,4 +130,4 @@ jobs:
         run: |
           source .env/bin/activate
           pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s examples --durations=50
+          python -m pytest -n 1 --dist=loadfile -s examples --durations=0

From 6ad935b5532ae7f39764cc472eacdeb2d186515c Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Thu, 22 Oct 2020 17:29:47 -0400
Subject: [PATCH 15/40] Equivalence test failing, all others fixed

---
 src/transformers/modeling_tf_marian.py  | 9 +++++++++
 src/transformers/modeling_tf_pegasus.py | 9 +++++++++
 tests/test_modeling_tf_bart.py          | 4 ++--
 tests/test_modeling_tf_blenderbot.py    | 3 +++
 tests/test_modeling_tf_marian.py        | 4 +++-
 tests/test_modeling_tf_mbart.py         | 7 +++++--
 tests/test_modeling_tf_pegasus.py       | 3 +++
 7 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 305e4b855021..d9e2b02cf933 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -32,6 +32,15 @@
 
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
+    authorized_missing_keys = [
+        r"final_logits_bias",
+        r"encoder\.version",
+        r"decoder\.version",
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
+        "model.encoder.embed_positions.weight",
+        "model.decoder.embed_positions.weight",
+    ]
     config_class = MarianConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
diff --git a/src/transformers/modeling_tf_pegasus.py b/src/transformers/modeling_tf_pegasus.py
index db5c434d9a47..40f83a4614af 100644
--- a/src/transformers/modeling_tf_pegasus.py
+++ b/src/transformers/modeling_tf_pegasus.py
@@ -32,5 +32,14 @@
 
 @add_start_docstrings("Pegasus model for summarization", START_DOCSTRING)
 class TFPegasusForConditionalGeneration(TFBartForConditionalGeneration):
+    authorized_missing_keys = [
+        r"final_logits_bias",
+        r"encoder\.version",
+        r"decoder\.version",
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
+        "model.encoder.embed_positions.weight",
+        "model.decoder.embed_positions.weight",
+    ]
     config_class = PegasusConfig
     # All the code is in src/transformers/modeling_tf_bart.py
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 3a66ac6a5e1e..6bf3120babd3 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -39,6 +39,7 @@
 class ModelTester:
     kwargs = {}
     hidden_act = "gelu"
+    config_cls = BartConfig
 
     def __init__(self, parent):
         self.parent = parent
@@ -58,7 +59,6 @@ def __init__(self, parent):
         self.eos_token_ids = [2]
         self.pad_token_id = 1
         self.bos_token_id = 0
-        # torch.manual_seed(0)
 
     def prepare_config_and_inputs_for_common(self):
         input_ids = ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size)
@@ -66,7 +66,7 @@ def prepare_config_and_inputs_for_common(self):
         input_ids = tf.concat([input_ids, eos_tensor], axis=1)
         input_ids = tf.clip_by_value(input_ids, 3, self.vocab_size + 1)
 
-        config = BartConfig(
+        config = self.config_cls(
             vocab_size=self.vocab_size,
             d_model=self.hidden_size,
             encoder_layers=self.num_hidden_layers,
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index b8a20e273c77..d7e7a294f983 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -74,6 +74,7 @@ class BlenderbotModelTester(ModelTester):
         do_blenderbot_90_layernorm=True,
         normalize_embeddings=True,
     )
+    config_cls = BlenderbotConfig
 
 
 @require_tf
@@ -81,6 +82,8 @@ class TestTFBlenderbotCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFBlenderbotForConditionalGeneration,)
     all_generative_model_classes = (TFBlenderbotForConditionalGeneration,)
     model_tester_cls = BlenderbotModelTester
+    is_encoder_decoder = True
+    test_pruning = False
 
     def setUp(self):
         self.model_tester = self.model_tester_cls(self)
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 85ed026e7bb0..fd7056722a72 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -34,6 +34,7 @@
 
 class MarianModelTester(ModelTester):
     kwargs = dict(static_position_embeddings=True, add_bias_logits=True)
+    config_cls = MarianConfig
 
 
 @require_tf
@@ -41,6 +42,8 @@ class TestTFMarianCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFMarianMTModel,)
     all_generative_model_classes = (TFMarianMTModel,)
     model_tester_cls = MarianModelTester
+    is_encoder_decoder = True
+    test_pruning = False
 
     def setUp(self):
         self.model_tester = self.model_tester_cls(self)
@@ -58,7 +61,6 @@ def test_saved_model_with_hidden_states_output(self):
         pass
 
     def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
         pass
 
     def test_compile_tf_model(self):
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index 40a702cb0034..a213ccdcdabb 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -72,15 +72,18 @@ def test_batch_generation_en_ro(self):
         self._assert_generated_batch_equal_expected()
 
 
-class MbartModelTester(ModelTester):
+class MBartModelTester(ModelTester):
     kwargs = dict(normalize_before=True, add_final_layer_norm=True)
+    config_cls = MBartConfig
 
 
 @require_tf
 class TestTFMBartCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFMBartForConditionalGeneration,)
     all_generative_model_classes = (TFMBartForConditionalGeneration,)
-    model_tester_cls = MbartModelTester
+    model_tester_cls = MBartModelTester
+    is_encoder_decoder = True
+    test_pruning = False
 
     def setUp(self):
         self.model_tester = self.model_tester_cls(self)
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index d1cedb80b144..50619824031f 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -79,6 +79,7 @@ class PegasusModelTester(ModelTester):
         static_position_embeddings=True,
     )
     hidden_act = "relu"
+    config_cls = PegasusConfig
 
 
 @require_tf
@@ -86,6 +87,8 @@ class TestTFPegasusCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFPegasusForConditionalGeneration,)
     all_generative_model_classes = (TFPegasusForConditionalGeneration,)
     model_tester_cls = PegasusModelTester
+    is_encoder_decoder = True
+    test_pruning = False
 
     def setUp(self):
         self.model_tester = self.model_tester_cls(self)

From 5976bc7c58eb57a4ed07f6aed6bf9662ab7ce626 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 12:22:15 -0400
Subject: [PATCH 16/40] cleanup

---
 tests/test_modeling_tf_blenderbot.py | 71 +++++++++++++---------------
 tests/test_modeling_tf_marian.py     |  4 +-
 2 files changed, 36 insertions(+), 39 deletions(-)

diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index d7e7a294f983..75c7402aafde 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -27,43 +27,6 @@
 
 if is_tf_available():
     from transformers import TFBlenderbotForConditionalGeneration
-
-
-@require_tf
-@require_tokenizers
-class TFBlenderbot90MIntegrationTests(unittest.TestCase):
-    src_text = [
-        "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like   i'm going to throw up.\nand why is that?"
-    ]
-    model_name = "facebook/blenderbot-90M"
-
-    @cached_property
-    def tokenizer(self):
-        return BlenderbotSmallTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    @slow
-    def test_90_generation_from_long_input(self):
-        model_inputs = self.tokenizer(self.src_text, return_tensors="tf")
-        generated_ids = self.model.generate(
-            model_inputs.input_ids,
-            attention_mask=model_inputs.attention_mask,
-            num_beams=2,
-            use_cache=True,
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)[0]
-        assert generated_words in (
-            "i don't know. i just feel like i'm going to throw up. it's not fun.",
-            "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
-            "i'm not sure. i just feel like i've been in a bad situation.",
-        )
-
-
-if is_tf_available():
     from transformers import TFAutoModelForSeq2SeqLM
 
 
@@ -134,3 +97,37 @@ def test_compile_tf_model(self):
         # Compile extended model
         extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
+
+
+@require_tf
+@require_tokenizers
+class TFBlenderbot90MIntegrationTests(unittest.TestCase):
+    src_text = [
+        "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like   i'm going to throw up.\nand why is that?"
+    ]
+    model_name = "facebook/blenderbot-90M"
+
+    @cached_property
+    def tokenizer(self):
+        return BlenderbotSmallTokenizer.from_pretrained(self.model_name)
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    @slow
+    def test_90_generation_from_long_input(self):
+        model_inputs = self.tokenizer(self.src_text, return_tensors="tf")
+        generated_ids = self.model.generate(
+            model_inputs.input_ids,
+            attention_mask=model_inputs.attention_mask,
+            num_beams=2,
+            use_cache=True,
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)[0]
+        assert generated_words in (
+            "i don't know. i just feel like i'm going to throw up. it's not fun.",
+            "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
+            "i'm not sure. i just feel like i've been in a bad situation.",
+        )
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index fd7056722a72..aa02b160e665 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -79,7 +79,7 @@ def test_compile_tf_model(self):
         # Prepare our model
         model = model_class(config)
         model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
-        # Let's load it from the disk to be sure we can use pretrained weights
+        # Let's load it from the disk to be sure we can use pre-trained weights
         with tempfile.TemporaryDirectory() as tmpdirname:
             model.save_pretrained(tmpdirname)
             model = model_class.from_pretrained(tmpdirname)
@@ -130,7 +130,7 @@ def translate_src_text(self, **tokenizer_kwargs):
         generated_ids = self.model.generate(
             model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
         )
-        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
         return generated_words
 
 

From 7dde6d91daa008b3c4d55feb7018060c97007f80 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 12:50:28 -0400
Subject: [PATCH 17/40] Fix embed_scale

---
 src/transformers/modeling_tf_bart.py | 4 ++--
 tests/test_modeling_tf_blenderbot.py | 3 +--
 utils/check_repo.py                  | 3 ++-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index dafa24738dd3..0b1cc1b3725f 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -349,7 +349,7 @@ def call(
             ), f"expected attention_mask._rank() to be a 2D tensor got {attention_mask._rank()}"
             attention_mask = tf.cast(attention_mask, dtype=tf.float32)
             attention_mask = (1.0 - attention_mask) * LARGE_NEGATIVE
-        inputs_embeds = self.embed_tokens(input_ids)
+        inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
         embed_pos = self.embed_positions(input_ids)
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
@@ -1196,7 +1196,7 @@ def adjust_logits_during_generation(self, logits, cur_len, max_length):
     @staticmethod
     def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
-        # See
+        # TODO: https://github.com/huggingface/transformers/issues/7954
         output_list = []
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
         bs, vocab_size = scores.shape
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 75c7402aafde..519c45db1b42 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -26,8 +26,7 @@
 
 
 if is_tf_available():
-    from transformers import TFBlenderbotForConditionalGeneration
-    from transformers import TFAutoModelForSeq2SeqLM
+    from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration
 
 
 class BlenderbotModelTester(ModelTester):
diff --git a/utils/check_repo.py b/utils/check_repo.py
index 69dfd9ef0d40..99132ab02e6a 100644
--- a/utils/check_repo.py
+++ b/utils/check_repo.py
@@ -67,6 +67,7 @@
     "xlm_prophetnet": "xlmprophetnet.rst",
     "xlm_roberta": "xlmroberta.rst",
     "bert_generation": "bertgeneration.rst",
+    "marian": "marian.rst",
 }
 
 # This is to make sure the transformers module imported is the one in the repo.
@@ -148,7 +149,6 @@ def get_model_doc_files():
     _ignore_modules = [
         "auto",
         "dialogpt",
-        "marian",
         "retribert",
     ]
     doc_files = []
@@ -245,6 +245,7 @@ def check_models_are_documented(module, doc_file):
 def _get_model_name(module):
     """ Get the model name for the module defining it."""
     splits = module.__name__.split("_")
+
     # Secial case for transfo_xl
     if splits[-1] == "xl":
         return "_".join(splits[-2:])

From 87b1ad6f7bc9017f3de9ed4efa76f2a337614ecb Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 12:58:51 -0400
Subject: [PATCH 18/40] Cleanup marian pipeline test

---
 src/transformers/generation_tf_utils.py |  1 -
 src/transformers/modeling_tf_marian.py  |  1 -
 tests/test_modeling_tf_marian.py        | 12 +++---------
 3 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/transformers/generation_tf_utils.py b/src/transformers/generation_tf_utils.py
index e4be6e61d785..6d2e056b667b 100644
--- a/src/transformers/generation_tf_utils.py
+++ b/src/transformers/generation_tf_utils.py
@@ -619,7 +619,6 @@ def _generate_beam_search(
         done = [False for _ in range(batch_size)]
 
         while cur_len < max_length:
-            print(f"cur_len: {cur_len}, generated: {input_ids}")
             model_inputs = self.prepare_inputs_for_generation(
                 input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache
             )
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index d9e2b02cf933..92a8901fcfcc 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -44,7 +44,6 @@ class TFMarianMTModel(TFBartForConditionalGeneration):
     config_class = MarianConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        print(f"Gen Step: {cur_len}")
         self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
         if cur_len == max_length - 1 and self.config.eos_token_id is not None:
             logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index aa02b160e665..692f1f581e6a 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -141,7 +141,7 @@ class TestMarian_en_zh(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "zh"
     src_text = ["My name is Wolfgang and I live in Berlin"]
-    expected_text = ["我叫沃尔夫冈 我住在柏林"]
+    expected_text = ['我的名字是沃尔夫冈 我住在柏林']
 
     @slow
     def test_batch_generation_en_zh(self):
@@ -157,12 +157,12 @@ class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "ROMANCE"
     src_text = [
-        ">>fr<< Don't spend so much time watching TV.",
+        #">>fr<< Don't spend so much time watching TV.",
         ">>pt<< Your message has been sent.",
         ">>es<< He's two years older than me.",
     ]
     expected_text = [
-        "Ne regardez pas tant de temps à la télé.",
+        #"Ne regardez pas tant de temps à la télé.",
         "A sua mensagem foi enviada.",
         "Tiene dos años más que yo.",
     ]
@@ -171,12 +171,6 @@ class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
     def test_batch_generation_en_ROMANCE_multi(self):
         self._assert_generated_batch_equal_expected()
 
-    def test_tokenizer_handles_empty(self):
-        normalized = self.tokenizer.normalize("")
-        self.assertIsInstance(normalized, str)
-        with self.assertRaises(ValueError):
-            self.tokenizer.prepare_seq2seq_batch([""])
-
     @slow
     def test_pipeline(self):
         pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf")

From 91f642ba637c0a7182d1e2726be263a14208b048 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:01:29 -0400
Subject: [PATCH 19/40] Undo extra changes

---
 tests/test_modeling_bart.py      | 102 +++++++++++++++----------------
 tests/test_modeling_tf_marian.py |   4 +-
 2 files changed, 52 insertions(+), 54 deletions(-)

diff --git a/tests/test_modeling_bart.py b/tests/test_modeling_bart.py
index 4c72a76a2b8c..2f085eb4981c 100644
--- a/tests/test_modeling_bart.py
+++ b/tests/test_modeling_bart.py
@@ -602,6 +602,57 @@ def test_cnn_summarization_same_as_fairseq(self):
         assert generated_summaries == EXPECTED
 
 
+@require_torch
+class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
+    desired_weights = [
+        [0, 0, 0, 0, 0],
+        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
+        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
+    ]
+
+    def test_positional_emb_cache_logic(self):
+        pad = 1
+        input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device)
+        emb1 = SinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad).to(torch_device)
+        no_cache = emb1(input_ids, use_cache=False)
+        yes_cache = emb1(input_ids, use_cache=True)
+        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
+        self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())
+
+    def test_odd_embed_dim(self):
+        with self.assertRaises(NotImplementedError):
+            SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
+
+        # odd num_positions is allowed
+        SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
+
+    def test_positional_emb_weights_against_marian(self):
+        pad = 1
+        emb1 = SinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(torch_device)
+        weights = emb1.weight.data[:3, :5].tolist()
+        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
+            for j in range(5):
+                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
+
+        # test that forward pass is just a lookup, there is no ignore padding logic
+        input_ids = torch.tensor([[4, 10, pad, pad, pad]], dtype=torch.long, device=torch_device)
+        no_cache_pad_zero = emb1(input_ids)
+        self.assertTrue(
+            torch.allclose(
+                torch.tensor(self.desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3
+            )
+        )
+
+    def test_child_config_equivalence(self):
+        """Test that configs associated with children of BartForConditionalGeneration are identical."""
+        child_classes = [BlenderbotConfig, MBartConfig, MarianConfig, PegasusConfig]
+        parent_keys = BartConfig().to_dict().keys()
+        for c in child_classes:
+            assert c().to_dict().keys() == parent_keys  # traceback is very nice on it's own
+        # check that test is not stupid
+        assert BertConfig().to_dict().keys() != parent_keys
+
+
 @require_torch
 @slow
 class FastIntegrationTests(unittest.TestCase):
@@ -664,54 +715,3 @@ def test_encoder_equiv(self):
         features = self.xsum_1_1_model.get_encoder()(**batch, return_dict=True).last_hidden_state
         expected = [[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]]
         assert_tensors_close(features[0, :3, :3], torch.tensor(expected), atol=1e-3)
-
-
-@require_torch
-class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
-    desired_weights = [
-        [0, 0, 0, 0, 0],
-        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
-        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
-    ]
-
-    def test_positional_emb_cache_logic(self):
-        pad = 1
-        input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device)
-        emb1 = SinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad).to(torch_device)
-        no_cache = emb1(input_ids, use_cache=False)
-        yes_cache = emb1(input_ids, use_cache=True)
-        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
-        self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())
-
-    def test_odd_embed_dim(self):
-        with self.assertRaises(NotImplementedError):
-            SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
-
-        # odd num_positions is allowed
-        SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
-
-    def test_positional_emb_weights_against_marian(self):
-        pad = 1
-        emb1 = SinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(torch_device)
-        weights = emb1.weight.data[:3, :5].tolist()
-        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
-            for j in range(5):
-                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
-
-        # test that forward pass is just a lookup, there is no ignore padding logic
-        input_ids = torch.tensor([[4, 10, pad, pad, pad]], dtype=torch.long, device=torch_device)
-        no_cache_pad_zero = emb1(input_ids)
-        self.assertTrue(
-            torch.allclose(
-                torch.tensor(self.desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3
-            )
-        )
-
-    def test_child_config_equivalence(self):
-        """Test that configs associated with children of BartForConditionalGeneration are identical."""
-        child_classes = [BlenderbotConfig, MBartConfig, MarianConfig, PegasusConfig]
-        parent_keys = BartConfig().to_dict().keys()
-        for c in child_classes:
-            assert c().to_dict().keys() == parent_keys  # traceback is very nice on it's own
-        # check that test is not stupid
-        assert BertConfig().to_dict().keys() != parent_keys
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 692f1f581e6a..ac73a3d49a42 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -141,7 +141,7 @@ class TestMarian_en_zh(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "zh"
     src_text = ["My name is Wolfgang and I live in Berlin"]
-    expected_text = ['我的名字是沃尔夫冈 我住在柏林']
+    expected_text = ["我的名字是沃尔夫冈 我住在柏林"]
 
     @slow
     def test_batch_generation_en_zh(self):
@@ -157,12 +157,10 @@ class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "ROMANCE"
     src_text = [
-        #">>fr<< Don't spend so much time watching TV.",
         ">>pt<< Your message has been sent.",
         ">>es<< He's two years older than me.",
     ]
     expected_text = [
-        #"Ne regardez pas tant de temps à la télé.",
         "A sua mensagem foi enviada.",
         "Tiene dos años más que yo.",
     ]

From e0f06e81d6a9343261bf3c4dcdcfb9fc16fe2794 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:06:35 -0400
Subject: [PATCH 20/40] Smaller delta

---
 tests/test_modeling_marian.py  |  1 +
 tests/test_modeling_mbart.py   | 29 ++++++++++++++++++++++++++++-
 tests/test_modeling_pegasus.py | 11 +++++------
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py
index 3859f4348248..20975f4bf555 100644
--- a/tests/test_modeling_marian.py
+++ b/tests/test_modeling_marian.py
@@ -37,6 +37,7 @@
     from transformers.pipelines import TranslationPipeline
 
 
+@require_torch
 class ModelTester:
     def __init__(self, parent):
         self.config = MarianConfig(
diff --git a/tests/test_modeling_mbart.py b/tests/test_modeling_mbart.py
index ced627907c83..29dac21562a1 100644
--- a/tests/test_modeling_mbart.py
+++ b/tests/test_modeling_mbart.py
@@ -4,6 +4,7 @@
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
+from .test_modeling_bart import TOLERANCE, _long_tensor, assert_tensors_close
 from .test_modeling_common import ModelTesterMixin
 
 
@@ -90,6 +91,32 @@ class MBartEnroIntegrationTest(AbstractSeq2SeqIntegrationTest):
     ]
     expected_src_tokens = [8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2, EN_CODE]
 
+    @slow
+    @unittest.skip("This has been failing since June 20th at least.")
+    def test_enro_forward(self):
+        model = self.model
+        net_input = {
+            "input_ids": _long_tensor(
+                [
+                    [3493, 3060, 621, 104064, 1810, 100, 142, 566, 13158, 6889, 5, 2, 250004],
+                    [64511, 7, 765, 2837, 45188, 297, 4049, 237, 10, 122122, 5, 2, 250004],
+                ]
+            ),
+            "decoder_input_ids": _long_tensor(
+                [
+                    [250020, 31952, 144, 9019, 242307, 21980, 55749, 11, 5, 2, 1, 1],
+                    [250020, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2],
+                ]
+            ),
+        }
+        net_input["attention_mask"] = net_input["input_ids"].ne(1)
+        with torch.no_grad():
+            logits, *other_stuff = model(**net_input)
+
+        expected_slice = torch.tensor([9.0078, 10.1113, 14.4787], device=logits.device, dtype=logits.dtype)
+        result_slice = logits[0, 0, :3]
+        assert_tensors_close(expected_slice, result_slice, atol=TOLERANCE)
+
     @slow
     def test_enro_generate_one(self):
         batch: BatchEncoding = self.tokenizer.prepare_seq2seq_batch(
@@ -101,7 +128,7 @@ def test_enro_generate_one(self):
         # self.assertEqual(self.tgt_text[1], decoded[1])
 
     @slow
-    def test_enro_generate_batch(self):
+    def test_enro_generate(self):
         batch: BatchEncoding = self.tokenizer.prepare_seq2seq_batch(self.src_text).to(torch_device)
         translated_tokens = self.model.generate(**batch)
         decoded = self.tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)
diff --git a/tests/test_modeling_pegasus.py b/tests/test_modeling_pegasus.py
index cbc044fa833e..6896976fb7a8 100644
--- a/tests/test_modeling_pegasus.py
+++ b/tests/test_modeling_pegasus.py
@@ -15,10 +15,6 @@
     from transformers import AutoModelForSeq2SeqLM, PegasusConfig, PegasusForConditionalGeneration
 
 XSUM_ENTRY_LONGER = """ The London trio are up for best UK act and best album, as well as getting two nominations in the best song category."We got told like this morning 'Oh I think you're nominated'", said Dappy."And I was like 'Oh yeah, which one?' And now we've got nominated for four awards. I mean, wow!"Bandmate Fazer added: "We thought it's best of us to come down and mingle with everyone and say hello to the cameras. And now we find we've got four nominations."The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn't be too disappointed if they didn't win this time around."At the end of the day we're grateful to be where we are in our careers."If it don't happen then it don't happen - live to fight another day and keep on making albums and hits for the fans."Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers' All These Things That I've Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year's Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border."We just done Edinburgh the other day," said Dappy."We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!" """
-EXPECTED_SUMMARIES = [
-    "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
-    "Pop group N-Dubz have revealed they were surprised to get four nominations for this year's Mobo Awards.",
-]
 
 set_verbosity(ERROR)
 
@@ -60,7 +56,10 @@ def setUp(self):
 class PegasusXSUMIntegrationTest(AbstractSeq2SeqIntegrationTest):
     checkpoint_name = "google/pegasus-xsum"
     src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
-    tgt_text = EXPECTED_SUMMARIES
+    tgt_text = [
+        "California's largest electricity provider has turned off power to hundreds of thousands of customers.",
+        "N-Dubz have said they were surprised to get four nominations for this year's Mobo Awards.",
+    ]
 
     @cached_property
     def model(self):
@@ -73,7 +72,7 @@ def test_pegasus_xsum_summary(self):
             torch_device
         )
         assert inputs.input_ids.shape == (2, 421)
-        translated_tokens = self.model.generate(**inputs, num_beams=2)
+        translated_tokens = self.model.generate(**inputs)
         decoded = self.tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)
         assert self.tgt_text == decoded
 

From 33cba02652116322bb4f698f7be215ff29fe28a7 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:17:58 -0400
Subject: [PATCH 21/40] Cleanup model testers

---
 tests/test_modeling_tf_bart.py       | 10 +++++-----
 tests/test_modeling_tf_blenderbot.py |  8 ++++----
 tests/test_modeling_tf_marian.py     |  8 ++++----
 tests/test_modeling_tf_mbart.py      |  8 ++++----
 tests/test_modeling_tf_pegasus.py    |  8 ++++----
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 022dee1994c2..836772d839a2 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -36,10 +36,10 @@
 
 
 @require_tf
-class ModelTester:
-    kwargs = {}
-    hidden_act = "gelu"
+class TFBartModelTester:
     config_cls = BartConfig
+    config_updates = {}
+    hidden_act = "gelu"
 
     def __init__(self, parent):
         self.parent = parent
@@ -82,7 +82,7 @@ def prepare_config_and_inputs_for_common(self):
             bos_token_id=self.bos_token_id,
             pad_token_id=self.pad_token_id,
             decoder_start_token_id=self.pad_token_id,
-            **self.kwargs,
+            **self.config_updates,
         )
         inputs_dict = prepare_bart_inputs_dict(config, input_ids)
         return config, inputs_dict
@@ -108,7 +108,7 @@ class TestTFBart(TFModelTesterMixin, unittest.TestCase):
     all_generative_model_classes = (TFBartForConditionalGeneration,) if is_tf_available() else ()
     is_encoder_decoder = True
     test_pruning = False
-    model_tester_cls = ModelTester
+    model_tester_cls = TFBartModelTester
 
     def setUp(self):
         self.model_tester = self.model_tester_cls(self)
diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 519c45db1b42..4e6ded9cabba 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -18,7 +18,7 @@
 import tensorflow as tf
 
 from tests.test_configuration_common import ConfigTester
-from tests.test_modeling_tf_bart import ModelTester
+from tests.test_modeling_tf_bart import TFBartModelTester
 from tests.test_modeling_tf_common import TFModelTesterMixin
 from transformers import BlenderbotConfig, BlenderbotSmallTokenizer, is_tf_available
 from transformers.file_utils import cached_property
@@ -29,8 +29,8 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration
 
 
-class BlenderbotModelTester(ModelTester):
-    kwargs = dict(
+class ModelTester(TFBartModelTester):
+    config_updates = dict(
         normalize_before=True,
         static_position_embeddings=True,
         do_blenderbot_90_layernorm=True,
@@ -43,7 +43,7 @@ class BlenderbotModelTester(ModelTester):
 class TestTFBlenderbotCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFBlenderbotForConditionalGeneration,)
     all_generative_model_classes = (TFBlenderbotForConditionalGeneration,)
-    model_tester_cls = BlenderbotModelTester
+    model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
 
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index ac73a3d49a42..4f221c3162ef 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -28,12 +28,12 @@
     from transformers import MarianConfig, TFAutoModelForSeq2SeqLM, TFMarianMTModel
 
     from .test_configuration_common import ConfigTester
-    from .test_modeling_tf_bart import ModelTester
+    from .test_modeling_tf_bart import TFBartModelTester
     from .test_modeling_tf_common import TFModelTesterMixin
 
 
-class MarianModelTester(ModelTester):
-    kwargs = dict(static_position_embeddings=True, add_bias_logits=True)
+class ModelTester(TFBartModelTester):
+    config_updates = dict(static_position_embeddings=True, add_bias_logits=True)
     config_cls = MarianConfig
 
 
@@ -41,7 +41,7 @@ class MarianModelTester(ModelTester):
 class TestTFMarianCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFMarianMTModel,)
     all_generative_model_classes = (TFMarianMTModel,)
-    model_tester_cls = MarianModelTester
+    model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
 
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index a213ccdcdabb..4acab76407a0 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -18,7 +18,7 @@
 import tensorflow as tf
 
 from tests.test_configuration_common import ConfigTester
-from tests.test_modeling_tf_bart import ModelTester
+from tests.test_modeling_tf_bart import TFBartModelTester
 from tests.test_modeling_tf_common import TFModelTesterMixin
 from transformers import AutoTokenizer, MBartConfig, is_tf_available
 from transformers.file_utils import cached_property
@@ -72,8 +72,8 @@ def test_batch_generation_en_ro(self):
         self._assert_generated_batch_equal_expected()
 
 
-class MBartModelTester(ModelTester):
-    kwargs = dict(normalize_before=True, add_final_layer_norm=True)
+class ModelTester(TFBartModelTester):
+    config_updates = dict(normalize_before=True, add_final_layer_norm=True)
     config_cls = MBartConfig
 
 
@@ -81,7 +81,7 @@ class MBartModelTester(ModelTester):
 class TestTFMBartCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFMBartForConditionalGeneration,)
     all_generative_model_classes = (TFMBartForConditionalGeneration,)
-    model_tester_cls = MBartModelTester
+    model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
 
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index 50619824031f..d81915cf6bc2 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -23,7 +23,7 @@
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER
-from .test_modeling_tf_bart import ModelTester
+from .test_modeling_tf_bart import TFBartModelTester
 from .test_modeling_tf_common import TFModelTesterMixin
 
 
@@ -73,8 +73,8 @@ def test_batch_generation(self):
         self._assert_generated_batch_equal_expected()
 
 
-class PegasusModelTester(ModelTester):
-    kwargs = dict(
+class ModelTester(TFBartModelTester):
+    config_updates = dict(
         normalize_before=True,
         static_position_embeddings=True,
     )
@@ -86,7 +86,7 @@ class PegasusModelTester(ModelTester):
 class TestTFPegasusCommon(TFModelTesterMixin, unittest.TestCase):
     all_model_classes = (TFPegasusForConditionalGeneration,)
     all_generative_model_classes = (TFPegasusForConditionalGeneration,)
-    model_tester_cls = PegasusModelTester
+    model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
 

From aade43ed3a46474a986b3fba6104be4f8727704f Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:19:47 -0400
Subject: [PATCH 22/40] undo delta

---
 tests/test_modeling_tf_bart.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 836772d839a2..b739ad7b5292 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -121,14 +121,6 @@ def test_inputs_embeds(self):
         # inputs_embeds not supported
         pass
 
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
     def test_compile_tf_model(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
 
@@ -160,6 +152,14 @@ def test_compile_tf_model(self):
         extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
 
+    def test_saved_model_with_hidden_states_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
+    def test_saved_model_with_attentions_output(self):
+        # Should be uncommented during patrick TF refactor
+        pass
+
 
 @require_tf
 class TFBartHeadTests(unittest.TestCase):

From e8955069255399a80a25fcf0d775f0999d260e88 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:28:07 -0400
Subject: [PATCH 23/40] fix tests import structure

---
 tests/test_modeling_tf_bart.py    |  5 +-
 tests/test_modeling_tf_marian.py  | 13 +++--
 tests/test_modeling_tf_mbart.py   | 84 +++++++++++++++----------------
 tests/test_modeling_tf_pegasus.py |  4 +-
 4 files changed, 52 insertions(+), 54 deletions(-)

diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index b739ad7b5292..87dced257416 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -19,7 +19,7 @@
 
 import numpy as np
 
-from transformers import is_tf_available
+from transformers import BartConfig, BartTokenizer, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import is_pt_tf_cross_test, require_tf, slow
 
@@ -30,9 +30,8 @@
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers import BartConfig, TFBartForConditionalGeneration, TFBartModel
+    from transformers import TFBartForConditionalGeneration, TFBartModel
     from transformers.modeling_tf_bart import TFSinusoidalPositionalEmbedding
-    from transformers.tokenization_bart import BartTokenizer
 
 
 @require_tf
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 4f221c3162ef..6b0054ba0a34 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -16,20 +16,19 @@
 import tempfile
 import unittest
 
-from transformers import AutoTokenizer, MarianTokenizer, TranslationPipeline, is_tf_available
+from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
+from .test_configuration_common import ConfigTester
+from .test_modeling_tf_bart import TFBartModelTester
+from .test_modeling_tf_common import TFModelTesterMixin
 
-if is_tf_available():
 
+if is_tf_available():
     import tensorflow as tf
 
-    from transformers import MarianConfig, TFAutoModelForSeq2SeqLM, TFMarianMTModel
-
-    from .test_configuration_common import ConfigTester
-    from .test_modeling_tf_bart import TFBartModelTester
-    from .test_modeling_tf_common import TFModelTesterMixin
+    from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
 
 
 class ModelTester(TFBartModelTester):
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index 4acab76407a0..e45e8e1890d1 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -30,48 +30,6 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
 
 
-@require_tf
-@require_sentencepiece
-@require_tokenizers
-class TestMBartEnRO(unittest.TestCase):
-    src_text = [
-        " UN Chief Says There Is No Military Solution in Syria",
-        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
-    ]
-    expected_text = [
-        "Şeful ONU declară că nu există o soluţie militară în Siria",
-        # 'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
-    ]
-    model_name = "facebook/mbart-large-en-ro"
-
-    @cached_property
-    def tokenizer(self):
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        self.assertListEqual(self.expected_text, generated_words)
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
-        generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
-        return generated_words
-
-    @slow
-    def test_batch_generation_en_ro(self):
-        self._assert_generated_batch_equal_expected()
-
-
 class ModelTester(TFBartModelTester):
     config_updates = dict(normalize_before=True, add_final_layer_norm=True)
     config_cls = MBartConfig
@@ -134,3 +92,45 @@ def test_compile_tf_model(self):
         # Compile extended model
         extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
+
+
+@require_tf
+@require_sentencepiece
+@require_tokenizers
+class TestMBartEnRO(unittest.TestCase):
+    src_text = [
+        " UN Chief Says There Is No Military Solution in Syria",
+        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+    ]
+    expected_text = [
+        "Şeful ONU declară că nu există o soluţie militară în Siria",
+        # 'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
+    ]
+    model_name = "facebook/mbart-large-en-ro"
+
+    @cached_property
+    def tokenizer(self):
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        self.assertListEqual(self.expected_text, generated_words)
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
+        generated_ids = self.model.generate(
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+        return generated_words
+
+    @slow
+    def test_batch_generation_en_ro(self):
+        self._assert_generated_batch_equal_expected()
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index d81915cf6bc2..3df41e198921 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -17,7 +17,7 @@
 
 import tensorflow as tf
 
-from transformers import AutoTokenizer, is_tf_available
+from transformers import AutoTokenizer, PegasusConfig, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
@@ -28,7 +28,7 @@
 
 
 if is_tf_available():
-    from transformers import PegasusConfig, TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
+    from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
 
 
 @require_tf

From dd2ecc22f8509fc5309ea795c32d0af607b0be40 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Sun, 25 Oct 2020 13:45:00 -0400
Subject: [PATCH 24/40] cross test decorator

---
 tests/test_modeling_tf_blenderbot.py |  8 +--
 tests/test_modeling_tf_marian.py     | 10 ++--
 tests/test_modeling_tf_mbart.py      |  8 +--
 tests/test_modeling_tf_pegasus.py    | 90 ++++++++++++++--------------
 4 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 4e6ded9cabba..463fb844c436 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -22,7 +22,7 @@
 from tests.test_modeling_tf_common import TFModelTesterMixin
 from transformers import BlenderbotConfig, BlenderbotSmallTokenizer, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_tf, require_tokenizers, slow
+from transformers.testing_utils import is_pt_tf_cross_test, require_tf, require_tokenizers, slow
 
 
 if is_tf_available():
@@ -41,8 +41,8 @@ class ModelTester(TFBartModelTester):
 
 @require_tf
 class TestTFBlenderbotCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFBlenderbotForConditionalGeneration,)
-    all_generative_model_classes = (TFBlenderbotForConditionalGeneration,)
+    all_model_classes = (TFBlenderbotForConditionalGeneration,) if is_tf_available() else ()
+    all_generative_model_classes = (TFBlenderbotForConditionalGeneration,) if is_tf_available() else ()
     model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
@@ -98,7 +98,7 @@ def test_compile_tf_model(self):
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
 
 
-@require_tf
+@is_pt_tf_cross_test
 @require_tokenizers
 class TFBlenderbot90MIntegrationTests(unittest.TestCase):
     src_text = [
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 6b0054ba0a34..7195d9a367d7 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -18,7 +18,7 @@
 
 from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
+from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_bart import TFBartModelTester
@@ -38,8 +38,8 @@ class ModelTester(TFBartModelTester):
 
 @require_tf
 class TestTFMarianCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFMarianMTModel,)
-    all_generative_model_classes = (TFMarianMTModel,)
+    all_model_classes = (TFMarianMTModel,) if is_tf_available() else ()
+    all_generative_model_classes = (TFMarianMTModel,) if is_tf_available() else ()
     model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
@@ -133,7 +133,7 @@ def translate_src_text(self, **tokenizer_kwargs):
         return generated_words
 
 
-@require_tf
+@is_pt_tf_cross_test
 @require_sentencepiece
 @require_tokenizers
 class TestMarian_en_zh(AbstractMarianIntegrationTest):
@@ -147,7 +147,7 @@ def test_batch_generation_en_zh(self):
         self._assert_generated_batch_equal_expected()
 
 
-@require_tf
+@is_pt_tf_cross_test
 @require_sentencepiece
 @require_tokenizers
 class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index e45e8e1890d1..a7cb06cabf04 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -22,7 +22,7 @@
 from tests.test_modeling_tf_common import TFModelTesterMixin
 from transformers import AutoTokenizer, MBartConfig, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
+from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
 
 
 if is_tf_available():
@@ -37,8 +37,8 @@ class ModelTester(TFBartModelTester):
 
 @require_tf
 class TestTFMBartCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFMBartForConditionalGeneration,)
-    all_generative_model_classes = (TFMBartForConditionalGeneration,)
+    all_model_classes = (TFMBartForConditionalGeneration,) if is_tf_available() else ()
+    all_generative_model_classes = (TFMBartForConditionalGeneration,) if is_tf_available() else ()
     model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
@@ -94,7 +94,7 @@ def test_compile_tf_model(self):
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
 
 
-@require_tf
+@is_pt_tf_cross_test
 @require_sentencepiece
 @require_tokenizers
 class TestMBartEnRO(unittest.TestCase):
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index 3df41e198921..56798b4a36ff 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -19,7 +19,7 @@
 
 from transformers import AutoTokenizer, PegasusConfig, is_tf_available
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
+from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER
@@ -31,48 +31,6 @@
     from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
 
 
-@require_tf
-@require_sentencepiece
-@require_tokenizers
-class TFPegasusIntegrationTests(unittest.TestCase):
-    src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
-    expected_text = [
-        "California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to reduce the risk of wildfires.",
-        'N-Dubz have revealed they\'re "grateful" to have been nominated for four Mobo Awards.',
-    ]  # differs slightly from pytorch, likely due to numerical differences in linear layers
-    model_name = "google/pegasus-xsum"
-
-    @cached_property
-    def tokenizer(self):
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        assert self.expected_text == generated_words
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
-        generated_ids = self.model.generate(
-            model_inputs.input_ids,
-            attention_mask=model_inputs.attention_mask,
-            num_beams=2,
-            use_cache=True,
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
-        return generated_words
-
-    @slow
-    def test_batch_generation(self):
-        self._assert_generated_batch_equal_expected()
-
-
 class ModelTester(TFBartModelTester):
     config_updates = dict(
         normalize_before=True,
@@ -84,8 +42,8 @@ class ModelTester(TFBartModelTester):
 
 @require_tf
 class TestTFPegasusCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFPegasusForConditionalGeneration,)
-    all_generative_model_classes = (TFPegasusForConditionalGeneration,)
+    all_model_classes = (TFPegasusForConditionalGeneration,) if is_tf_available() else ()
+    all_generative_model_classes = (TFPegasusForConditionalGeneration,) if is_tf_available() else ()
     model_tester_cls = ModelTester
     is_encoder_decoder = True
     test_pruning = False
@@ -139,3 +97,45 @@ def test_compile_tf_model(self):
         # Compile extended model
         extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
         extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
+
+
+@is_pt_tf_cross_test
+@require_sentencepiece
+@require_tokenizers
+class TFPegasusIntegrationTests(unittest.TestCase):
+    src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
+    expected_text = [
+        "California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to reduce the risk of wildfires.",
+        'N-Dubz have revealed they\'re "grateful" to have been nominated for four Mobo Awards.',
+    ]  # differs slightly from pytorch, likely due to numerical differences in linear layers
+    model_name = "google/pegasus-xsum"
+
+    @cached_property
+    def tokenizer(self):
+        return AutoTokenizer.from_pretrained(self.model_name)
+
+    @cached_property
+    def model(self):
+        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
+        return model
+
+    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
+        generated_words = self.translate_src_text(**tokenizer_kwargs)
+        assert self.expected_text == generated_words
+
+    def translate_src_text(self, **tokenizer_kwargs):
+        model_inputs = self.tokenizer.prepare_seq2seq_batch(
+            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
+        )
+        generated_ids = self.model.generate(
+            model_inputs.input_ids,
+            attention_mask=model_inputs.attention_mask,
+            num_beams=2,
+            use_cache=True,
+        )
+        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
+        return generated_words
+
+    @slow
+    def test_batch_generation(self):
+        self._assert_generated_batch_equal_expected()

From fa8c5eb53c3dfeba70a3cf3f2d38def27de2a330 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 12:03:12 -0400
Subject: [PATCH 25/40] Cleaner set_weights

---
 src/transformers/modeling_tf_bart.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index fc550ad0b6c8..43f5d2e8a1b6 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -814,7 +814,8 @@ def build(self, input_shape):
             https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         super().build(input_shape)  # Instantiates self.weight so it can be loaded
-        self.weight = self._init_weight(self.vocab_size, self.hidden_size)  # overwrite with good defaults
+        weight: np.ndarray = self._init_weight(self.vocab_size, self.hidden_size)
+        self.set_weights([weight])  # overwrite self.weight to correct value
 
     @staticmethod
     def _init_weight(n_pos, dim):

From b07d20b14085a8a235f19d903c2985458bd4a3a6 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 12:42:38 -0400
Subject: [PATCH 26/40] Respect authorized_unexpected_keys

---
 src/transformers/modeling_tf_bart.py          | 2 --
 src/transformers/modeling_tf_marian.py        | 9 ++++-----
 src/transformers/modeling_tf_mbart.py         | 5 +++++
 src/transformers/modeling_tf_pytorch_utils.py | 4 +++-
 tests/test_modeling_tf_marian.py              | 8 ++++++--
 5 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 43f5d2e8a1b6..c60527825803 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -1013,8 +1013,6 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
     base_model_prefix = "model"
     authorized_missing_keys = [
         r"final_logits_bias",
-        r"encoder\.version",
-        r"decoder\.version",
         "model.encoder.embed_tokens.weight",
         "model.decoder.embed_tokens.weight",
     ]
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 92a8901fcfcc..2384c97b28cf 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -33,14 +33,13 @@
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
     authorized_missing_keys = [
-        r"final_logits_bias",
-        r"encoder\.version",
-        r"decoder\.version",
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
         "model.encoder.embed_positions.weight",
         "model.decoder.embed_positions.weight",
     ]
+    authorized_unexpected_keys = [
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
+    ]
     config_class = MarianConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
diff --git a/src/transformers/modeling_tf_mbart.py b/src/transformers/modeling_tf_mbart.py
index 606146a050d2..606467e8071c 100644
--- a/src/transformers/modeling_tf_mbart.py
+++ b/src/transformers/modeling_tf_mbart.py
@@ -33,3 +33,8 @@
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMBartForConditionalGeneration(TFBartForConditionalGeneration):
     config_class = MBartConfig
+    authorized_missing_keys = [
+        r"final_logits_bias",
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
+    ]
diff --git a/src/transformers/modeling_tf_pytorch_utils.py b/src/transformers/modeling_tf_pytorch_utils.py
index 48a32f711e04..b2dcb449c857 100644
--- a/src/transformers/modeling_tf_pytorch_utils.py
+++ b/src/transformers/modeling_tf_pytorch_utils.py
@@ -208,6 +208,9 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
     if tf_model.authorized_missing_keys is not None:
         for pat in tf_model.authorized_missing_keys:
             missing_keys = [k for k in missing_keys if re.search(pat, k) is None]
+    if tf_model.authorized_unexpected_keys is not None:
+        for pat in tf_model.authorized_unexpected_keys:
+            unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]
 
     if len(unexpected_keys) > 0:
         logger.warning(
@@ -235,7 +238,6 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
 
     return tf_model
 
-
 #####################
 # TF 2.0 => PyTorch #
 #####################
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 7195d9a367d7..900cc9862683 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -15,6 +15,7 @@
 
 import tempfile
 import unittest
+import warnings
 
 from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
 from transformers.file_utils import cached_property
@@ -110,6 +111,7 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
+        warnings.simplefilter('error')
         model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
         assert isinstance(model, TFMarianMTModel)
         c = model.config
@@ -140,7 +142,7 @@ class TestMarian_en_zh(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "zh"
     src_text = ["My name is Wolfgang and I live in Berlin"]
-    expected_text = ["我的名字是沃尔夫冈 我住在柏林"]
+    expected_text = ["我叫沃尔夫冈 我住在柏林"]
 
     @slow
     def test_batch_generation_en_zh(self):
@@ -156,12 +158,14 @@ class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
     src = "en"
     tgt = "ROMANCE"
     src_text = [
+        ">>fr<< Don't spend so much time watching TV.",
         ">>pt<< Your message has been sent.",
         ">>es<< He's two years older than me.",
     ]
     expected_text = [
+        "Ne passez pas autant de temps à regarder la télé.",
         "A sua mensagem foi enviada.",
-        "Tiene dos años más que yo.",
+        "Es dos años más viejo que yo.",
     ]
 
     @slow

From 7c3e1f7fd6675d5a2d5777d1b6095668f42403a6 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 12:58:31 -0400
Subject: [PATCH 27/40] No warnings

---
 src/transformers/modeling_tf_bart.py          |  9 ++++++---
 src/transformers/modeling_tf_marian.py        | 11 +++++------
 src/transformers/modeling_tf_mbart.py         | 10 +++-------
 src/transformers/modeling_tf_pegasus.py       |  8 ++------
 src/transformers/modeling_tf_pytorch_utils.py |  1 +
 tests/test_modeling_tf_marian.py              |  2 +-
 6 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index c60527825803..6565fd1f6da3 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -1013,8 +1013,10 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
     base_model_prefix = "model"
     authorized_missing_keys = [
         r"final_logits_bias",
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
+    ]
+    authorized_unexpected_keys = [
+        r"model.encoder.embed_tokens.weight",
+        r"model.decoder.embed_tokens.weight",
     ]
 
     def __init__(self, config: BartConfig, *args, **kwargs):
@@ -1022,7 +1024,8 @@ def __init__(self, config: BartConfig, *args, **kwargs):
         self.model = TFBartModel(config, name="model")
         self.use_cache = config.use_cache
         self.final_logits_bias = self.add_weight(
-            name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=True
+            name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False
+            # final_bias_logits is registered as a buffer in pytorch, so not trainable.
         )
 
     @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 2384c97b28cf..432a0743de35 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -33,17 +33,16 @@
 @add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
 class TFMarianMTModel(TFBartForConditionalGeneration):
     authorized_missing_keys = [
-        "model.encoder.embed_positions.weight",
-        "model.decoder.embed_positions.weight",
-    ]
-    authorized_unexpected_keys = [
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
+        r"model.encoder.embed_positions.weight",
+        r"model.decoder.embed_positions.weight",
     ]
     config_class = MarianConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
+        """Never predict pad_token_id. Predict </s> when max_length is reached."""
         self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
         if cur_len == max_length - 1 and self.config.eos_token_id is not None:
             logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
         return logits
+
+    # All the code is in src/transformers/modeling_tf_bart.py
diff --git a/src/transformers/modeling_tf_mbart.py b/src/transformers/modeling_tf_mbart.py
index 606467e8071c..804324a31631 100644
--- a/src/transformers/modeling_tf_mbart.py
+++ b/src/transformers/modeling_tf_mbart.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TF BART model, ported from the fairseq repo."""
+"""TF mBART model, originally from fairseq."""
 from .configuration_mbart import MBartConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
@@ -30,11 +30,7 @@
 logger = logging.get_logger(__name__)
 
 
-@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
+@add_start_docstrings("mBART (multilingual BART) model for machine translation", START_DOCSTRING)
 class TFMBartForConditionalGeneration(TFBartForConditionalGeneration):
     config_class = MBartConfig
-    authorized_missing_keys = [
-        r"final_logits_bias",
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
-    ]
+    # All the code is in src/transformers/modeling_tf_bart.py
diff --git a/src/transformers/modeling_tf_pegasus.py b/src/transformers/modeling_tf_pegasus.py
index 40f83a4614af..5a41d6fa3817 100644
--- a/src/transformers/modeling_tf_pegasus.py
+++ b/src/transformers/modeling_tf_pegasus.py
@@ -34,12 +34,8 @@
 class TFPegasusForConditionalGeneration(TFBartForConditionalGeneration):
     authorized_missing_keys = [
         r"final_logits_bias",
-        r"encoder\.version",
-        r"decoder\.version",
-        "model.encoder.embed_tokens.weight",
-        "model.decoder.embed_tokens.weight",
-        "model.encoder.embed_positions.weight",
-        "model.decoder.embed_positions.weight",
+        r"model.encoder.embed_positions.weight",
+        r"model.decoder.embed_positions.weight",
     ]
     config_class = PegasusConfig
     # All the code is in src/transformers/modeling_tf_bart.py
diff --git a/src/transformers/modeling_tf_pytorch_utils.py b/src/transformers/modeling_tf_pytorch_utils.py
index b2dcb449c857..82120d2cc16a 100644
--- a/src/transformers/modeling_tf_pytorch_utils.py
+++ b/src/transformers/modeling_tf_pytorch_utils.py
@@ -238,6 +238,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
 
     return tf_model
 
+
 #####################
 # TF 2.0 => PyTorch #
 #####################
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 900cc9862683..85348981d56c 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -111,7 +111,7 @@ def eos_token_id(self) -> int:
 
     @cached_property
     def model(self):
-        warnings.simplefilter('error')
+        warnings.simplefilter("error")
         model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
         assert isinstance(model, TFMarianMTModel)
         c = model.config

From d097e278a7e8f8865c638a9a29dd343b166e3581 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 12:58:50 -0400
Subject: [PATCH 28/40] No warnings

---
 src/transformers/modeling_tf_pytorch_utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/transformers/modeling_tf_pytorch_utils.py b/src/transformers/modeling_tf_pytorch_utils.py
index 82120d2cc16a..48a32f711e04 100644
--- a/src/transformers/modeling_tf_pytorch_utils.py
+++ b/src/transformers/modeling_tf_pytorch_utils.py
@@ -208,9 +208,6 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
     if tf_model.authorized_missing_keys is not None:
         for pat in tf_model.authorized_missing_keys:
             missing_keys = [k for k in missing_keys if re.search(pat, k) is None]
-    if tf_model.authorized_unexpected_keys is not None:
-        for pat in tf_model.authorized_unexpected_keys:
-            unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None]
 
     if len(unexpected_keys) > 0:
         logger.warning(

From cbee9965af54d8426f1ceac3634d2d440636c6b7 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 13:02:10 -0400
Subject: [PATCH 29/40] style

---
 src/transformers/modeling_tf_bart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 6565fd1f6da3..1167f6c82cd9 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -1023,9 +1023,9 @@ def __init__(self, config: BartConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         self.model = TFBartModel(config, name="model")
         self.use_cache = config.use_cache
+        # final_bias_logits is registered as a buffer in pytorch, so not trainable for the the sake of consistency.
         self.final_logits_bias = self.add_weight(
             name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False
-            # final_bias_logits is registered as a buffer in pytorch, so not trainable.
         )
 
     @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)

From 5340a6762784c79a2e0978467ad16a62a4cd8aa4 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 13:09:09 -0400
Subject: [PATCH 30/40] Nest tf import

---
 tests/test_modeling_tf_blenderbot.py | 3 ++-
 tests/test_modeling_tf_mbart.py      | 2 +-
 tests/test_modeling_tf_pegasus.py    | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 463fb844c436..89313851096f 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -15,7 +15,7 @@
 import tempfile
 import unittest
 
-import tensorflow as tf
+
 
 from tests.test_configuration_common import ConfigTester
 from tests.test_modeling_tf_bart import TFBartModelTester
@@ -26,6 +26,7 @@
 
 
 if is_tf_available():
+    import tensorflow as tf
     from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration
 
 
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index a7cb06cabf04..b2ce9a5c9ff7 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -15,7 +15,6 @@
 import tempfile
 import unittest
 
-import tensorflow as tf
 
 from tests.test_configuration_common import ConfigTester
 from tests.test_modeling_tf_bart import TFBartModelTester
@@ -28,6 +27,7 @@
 if is_tf_available():
 
     from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
+    import tensorflow as tf
 
 
 class ModelTester(TFBartModelTester):
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index 56798b4a36ff..35a0bad25225 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -15,7 +15,7 @@
 import tempfile
 import unittest
 
-import tensorflow as tf
+
 
 from transformers import AutoTokenizer, PegasusConfig, is_tf_available
 from transformers.file_utils import cached_property
@@ -29,6 +29,7 @@
 
 if is_tf_available():
     from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
+    import tensorflow as tf
 
 
 class ModelTester(TFBartModelTester):

From be9de105c1758e0ebd87d4f6219fcc7b2ee99d8f Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 13:52:58 -0400
Subject: [PATCH 31/40] black

---
 tests/test_modeling_tf_blenderbot.py | 3 +--
 tests/test_modeling_tf_mbart.py      | 4 ++--
 tests/test_modeling_tf_pegasus.py    | 5 ++---
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py
index 89313851096f..df11567e41a8 100644
--- a/tests/test_modeling_tf_blenderbot.py
+++ b/tests/test_modeling_tf_blenderbot.py
@@ -15,8 +15,6 @@
 import tempfile
 import unittest
 
-
-
 from tests.test_configuration_common import ConfigTester
 from tests.test_modeling_tf_bart import TFBartModelTester
 from tests.test_modeling_tf_common import TFModelTesterMixin
@@ -27,6 +25,7 @@
 
 if is_tf_available():
     import tensorflow as tf
+
     from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration
 
 
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index b2ce9a5c9ff7..ec0616991f53 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -15,7 +15,6 @@
 import tempfile
 import unittest
 
-
 from tests.test_configuration_common import ConfigTester
 from tests.test_modeling_tf_bart import TFBartModelTester
 from tests.test_modeling_tf_common import TFModelTesterMixin
@@ -26,9 +25,10 @@
 
 if is_tf_available():
 
-    from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
     import tensorflow as tf
 
+    from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
+
 
 class ModelTester(TFBartModelTester):
     config_updates = dict(normalize_before=True, add_final_layer_norm=True)
diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py
index 35a0bad25225..32d98bfd7bf6 100644
--- a/tests/test_modeling_tf_pegasus.py
+++ b/tests/test_modeling_tf_pegasus.py
@@ -15,8 +15,6 @@
 import tempfile
 import unittest
 
-
-
 from transformers import AutoTokenizer, PegasusConfig, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
@@ -28,9 +26,10 @@
 
 
 if is_tf_available():
-    from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
     import tensorflow as tf
 
+    from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
+
 
 class ModelTester(TFBartModelTester):
     config_updates = dict(

From 1d86346634d6b90daf5891418b0dfb075a3a0edc Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 15:13:04 -0400
Subject: [PATCH 32/40] Apply suggestions from code review

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
---
 src/transformers/modeling_tf_blenderbot.py | 2 +-
 src/transformers/modeling_tf_marian.py     | 2 +-
 src/transformers/modeling_tf_pegasus.py    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/transformers/modeling_tf_blenderbot.py b/src/transformers/modeling_tf_blenderbot.py
index ab4942dee3c2..04b9816980c0 100644
--- a/src/transformers/modeling_tf_blenderbot.py
+++ b/src/transformers/modeling_tf_blenderbot.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TF BART model, ported from the fairseq repo."""
+"""TF BlenderBot model, ported from the fairseq repo."""
 from .configuration_blenderbot import BlenderbotConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 432a0743de35..686529ac1aa4 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TF BART model, ported from the fairseq repo."""
+"""TF Marian model, ported from the fairseq repo."""
 from .configuration_marian import MarianConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
diff --git a/src/transformers/modeling_tf_pegasus.py b/src/transformers/modeling_tf_pegasus.py
index 5a41d6fa3817..262c7bdb28c3 100644
--- a/src/transformers/modeling_tf_pegasus.py
+++ b/src/transformers/modeling_tf_pegasus.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TF BART model, ported from the fairseq repo."""
+"""TF Pegasus model, ported from the fairseq repo."""
 from .configuration_pegasus import PegasusConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration

From 66f1d06ce07176ba8286b594ae2fc0043b377fcb Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 17:30:45 -0400
Subject: [PATCH 33/40] functional dropout

---
 src/transformers/modeling_tf_bart.py       | 22 +++++++++++-----------
 src/transformers/modeling_tf_blenderbot.py |  2 +-
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 1167f6c82cd9..40c0924cd096 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -22,7 +22,7 @@
 import numpy as np
 import tensorflow as tf
 from tensorflow import Tensor
-from tensorflow.keras.layers import Dense, Dropout, Layer, LayerNormalization
+from tensorflow.keras.layers import Dense, Layer, LayerNormalization
 
 from .activations_tf import ACT2FN
 from .configuration_bart import BartConfig
@@ -217,7 +217,7 @@ def make_padding_mask(input_ids, padding_idx=1):
 )
 
 
-class TFEncoderLayer(tf.keras.layers.Layer):
+class TFEncoderLayer(Layer):
     def __init__(self, config: BartConfig, **kwargs):
         super().__init__(**kwargs)
         self.embed_dim = config.d_model
@@ -226,9 +226,9 @@ def __init__(self, config: BartConfig, **kwargs):
         )
         self.normalize_before = config.normalize_before
         self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
-        self.dropout = Dropout(config.dropout)
+        self.dropout = config.dropout
         self.activation_fn = ACT2FN[config.activation_function]
-        self.activation_dropout = Dropout(config.activation_dropout)
+        self.activation_dropout = config.activation_dropout
         self.fc1 = Dense(config.encoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
         self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
@@ -259,7 +259,7 @@ def call(self, x, encoder_padding_mask, training=False):
         if self.normalize_before:
             x = self.final_layer_norm(x)
         x = self.activation_fn(self.fc1(x))
-        x = self.activation_dropout(x, training=training)
+        x = tf.nn.dropout(x, rate=self.self.activation_dropout if training else 0)
         x = self.fc2(x)
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
@@ -269,7 +269,7 @@ def call(self, x, encoder_padding_mask, training=False):
         return x, self_attn_weights
 
 
-class TFBartEncoder(tf.keras.layers.Layer):
+class TFBartEncoder(Layer):
     # config_class = BartConfig
     """
     Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer
@@ -386,7 +386,7 @@ def call(
         return TFBaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions)
 
 
-class TFDecoderLayer(tf.keras.layers.Layer):
+class TFDecoderLayer(Layer):
     def __init__(self, config: BartConfig, **kwargs):
         super().__init__(**kwargs)
         self.embed_dim = config.d_model
@@ -487,7 +487,7 @@ def call(
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
 
-class TFBartDecoder(tf.keras.layers.Layer):
+class TFBartDecoder(Layer):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer
     is a :class:`TFDecoderLayer`.
@@ -523,7 +523,7 @@ def __init__(self, config: BartConfig, embed_tokens, **kwargs):
         )
         self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
 
-        self.dropout = tf.keras.layers.Dropout(config.dropout)
+        self.dropout = config.dropout
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.use_cache = config.use_cache
@@ -566,7 +566,7 @@ def call(
             x = self.layernorm_embedding(x) + positions
         else:
             x = self.layernorm_embedding(x + positions)
-        x = self.dropout(x)
+        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
         x = tf.transpose(x, perm=(1, 0, 2))
@@ -636,7 +636,7 @@ def _reorder_buffer(attn_cache, new_order):
     return attn_cache
 
 
-class TFAttention(tf.keras.layers.Layer):
+class TFAttention(Layer):
     """Multi-headed attention from "Attention Is All You Need"""
 
     def __init__(
diff --git a/src/transformers/modeling_tf_blenderbot.py b/src/transformers/modeling_tf_blenderbot.py
index ab4942dee3c2..567b0a0e0cfa 100644
--- a/src/transformers/modeling_tf_blenderbot.py
+++ b/src/transformers/modeling_tf_blenderbot.py
@@ -30,7 +30,7 @@
 logger = logging.get_logger(__name__)
 
 
-@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
+@add_start_docstrings("Blenderbot model for open domain dialogue", START_DOCSTRING)
 class TFBlenderbotForConditionalGeneration(TFBartForConditionalGeneration):
     config_class = BlenderbotConfig
 

From 1b82b634aa3f11841662a1ca006e843c124ec981 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 21:24:12 -0400
Subject: [PATCH 34/40] fixup

---
 src/transformers/modeling_tf_bart.py | 11 ++++++-----
 tests/test_modeling_tf_mbart.py      |  6 +++++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 58a8962ea466..b07761028320 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -808,9 +808,9 @@ def __init__(self, num_positions, embedding_dim, **kwargs):
         # self.weight = self._init_weight(*self.weight.shape)
 
     def build(self, input_shape):
-        """Build shared token embedding layer
-        Shared weights logic adapted from
-            https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
+        """
+        Build shared token embedding layer Shared weights logic adapted from
+        https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         super().build(input_shape)  # Instantiates self.weight so it can be loaded
         weight: np.ndarray = self._init_weight(self.vocab_size, self.hidden_size)
@@ -818,8 +818,9 @@ def build(self, input_shape):
 
     @staticmethod
     def _init_weight(n_pos, dim):
-        """Identical to the XLM create_sinusoidal_embeddings except features are not interleaved.
-        The cos features are in the 2nd half of the vector. [dim // 2:]
+        """
+        Identical to the XLM create_sinusoidal_embeddings except features are not interleaved. The cos features are in
+        the 2nd half of the vector. [dim // 2:]
         """
         position_enc = np.array(
             [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index ec0616991f53..3f6c60468b42 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -100,7 +100,11 @@ def test_compile_tf_model(self):
 class TestMBartEnRO(unittest.TestCase):
     src_text = [
         " UN Chief Says There Is No Military Solution in Syria",
-        # """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
+        # """
+ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no
+ military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for
+ millions of people.
+ """,
     ]
     expected_text = [
         "Şeful ONU declară că nu există o soluţie militară în Siria",

From 9ebbe55bc349fedf676acbf23b5689e39522a94d Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 21:35:37 -0400
Subject: [PATCH 35/40] Fixup

---
 tests/test_modeling_tf_mbart.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py
index 3f6c60468b42..d631971c43b6 100644
--- a/tests/test_modeling_tf_mbart.py
+++ b/tests/test_modeling_tf_mbart.py
@@ -100,15 +100,9 @@ def test_compile_tf_model(self):
 class TestMBartEnRO(unittest.TestCase):
     src_text = [
         " UN Chief Says There Is No Military Solution in Syria",
-        # """
- Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that "there is no
- military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for
- millions of people.
- """,
     ]
     expected_text = [
         "Şeful ONU declară că nu există o soluţie militară în Siria",
-        # 'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor face decât să înrăutăţească violenţa şi mizeria pentru milioane de oameni.',
     ]
     model_name = "facebook/mbart-large-en-ro"
 

From b03c9533363bbf60fbb2f7ab23162601c205a161 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Mon, 26 Oct 2020 22:53:51 -0400
Subject: [PATCH 36/40] style_doc

---
 docs/source/model_doc/blenderbot.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/source/model_doc/blenderbot.rst b/docs/source/model_doc/blenderbot.rst
index 8f042cd647c4..4d79144e8e44 100644
--- a/docs/source/model_doc/blenderbot.rst
+++ b/docs/source/model_doc/blenderbot.rst
@@ -99,6 +99,7 @@ See :obj:`transformers.BartForConditionalGeneration` for arguments to `forward`
 
 TFBlenderbotForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 See :obj:`transformers.TFBartForConditionalGeneration` for arguments to `forward` and `generate`
 
 .. autoclass:: transformers.TFBlenderbotForConditionalGeneration

From 13583d78350ec8fbdc1007fdb0923330fa53e227 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 28 Oct 2020 01:32:04 -0400
Subject: [PATCH 37/40] embs

---
 src/transformers/modeling_tf_bart.py | 5 ++---
 tests/test_modeling_tf_bart.py       | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index b07761028320..299ec694159b 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -793,7 +793,7 @@ def call(self, input_ids: tf.Tensor, use_cache=False):
         return super().call(positions + self.offset)  # super object is not callable for some reason
 
 
-class TFSinusoidalPositionalEmbedding(TFSharedEmbeddings):
+class TFSinusoidalPositionalEmbedding(tf.keras.layers.Embedding):
     """This module produces sinusoidal positional embeddings of any length."""
 
     def __init__(self, num_positions, embedding_dim, **kwargs):
@@ -805,7 +805,6 @@ def __init__(self, num_positions, embedding_dim, **kwargs):
             embedding_dim,
             **kwargs,
         )
-        # self.weight = self._init_weight(*self.weight.shape)
 
     def build(self, input_shape):
         """
@@ -813,7 +812,7 @@ def build(self, input_shape):
         https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
         """
         super().build(input_shape)  # Instantiates self.weight so it can be loaded
-        weight: np.ndarray = self._init_weight(self.vocab_size, self.hidden_size)
+        weight: np.ndarray = self._init_weight(self.input_dim, self.output_dim)
         self.set_weights([weight])  # overwrite self.weight to correct value
 
     @staticmethod
diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py
index 87dced257416..4efdd3b08b09 100644
--- a/tests/test_modeling_tf_bart.py
+++ b/tests/test_modeling_tf_bart.py
@@ -383,7 +383,7 @@ def test_positional_emb_cache_logic(self):
     def test_positional_emb_weights_against_marian(self):
         emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512)
         emb1.build(None)
-        weights = emb1.weight.numpy()
+        weights = emb1.embeddings.numpy()
         for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
             for j in range(5):
                 self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)

From eb613d81e80ff97fc9fd2fc81f7f5d4ad318c6f4 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 28 Oct 2020 02:02:28 -0400
Subject: [PATCH 38/40] shape list

---
 src/transformers/modeling_tf_bart.py   | 26 +++++++++++++++-----------
 src/transformers/modeling_tf_marian.py | 18 ++++++++++++++++++
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 299ec694159b..0c4a659ebf7d 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -250,7 +250,9 @@ def call(self, x, encoder_padding_mask, training=False):
         if self.normalize_before:
             x = self.self_attn_layer_norm(x)
         x, self_attn_weights = self.self_attn(query=x, key=x, key_padding_mask=encoder_padding_mask)
-        assert x.shape == residual.shape, f"Self attn modified the shape of query {residual.shape} to {x.shape}"
+        assert shape_list(x) == shape_list(
+            residual
+        ), f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(x)}"
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
         if not self.normalize_before:
@@ -570,7 +572,7 @@ def call(
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
         x = tf.transpose(x, perm=(1, 0, 2))
-        assert len(encoder_hidden_states.shape) == 3, "encoder_hidden_states must be a 3D tensor"
+        assert len(shape_list(encoder_hidden_states)) == 3, "encoder_hidden_states must be a 3D tensor"
         encoder_hidden_states = tf.transpose(encoder_hidden_states, perm=(1, 0, 2))
 
         # decoder layers
@@ -691,8 +693,10 @@ def call(
                 (default: None).
         """
         static_kv = self.encoder_decoder_attention  # value=key=encoder_hidden_states,
-        tgt_len, bsz, embed_dim = query.shape
-        assert embed_dim == self.embed_dim, f"query must be shaped {(tgt_len, bsz, self.embed_dim)} got {query.shape}"
+        tgt_len, bsz, embed_dim = shape_list(query)
+        assert (
+            embed_dim == self.embed_dim
+        ), f"query must be shaped {(tgt_len, bsz, self.embed_dim)} got {shape_list(query)}"
         # get here for encoder decoder cause of static_kv
         if layer_state is not None:  # get the last k and v for reuse
             saved_state = layer_state.get(self.cache_key, {})
@@ -731,7 +735,7 @@ def call(
             )
 
         # Compute multi-headed attention
-        src_len = k.shape[1]
+        src_len = shape_list(k)[1]
         attn_weights = tf.matmul(q, k, transpose_b=True)  # shape (bsz * self.num_heads, tgt_len, src_len)
 
         if attn_mask is not None:
@@ -783,7 +787,7 @@ def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, of
 
     def call(self, input_ids: tf.Tensor, use_cache=False):
         """Input is expected to be of size [bsz x seqlen]."""
-        bsz, seq_len = input_ids.shape[:2]
+        bsz, seq_len = shape_list(input_ids)[:2]
 
         if use_cache:
             positions = tf.fill((1, 1), seq_len - 1)
@@ -834,7 +838,7 @@ def _init_weight(n_pos, dim):
 
     def call(self, input_ids, use_cache=False):
         """Input is expected to be of size [bsz x seqlen]."""
-        bsz, seq_len = input_ids.shape[:2]
+        bsz, seq_len = shape_list(input_ids)[:2]
         if use_cache:
             positions = tf.fill((1, 1), seq_len - 1)
         else:
@@ -881,7 +885,7 @@ def _prepare_bart_decoder_inputs(
         pad_token_id = self.config.pad_token_id
         if decoder_input_ids is None:
             decoder_input_ids = self._shift_right(inputs)
-        bsz, tgt_len = decoder_input_ids.shape[:2]
+        bsz, tgt_len = shape_list(decoder_input_ids)[:2]
         if decoder_attn_mask is None:
             decoder_padding_mask = make_padding_mask(decoder_input_ids, pad_token_id)
         else:
@@ -1180,7 +1184,7 @@ def _reorder_cache(past, beam_idx):
         (encoder_out, decoder_cached_states) = past
         reordered_past = []
         for layer_past in decoder_cached_states:
-            # get the correct batch idx from decoder layer's batch dim for cross and self-attn
+            # get the correct batch idx from decod    er layer's batch dim for cross and self-attn
 
             layer_past_new = {
                 attn_key: _reorder_buffer(attn_cache, beam_idx) for attn_key, attn_cache in layer_past.items()
@@ -1203,7 +1207,7 @@ def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
         # TODO: https://github.com/huggingface/transformers/issues/7954
         output_list = []
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
-        bs, vocab_size = scores.shape
+        bs, vocab_size = shape_list(scores)
         inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
         for x in range(vocab_size):
             do_inf = (x == token_id) if inverted else (x != token_id)
@@ -1212,7 +1216,7 @@ def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
             else:
                 output_list.append(scores[:, x])
         scores = tf.stack(output_list, axis=1, name="scores")
-        assert scores.shape == (bs, vocab_size)
+        assert shape_list(scores) == [bs, vocab_size]
         return scores
 
     def get_output_embeddings(self):
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 686529ac1aa4..51e90de6e03e 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -46,3 +46,21 @@ def adjust_logits_during_generation(self, logits, cur_len, max_length):
         return logits
 
     # All the code is in src/transformers/modeling_tf_bart.py
+    # # This is copied from src/transformers/modeling_tf_bart.py for reference
+    # @staticmethod
+    # def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
+    #     """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
+    #     # TODO: https://github.com/huggingface/transformers/issues/7954
+    #     output_list = []
+    #     # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
+    #     bs, vocab_size = scores.shape
+    #     inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
+    #     for x in range(vocab_size):
+    #         do_inf = (x == token_id) if inverted else (x != token_id)
+    #         if do_inf:
+    #             output_list.append(inf_tensor)
+    #         else:
+    #             output_list.append(scores[:, x])
+    #     scores = tf.stack(output_list, axis=1, name="scores")
+    #     assert scores.shape == (bs, vocab_size)
+    #     return scores

From 0618986b122ce813da6e9d5e08921a670c960d31 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 28 Oct 2020 14:41:31 -0400
Subject: [PATCH 39/40] delete slow force_token_id_to_be_generated func

---
 src/transformers/modeling_tf_bart.py       | 29 +++++------------
 src/transformers/modeling_tf_blenderbot.py | 16 +++++++---
 src/transformers/modeling_tf_marian.py     | 36 +++++++---------------
 tests/test_modeling_marian.py              |  4 ++-
 tests/test_modeling_tf_marian.py           | 20 +++++++++++-
 5 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 0c4a659ebf7d..5c1d3ac835aa 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -1196,28 +1196,13 @@ def _reorder_cache(past, beam_idx):
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
         if cur_len == 1 and self.config.force_bos_token_to_be_generated:
-            logits = self._force_token_id_to_be_generated(logits, self.config.bos_token_id)
-        elif cur_len == max_length - 1 and self.config.eos_token_id is not None:
-            logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
-        return logits
-
-    @staticmethod
-    def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
-        """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
-        # TODO: https://github.com/huggingface/transformers/issues/7954
-        output_list = []
-        # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
-        bs, vocab_size = shape_list(scores)
-        inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
-        for x in range(vocab_size):
-            do_inf = (x == token_id) if inverted else (x != token_id)
-            if do_inf:
-                output_list.append(inf_tensor)
-            else:
-                output_list.append(scores[:, x])
-        scores = tf.stack(output_list, axis=1, name="scores")
-        assert shape_list(scores) == [bs, vocab_size]
-        return scores
+            vocab_range = tf.constant(range(self.config.vocab_size))
+            return tf.where(vocab_range != self.config.bos_token_id, LARGE_NEGATIVE, logits)
+        elif cur_len == max_length - 1:
+            vocab_range = tf.constant(range(self.config.vocab_size))
+            return tf.where(vocab_range != self.config.eos_token_id, LARGE_NEGATIVE, logits)
+        else:
+            return logits
 
     def get_output_embeddings(self):
         return self.model.shared
diff --git a/src/transformers/modeling_tf_blenderbot.py b/src/transformers/modeling_tf_blenderbot.py
index 4df826f3c4e4..633b50ec7757 100644
--- a/src/transformers/modeling_tf_blenderbot.py
+++ b/src/transformers/modeling_tf_blenderbot.py
@@ -14,11 +14,15 @@
 # limitations under the License.
 """TF BlenderBot model, ported from the fairseq repo."""
 from .configuration_blenderbot import BlenderbotConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
+from .file_utils import add_start_docstrings, is_tf_available
+from .modeling_tf_bart import BART_START_DOCSTRING, LARGE_NEGATIVE, TFBartForConditionalGeneration
 from .utils import logging
 
 
+if is_tf_available():
+    import tensorflow as tf
+
+
 _CONFIG_FOR_DOC = "BlenderbotConfig"
 
 START_DOCSTRING = BART_START_DOCSTRING.replace(
@@ -35,7 +39,9 @@ class TFBlenderbotForConditionalGeneration(TFBartForConditionalGeneration):
     config_class = BlenderbotConfig
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
-        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
-            self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
+        """Never predict pad_token_id. Predict </s> when max_length is reached."""
+        vocab_range = tf.constant(range(self.config.vocab_size))
+        logits = tf.where(vocab_range == self.config.pad_token_id, LARGE_NEGATIVE, logits)
+        if cur_len == max_length - 1:
+            logits = tf.where(vocab_range != self.config.eos_token_id, LARGE_NEGATIVE, logits)
         return logits
diff --git a/src/transformers/modeling_tf_marian.py b/src/transformers/modeling_tf_marian.py
index 51e90de6e03e..9dcd5489660a 100644
--- a/src/transformers/modeling_tf_marian.py
+++ b/src/transformers/modeling_tf_marian.py
@@ -13,12 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """TF Marian model, ported from the fairseq repo."""
+
 from .configuration_marian import MarianConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
+from .file_utils import add_start_docstrings, is_tf_available
+from .modeling_tf_bart import BART_START_DOCSTRING, LARGE_NEGATIVE, TFBartForConditionalGeneration
 from .utils import logging
 
 
+if is_tf_available():
+    import tensorflow as tf
+
+
 _CONFIG_FOR_DOC = "MarianConfig"
 
 START_DOCSTRING = BART_START_DOCSTRING.replace(
@@ -40,27 +45,8 @@ class TFMarianMTModel(TFBartForConditionalGeneration):
 
     def adjust_logits_during_generation(self, logits, cur_len, max_length):
         """Never predict pad_token_id. Predict </s> when max_length is reached."""
-        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
-        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
-            logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
+        vocab_range = tf.constant(range(self.config.vocab_size))
+        logits = tf.where(vocab_range == self.config.pad_token_id, LARGE_NEGATIVE, logits)
+        if cur_len == max_length - 1:
+            logits = tf.where(vocab_range != self.config.eos_token_id, LARGE_NEGATIVE, logits)
         return logits
-
-    # All the code is in src/transformers/modeling_tf_bart.py
-    # # This is copied from src/transformers/modeling_tf_bart.py for reference
-    # @staticmethod
-    # def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
-    #     """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
-    #     # TODO: https://github.com/huggingface/transformers/issues/7954
-    #     output_list = []
-    #     # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
-    #     bs, vocab_size = scores.shape
-    #     inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
-    #     for x in range(vocab_size):
-    #         do_inf = (x == token_id) if inverted else (x != token_id)
-    #         if do_inf:
-    #             output_list.append(inf_tensor)
-    #         else:
-    #             output_list.append(scores[:, x])
-    #     scores = tf.stack(output_list, axis=1, name="scores")
-    #     assert scores.shape == (bs, vocab_size)
-    #     return scores
diff --git a/tests/test_modeling_marian.py b/tests/test_modeling_marian.py
index 3859f4348248..f19d4365e16a 100644
--- a/tests/test_modeling_marian.py
+++ b/tests/test_modeling_marian.py
@@ -137,7 +137,7 @@ def translate_src_text(self, **tokenizer_kwargs):
         )
         self.assertEqual(self.model.device, model_inputs.input_ids.device)
         generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2, max_length=128
         )
         generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
         return generated_words
@@ -243,6 +243,8 @@ def test_batch_generation_ru_fr(self):
 @require_sentencepiece
 @require_tokenizers
 class TestMarian_MT_EN(MarianIntegrationTest):
+    """Cover low resource/high perplexity setting. This breaks without adjust_logits_generation overwritten"""
+
     src = "mt"
     tgt = "en"
     src_text = ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py
index 85348981d56c..a713023d4f1f 100644
--- a/tests/test_modeling_tf_marian.py
+++ b/tests/test_modeling_tf_marian.py
@@ -96,6 +96,8 @@ def test_compile_tf_model(self):
 
 
 class AbstractMarianIntegrationTest(unittest.TestCase):
+    maxDiff = 1000  # show more chars for failing integration tests
+
     @classmethod
     def setUpClass(cls) -> None:
         cls.model_name = f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
@@ -129,12 +131,28 @@ def translate_src_text(self, **tokenizer_kwargs):
             src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
         )
         generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
+            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2, max_length=128
         )
         generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
         return generated_words
 
 
+@require_sentencepiece
+@require_tokenizers
+@is_pt_tf_cross_test
+class TestMarian_MT_EN(AbstractMarianIntegrationTest):
+    """Cover low resource/high perplexity setting. This breaks if pad_token_id logits not set to LARGE_NEGATIVE."""
+
+    src = "mt"
+    tgt = "en"
+    src_text = ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
+    expected_text = ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]
+
+    @slow
+    def test_batch_generation_mt_en(self):
+        self._assert_generated_batch_equal_expected()
+
+
 @is_pt_tf_cross_test
 @require_sentencepiece
 @require_tokenizers

From 60d2ab496310cf5d97e2d7d427bfbfb592fe6ce9 Mon Sep 17 00:00:00 2001
From: Sam Shleifer <sshleifer@gmail.com>
Date: Wed, 28 Oct 2020 14:42:42 -0400
Subject: [PATCH 40/40] fixup

---
 src/transformers/modeling_tf_bart.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/transformers/modeling_tf_bart.py b/src/transformers/modeling_tf_bart.py
index 5c1d3ac835aa..6814520a6342 100644
--- a/src/transformers/modeling_tf_bart.py
+++ b/src/transformers/modeling_tf_bart.py
@@ -1184,8 +1184,7 @@ def _reorder_cache(past, beam_idx):
         (encoder_out, decoder_cached_states) = past
         reordered_past = []
         for layer_past in decoder_cached_states:
-            # get the correct batch idx from decod    er layer's batch dim for cross and self-attn
-
+            # get the correct batch idx from decoder layer's batch dim for cross and self-attn
             layer_past_new = {
                 attn_key: _reorder_buffer(attn_cache, beam_idx) for attn_key, attn_cache in layer_past.items()
             }