Doctest longformer

src/transformers/models/longformer/modeling_longformer.py

@@ -1407,6 +1407,28 @@ def _set_gradient_checkpointing(self, module, value=False):
             module.gradient_checkpointing = value
 
 
+LONGFORMER_GENERATION_DOCSTRING = r"""
+    Mask filling example:
+
+    ```python
+    >>> from transformers import LongformerTokenizer, LongformerForMaskedLM
+
+    >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096")
+    >>> model = LongformerForMaskedLM.from_pretrained("allenai/longformer-base-4096")
+
+    >>> TXT = "My friends are <mask> but they eat too many carbs." + " That's why I decide not to eat with them." * 300
+    >>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"]
+    >>> logits = model(input_ids).logits
+
+    >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
+    >>> probs = logits[0, masked_index].softmax(dim=0)
+    >>> values, predictions = probs.topk(5)
+
+    >>> tokenizer.decode(predictions).split()
+    ['healthy', 'skinny', 'thin', 'good', 'vegetarian']
+    ```
+"""
+
 LONGFORMER_START_DOCSTRING = r"""
 
     This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
@@ -1852,9 +1874,11 @@ def __init__(self, config):
     @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
     @add_code_sample_docstrings(
         processor_class=_TOKENIZER_FOR_DOC,
-        checkpoint=_CHECKPOINT_FOR_DOC,
+        checkpoint="jpelhaw/longformer-base-plagiarism-detection",
         output_type=LongformerSequenceClassifierOutput,
         config_class=_CONFIG_FOR_DOC,
+        expected_output=[1, 2],
+        expected_loss=0.08,
     )
     def forward(
         self,
@@ -2118,9 +2142,11 @@ def __init__(self, config):
     @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
     @add_code_sample_docstrings(
         processor_class=_TOKENIZER_FOR_DOC,
-        checkpoint=_CHECKPOINT_FOR_DOC,
+        checkpoint="brad1141/Longformer-finetuned-norm",
         output_type=LongformerTokenClassifierOutput,
         config_class=_CONFIG_FOR_DOC,
+        expected_output="['Lead', 'Evidence', 'Lead', 'Evidence', 'Lead']",
+        expected_loss=0.01,
     )
     def forward(
         self,

src/transformers/models/longformer/modeling_tf_longformer.py

@@ -2081,10 +2081,12 @@ def get_prefix_bias_name(self):
     @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
     @add_code_sample_docstrings(
         processor_class=_TOKENIZER_FOR_DOC,
-        checkpoint=_CHECKPOINT_FOR_DOC,
+        checkpoint="saibo/legal-longformer-base-4096",
         output_type=TFLongformerMaskedLMOutput,
         config_class=_CONFIG_FOR_DOC,
         mask="<mask>",
+        expected_output="' no'",
+        expected_loss=4.5,
     )
     def call(
         self,
@@ -2178,6 +2180,8 @@ def __init__(self, config, *inputs, **kwargs):
         checkpoint="allenai/longformer-large-4096-finetuned-triviaqa",
         output_type=TFLongformerQuestionAnsweringModelOutput,
         config_class=_CONFIG_FOR_DOC,
+        expected_output="' puppet'",
+        expected_loss=0.96,
     )
     def call(
         self,
@@ -2325,6 +2329,8 @@ def __init__(self, config, *inputs, **kwargs):
         checkpoint=_CHECKPOINT_FOR_DOC,
         output_type=TFLongformerSequenceClassifierOutput,
         config_class=_CONFIG_FOR_DOC,
+        expected_output="LABEL_0",
+        expected_loss=0.58,
     )
     def call(
         self,
@@ -2565,6 +2571,8 @@ def __init__(self, config, *inputs, **kwargs):
         checkpoint=_CHECKPOINT_FOR_DOC,
         output_type=TFLongformerTokenClassifierOutput,
         config_class=_CONFIG_FOR_DOC,
+        expected_output="['LABEL_0', 'LABEL_0', 'LABEL_0', 'LABEL_0', 'LABEL_0', 'LABEL_0', 'LABEL_1', 'LABEL_1', 'LABEL_0', 'LABEL_0', 'LABEL_0', 'LABEL_0']",
+        expected_loss=0.62,
     )
     def call(
         self,

utils/documentation_tests.txt

@@ -18,6 +18,7 @@ src/transformers/models/data2vec/modeling_data2vec_audio.py
 src/transformers/models/deit/modeling_deit.py
 src/transformers/models/glpn/modeling_glpn.py
 src/transformers/models/hubert/modeling_hubert.py
+src/transformers/models/longformer/modeling_longformer.py
 src/transformers/models/marian/modeling_marian.py
 src/transformers/models/marian/modeling_marian.py
 src/transformers/models/mbart/modeling_mbart.py