From dfa0987e4f03a86a4486f4c37d82f3c0ec06a8e7 Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Wed, 21 Apr 2021 22:48:52 +0530
Subject: [PATCH 01/12] added changes for uniformity

---
 examples/pytorch/language-modeling/run_clm.py | 12 ++---
 examples/pytorch/language-modeling/run_mlm.py | 12 ++---
 examples/pytorch/language-modeling/run_plm.py | 12 ++---
 examples/pytorch/multiple-choice/run_swag.py  | 12 ++---
 examples/pytorch/question-answering/run_qa.py | 46 ++++++++--------
 .../question-answering/run_qa_beam_search.py  | 34 ++++++------
 .../run_qa_beam_search_no_trainer.py          | 54 +++++++++----------
 .../question-answering/run_qa_no_trainer.py   | 50 ++++++++---------
 .../summarization/run_summarization.py        | 44 +++++++--------
 .../pytorch/text-classification/run_glue.py   | 42 ++++++++-------
 .../pytorch/text-classification/run_xnli.py   | 42 ++++++++-------
 .../pytorch/token-classification/run_ner.py   | 34 ++++++------
 .../pytorch/translation/run_translation.py    | 54 ++++++++++---------
 src/transformers/trainer.py                   | 42 +++++++--------
 src/transformers/trainer_pt_utils.py          |  4 +-
 .../run_{{cookiecutter.example_shortcut}}.py  | 36 ++++++-------
 16 files changed, 271 insertions(+), 259 deletions(-)

diff --git a/examples/pytorch/language-modeling/run_clm.py b/examples/pytorch/language-modeling/run_clm.py
index 3bc3fe40ff5d..c1831c997db8 100755
--- a/examples/pytorch/language-modeling/run_clm.py
+++ b/examples/pytorch/language-modeling/run_clm.py
@@ -126,10 +126,10 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
@@ -397,8 +397,8 @@ def group_texts(examples):
         if "validation" not in tokenized_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = lm_datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     # Initialize our Trainer
     trainer = Trainer(
@@ -439,8 +439,8 @@ def group_texts(examples):
 
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
         perplexity = math.exp(metrics["eval_loss"])
         metrics["perplexity"] = perplexity
 
diff --git a/examples/pytorch/language-modeling/run_mlm.py b/examples/pytorch/language-modeling/run_mlm.py
index 1cb9fcc6cdd4..51b3cb8f5699 100755
--- a/examples/pytorch/language-modeling/run_mlm.py
+++ b/examples/pytorch/language-modeling/run_mlm.py
@@ -157,10 +157,10 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
@@ -419,8 +419,8 @@ def group_texts(examples):
         if "validation" not in tokenized_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = tokenized_datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     # Data collator
     # This one will take care of randomly masking the tokens.
@@ -468,8 +468,8 @@ def group_texts(examples):
 
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
         perplexity = math.exp(metrics["eval_loss"])
         metrics["perplexity"] = perplexity
 
diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py
index b57348157815..7ea5582d8cd8 100755
--- a/examples/pytorch/language-modeling/run_plm.py
+++ b/examples/pytorch/language-modeling/run_plm.py
@@ -154,10 +154,10 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
@@ -397,8 +397,8 @@ def group_texts(examples):
         if "validation" not in tokenized_datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = tokenized_datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     # Data collator
     data_collator = DataCollatorForPermutationLanguageModeling(
@@ -444,8 +444,8 @@ def group_texts(examples):
 
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
         perplexity = math.exp(metrics["eval_loss"])
         metrics["perplexity"] = perplexity
 
diff --git a/examples/pytorch/multiple-choice/run_swag.py b/examples/pytorch/multiple-choice/run_swag.py
index de012171938c..023105331bb1 100755
--- a/examples/pytorch/multiple-choice/run_swag.py
+++ b/examples/pytorch/multiple-choice/run_swag.py
@@ -127,10 +127,10 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
@@ -363,8 +363,8 @@ def preprocess_function(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         eval_dataset = eval_dataset.map(
             preprocess_function,
             batched=True,
@@ -422,8 +422,8 @@ def compute_metrics(eval_predictions):
         logger.info("*** Evaluate ***")
 
         metrics = trainer.evaluate()
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
diff --git a/examples/pytorch/question-answering/run_qa.py b/examples/pytorch/question-answering/run_qa.py
index d275a2992212..2ae860e0d504 100755
--- a/examples/pytorch/question-answering/run_qa.py
+++ b/examples/pytorch/question-answering/run_qa.py
@@ -133,17 +133,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -468,9 +468,9 @@ def prepare_validation_features(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_examples = datasets["validation"]
-        if data_args.max_val_samples is not None:
+        if data_args.max_eval_samples is not None:
             # We will select sample from whole data
-            eval_examples = eval_examples.select(range(data_args.max_val_samples))
+            eval_examples = eval_examples.select(range(data_args.max_eval_samples))
         # Validation Feature Creation
         eval_dataset = eval_examples.map(
             prepare_validation_features,
@@ -479,28 +479,28 @@ def prepare_validation_features(examples):
             remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
-        if data_args.max_val_samples is not None:
+        if data_args.max_eval_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     if training_args.do_predict:
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_examples = datasets["test"]
-        if data_args.max_test_samples is not None:
+        predict_examples = datasets["test"]
+        if data_args.max_predict_samples is not None:
             # We will select sample from whole data
-            test_examples = test_examples.select(range(data_args.max_test_samples))
-        # Test Feature Creation
-        test_dataset = test_examples.map(
+            predict_examples = predict_examples.select(range(data_args.max_predict_samples))
+        # Predict Feature Creation
+        predict_dataset = predict_examples.map(
             prepare_validation_features,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
             remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
-        if data_args.max_test_samples is not None:
+        if data_args.max_predict_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
 
     # Data collator
     # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data
@@ -581,8 +581,8 @@ def compute_metrics(p: EvalPrediction):
         logger.info("*** Evaluate ***")
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
@@ -590,14 +590,16 @@ def compute_metrics(p: EvalPrediction):
     # Prediction
     if training_args.do_predict:
         logger.info("*** Predict ***")
-        results = trainer.predict(test_dataset, test_examples)
+        results = trainer.predict(predict_dataset, predict_examples)
         metrics = results.metrics
 
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        max_predict_samples = (
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
+        )
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
 
 def _mp_fn(index):
diff --git a/examples/pytorch/question-answering/run_qa_beam_search.py b/examples/pytorch/question-answering/run_qa_beam_search.py
index 490ecae86eb7..0d1d14845cd1 100755
--- a/examples/pytorch/question-answering/run_qa_beam_search.py
+++ b/examples/pytorch/question-answering/run_qa_beam_search.py
@@ -132,17 +132,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -504,9 +504,9 @@ def prepare_validation_features(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_examples = datasets["validation"]
-        if data_args.max_val_samples is not None:
+        if data_args.max_eval_samples is not None:
             # Selecting Eval Samples from Dataset
-            eval_examples = eval_examples.select(range(data_args.max_val_samples))
+            eval_examples = eval_examples.select(range(data_args.max_eval_samples))
         # Create Features from Eval Dataset
         eval_dataset = eval_examples.map(
             prepare_validation_features,
@@ -515,17 +515,17 @@ def prepare_validation_features(examples):
             remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
-        if data_args.max_val_samples is not None:
+        if data_args.max_eval_samples is not None:
             # Selecting Samples from Dataset again since Feature Creation might increase samples size
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     if training_args.do_predict:
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
         test_examples = datasets["test"]
-        if data_args.max_test_samples is not None:
+        if data_args.max_predict_samples is not None:
             # We will select sample from whole data
-            test_examples = test_examples.select(range(data_args.max_test_samples))
+            test_examples = test_examples.select(range(data_args.max_predict_samples))
         # Test Feature Creation
         test_dataset = test_examples.map(
             prepare_validation_features,
@@ -534,9 +534,9 @@ def prepare_validation_features(examples):
             remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
-        if data_args.max_test_samples is not None:
+        if data_args.max_predict_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
+            test_dataset = test_dataset.select(range(data_args.max_predict_samples))
 
     # Data collator
     # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data
@@ -620,8 +620,8 @@ def compute_metrics(p: EvalPrediction):
         logger.info("*** Evaluate ***")
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
@@ -632,8 +632,10 @@ def compute_metrics(p: EvalPrediction):
         results = trainer.predict(test_dataset, test_examples)
         metrics = results.metrics
 
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        max_predict_samples = (
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(test_dataset)
+        )
+        metrics["test_samples"] = min(max_predict_samples, len(test_dataset))
 
         trainer.log_metrics("test", metrics)
         trainer.save_metrics("test", metrics)
diff --git a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py
index ca0d60c0f8d1..f1d5a2d03083 100644
--- a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py
+++ b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py
@@ -183,20 +183,20 @@ def parse_args():
         "value if set.",
     )
     parser.add_argument(
-        "--max_val_samples",
+        "--max_eval_samples",
         type=int,
         default=None,
-        help="For debugging purposes or quicker training, truncate the number of validation examples to this "
+        help="For debugging purposes or quicker training, truncate the number of evaluation examples to this "
         "value if set.",
     )
     parser.add_argument(
         "--overwrite_cache", type=bool, default=False, help="Overwrite the cached training and evaluation sets"
     )
     parser.add_argument(
-        "--max_test_samples",
+        "--max_predict_samples",
         type=int,
         default=None,
-        help="For debugging purposes or quicker training, truncate the number of test examples to this",
+        help="For debugging purposes or quicker training, truncate the number of prediction examples to this",
     )
 
     args = parser.parse_args()
@@ -481,9 +481,9 @@ def prepare_validation_features(examples):
     if "validation" not in raw_datasets:
         raise ValueError("--do_eval requires a validation dataset")
     eval_examples = raw_datasets["validation"]
-    if args.max_val_samples is not None:
+    if args.max_eval_samples is not None:
         # We will select sample from whole data
-        eval_examples = eval_examples.select(range(args.max_val_samples))
+        eval_examples = eval_examples.select(range(args.max_eval_samples))
     # Validation Feature Creation
     eval_dataset = eval_examples.map(
         prepare_validation_features,
@@ -493,28 +493,28 @@ def prepare_validation_features(examples):
         load_from_cache_file=not args.overwrite_cache,
     )
 
-    if args.max_val_samples is not None:
+    if args.max_eval_samples is not None:
         # During Feature creation dataset samples might increase, we will select required samples again
-        eval_dataset = eval_dataset.select(range(args.max_val_samples))
+        eval_dataset = eval_dataset.select(range(args.max_eval_samples))
 
     if args.do_predict:
         if "test" not in raw_datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_examples = raw_datasets["test"]
-        if args.max_test_samples is not None:
+        predict_examples = raw_datasets["test"]
+        if args.max_predict_samples is not None:
             # We will select sample from whole data
-            test_examples = test_examples.select(range(args.max_test_samples))
-        # Test Feature Creation
-        test_dataset = test_examples.map(
+            predict_examples = predict_examples.select(range(args.max_predict_samples))
+        # Predict Feature Creation
+        predict_dataset = predict_examples.map(
             prepare_validation_features,
             batched=True,
             num_proc=args.preprocessing_num_workers,
             remove_columns=column_names,
             load_from_cache_file=not args.overwrite_cache,
         )
-        if args.max_test_samples is not None:
+        if args.max_predict_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            test_dataset = test_dataset.select(range(args.max_test_samples))
+            predict_dataset = predict_dataset.select(range(args.max_predict_samples))
 
     # Log a few random samples from the training set:
     for index in random.sample(range(len(train_dataset)), 3):
@@ -539,9 +539,9 @@ def prepare_validation_features(examples):
     eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)
 
     if args.do_predict:
-        test_dataset.set_format(type="torch", columns=["attention_mask", "input_ids", "token_type_ids"])
-        test_dataloader = DataLoader(
-            test_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size
+        predict_dataset.set_format(type="torch", columns=["attention_mask", "input_ids", "token_type_ids"])
+        predict_dataloader = DataLoader(
+            predict_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size
         )
 
     # Post-processing:
@@ -737,7 +737,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
         all_end_top_log_probs = []
         all_end_top_index = []
         all_cls_logits = []
-        for step, batch in enumerate(test_dataloader):
+        for step, batch in enumerate(predict_dataloader):
             with torch.no_grad():
                 outputs = model(**batch)
                 start_top_log_probs = outputs.start_top_log_probs
@@ -762,10 +762,10 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
         max_len = max([x.shape[1] for x in all_end_top_log_probs])  # Get the max_length of the tensor
 
         # concatenate all numpy arrays collected above
-        start_top_log_probs_concat = create_and_fill_np_array(all_start_top_log_probs, test_dataset, max_len)
-        start_top_index_concat = create_and_fill_np_array(all_start_top_index, test_dataset, max_len)
-        end_top_log_probs_concat = create_and_fill_np_array(all_end_top_log_probs, test_dataset, max_len)
-        end_top_index_concat = create_and_fill_np_array(all_end_top_index, test_dataset, max_len)
+        start_top_log_probs_concat = create_and_fill_np_array(all_start_top_log_probs, predict_dataset, max_len)
+        start_top_index_concat = create_and_fill_np_array(all_start_top_index, predict_dataset, max_len)
+        end_top_log_probs_concat = create_and_fill_np_array(all_end_top_log_probs, predict_dataset, max_len)
+        end_top_index_concat = create_and_fill_np_array(all_end_top_index, predict_dataset, max_len)
         all_cls_logits = np.concatenate(all_cls_logits, axis=0)
 
         # delete the list of numpy arrays
@@ -774,7 +774,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
         del end_top_log_probs
         del end_top_index
 
-        test_dataset.set_format(type=None, columns=list(test_dataset.features.keys()))
+        predict_dataset.set_format(type=None, columns=list(predict_dataset.features.keys()))
         outputs_numpy = (
             start_top_log_probs_concat,
             start_top_index_concat,
@@ -783,9 +783,9 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
             cls_logits,
         )
 
-        prediction = post_processing_function(test_examples, test_dataset, outputs_numpy)
-        test_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)
-        logger.info(f"Test metrics: {test_metric}")
+        prediction = post_processing_function(predict_examples, predict_dataset, outputs_numpy)
+        predict_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)
+        logger.info(f"Predict metrics: {predict_metric}")
 
     if args.output_dir is not None:
         accelerator.wait_for_everyone()
diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py
index 7a8b2215be75..6bdda4816a84 100755
--- a/examples/pytorch/question-answering/run_qa_no_trainer.py
+++ b/examples/pytorch/question-answering/run_qa_no_trainer.py
@@ -205,20 +205,20 @@ def parse_args():
         "value if set.",
     )
     parser.add_argument(
-        "--max_val_samples",
+        "--max_eval_samples",
         type=int,
         default=None,
-        help="For debugging purposes or quicker training, truncate the number of validation examples to this "
+        help="For debugging purposes or quicker training, truncate the number of evaluation examples to this "
         "value if set.",
     )
     parser.add_argument(
         "--overwrite_cache", type=bool, default=False, help="Overwrite the cached training and evaluation sets"
     )
     parser.add_argument(
-        "--max_test_samples",
+        "--max_predict_samples",
         type=int,
         default=None,
-        help="For debugging purposes or quicker training, truncate the number of test examples to this",
+        help="For debugging purposes or quicker training, truncate the number of prediction examples to this",
     )
     parser.add_argument(
         "--model_type",
@@ -486,9 +486,9 @@ def prepare_validation_features(examples):
     if "validation" not in raw_datasets:
         raise ValueError("--do_eval requires a validation dataset")
     eval_examples = raw_datasets["validation"]
-    if args.max_val_samples is not None:
+    if args.max_eval_samples is not None:
         # We will select sample from whole data
-        eval_examples = eval_examples.select(range(args.max_val_samples))
+        eval_examples = eval_examples.select(range(args.max_eval_samples))
     # Validation Feature Creation
     eval_dataset = eval_examples.map(
         prepare_validation_features,
@@ -498,28 +498,28 @@ def prepare_validation_features(examples):
         load_from_cache_file=not args.overwrite_cache,
     )
 
-    if args.max_val_samples is not None:
+    if args.max_eval_samples is not None:
         # During Feature creation dataset samples might increase, we will select required samples again
-        eval_dataset = eval_dataset.select(range(args.max_val_samples))
+        eval_dataset = eval_dataset.select(range(args.max_eval_samples))
 
     if args.do_predict:
         if "test" not in raw_datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_examples = raw_datasets["test"]
-        if args.max_test_samples is not None:
+        predict_examples = raw_datasets["test"]
+        if args.max_predict_samples is not None:
             # We will select sample from whole data
-            test_examples = test_examples.select(range(args.max_test_samples))
-        # Test Feature Creation
-        test_dataset = test_examples.map(
+            predict_examples = predict_examples.select(range(args.max_predict_samples))
+        # Predict Feature Creation
+        predict_examples = predict_examples.map(
             prepare_validation_features,
             batched=True,
             num_proc=args.preprocessing_num_workers,
             remove_columns=column_names,
             load_from_cache_file=not args.overwrite_cache,
         )
-        if args.max_test_samples is not None:
+        if args.max_predict_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            test_dataset = test_dataset.select(range(args.max_test_samples))
+            predict_dataset = predict_dataset.select(range(args.max_predict_samples))
 
     # Log a few random samples from the training set:
     for index in random.sample(range(len(train_dataset)), 3):
@@ -544,9 +544,9 @@ def prepare_validation_features(examples):
     eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)
 
     if args.do_predict:
-        test_dataset.set_format(type="torch", columns=["attention_mask", "input_ids", "token_type_ids"])
-        test_dataloader = DataLoader(
-            test_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size
+        predict_dataset.set_format(type="torch", columns=["attention_mask", "input_ids", "token_type_ids"])
+        predict_dataloader = DataLoader(
+            predict_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size
         )
 
     # Post-processing:
@@ -714,7 +714,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
     if args.do_predict:
         all_start_logits = []
         all_end_logits = []
-        for step, batch in enumerate(test_dataloader):
+        for step, batch in enumerate(predict_dataloader):
             with torch.no_grad():
                 outputs = model(**batch)
                 start_logits = outputs.start_logits
@@ -729,19 +729,19 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
 
         max_len = max([x.shape[1] for x in all_start_logits])  # Get the max_length of the tensor
         # concatenate the numpy array
-        start_logits_concat = create_and_fill_np_array(all_start_logits, test_dataset, max_len)
-        end_logits_concat = create_and_fill_np_array(all_end_logits, test_dataset, max_len)
+        start_logits_concat = create_and_fill_np_array(all_start_logits, predict_dataset, max_len)
+        end_logits_concat = create_and_fill_np_array(all_end_logits, predict_dataset, max_len)
 
         # delete the list of numpy arrays
         del all_start_logits
         del all_end_logits
 
         # Now we need to add extra columns which we removed for post processing
-        test_dataset.set_format(type=None, columns=list(test_dataset.features.keys()))
+        predict_dataset.set_format(type=None, columns=list(predict_dataset.features.keys()))
         outputs_numpy = (start_logits_concat, end_logits_concat)
-        prediction = post_processing_function(test_examples, test_dataset, outputs_numpy)
-        eval_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)
-        logger.info(f"Test metrics: {eval_metric}")
+        prediction = post_processing_function(test_examples, predict_dataset, outputs_numpy)
+        predict_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)
+        logger.info(f"Predict metrics: {predict_metric}")
 
     if args.output_dir is not None:
         accelerator.wait_for_everyone()
diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py
index 0856d6dce5cb..aa457576101e 100755
--- a/examples/pytorch/summarization/run_summarization.py
+++ b/examples/pytorch/summarization/run_summarization.py
@@ -178,17 +178,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -438,8 +438,8 @@ def preprocess_function(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         eval_dataset = eval_dataset.map(
             preprocess_function,
             batched=True,
@@ -452,10 +452,10 @@ def preprocess_function(examples):
         max_target_length = data_args.val_max_target_length
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_dataset = datasets["test"]
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
-        test_dataset = test_dataset.map(
+        predict_dataset = datasets["test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
+        predict_dataset = predict_dataset.map(
             preprocess_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
@@ -547,27 +547,29 @@ def compute_metrics(eval_preds):
         metrics = trainer.evaluate(
             max_length=data_args.val_max_target_length, num_beams=data_args.num_beams, metric_key_prefix="eval"
         )
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
 
     if training_args.do_predict:
-        logger.info("*** Test ***")
+        logger.info("*** Predict ***")
 
-        test_results = trainer.predict(
-            test_dataset,
+        predict_results = trainer.predict(
+            predict_dataset,
             metric_key_prefix="test",
             max_length=data_args.val_max_target_length,
             num_beams=data_args.num_beams,
         )
-        metrics = test_results.metrics
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        metrics = predict_results.metrics
+        max_predict_samples = (
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
+        )
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
         if trainer.is_world_process_zero():
             if training_args.predict_with_generate:
@@ -575,7 +577,7 @@ def compute_metrics(eval_preds):
                     test_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
                 )
                 test_preds = [pred.strip() for pred in test_preds]
-                output_test_preds_file = os.path.join(training_args.output_dir, "test_generations.txt")
+                output_test_preds_file = os.path.join(training_args.output_dir, "predict_generations.txt")
                 with open(output_test_preds_file, "w") as writer:
                     writer.write("\n".join(test_preds))
 
diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py
index 0af73cf5ddb5..d1dc7fa89d30 100755
--- a/examples/pytorch/text-classification/run_glue.py
+++ b/examples/pytorch/text-classification/run_glue.py
@@ -100,17 +100,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -390,15 +390,15 @@ def preprocess_function(examples):
         if "validation" not in datasets and "validation_matched" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation_matched" if data_args.task_name == "mnli" else "validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None:
         if "test" not in datasets and "test_matched" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_dataset = datasets["test_matched" if data_args.task_name == "mnli" else "test"]
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
+        predict_dataset = datasets["test_matched" if data_args.task_name == "mnli" else "test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
 
     # Log a few random samples from the training set:
     if training_args.do_train:
@@ -483,32 +483,34 @@ def compute_metrics(p: EvalPrediction):
         for eval_dataset, task in zip(eval_datasets, tasks):
             metrics = trainer.evaluate(eval_dataset=eval_dataset)
 
-            max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-            metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+            max_eval_samples = (
+                data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+            )
+            metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
             trainer.log_metrics("eval", metrics)
             trainer.save_metrics("eval", metrics)
 
     if training_args.do_predict:
-        logger.info("*** Test ***")
+        logger.info("*** Predict ***")
 
         # Loop to handle MNLI double evaluation (matched, mis-matched)
         tasks = [data_args.task_name]
-        test_datasets = [test_dataset]
+        predict_datasets = [predict_dataset]
         if data_args.task_name == "mnli":
             tasks.append("mnli-mm")
-            test_datasets.append(datasets["test_mismatched"])
+            predict_datasets.append(datasets["test_mismatched"])
 
-        for test_dataset, task in zip(test_datasets, tasks):
+        for predict_dataset, task in zip(predict_datasets, tasks):
             # Removing the `label` columns because it contains -1 and Trainer won't like that.
-            test_dataset.remove_columns_("label")
-            predictions = trainer.predict(test_dataset=test_dataset).predictions
+            predict_dataset.remove_columns_("label")
+            predictions = trainer.predict(test_dataset=predict_dataset).predictions
             predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
 
-            output_test_file = os.path.join(training_args.output_dir, f"test_results_{task}.txt")
+            output_predict_file = os.path.join(training_args.output_dir, f"predict_results_{task}.txt")
             if trainer.is_world_process_zero():
-                with open(output_test_file, "w") as writer:
-                    logger.info(f"***** Test results {task} *****")
+                with open(output_predict_file, "w") as writer:
+                    logger.info(f"***** Predict results {task} *****")
                     writer.write("index\tprediction\n")
                     for index, item in enumerate(predictions):
                         if is_regression:
diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py
index 1c74e6e7bc0f..142d921f78c1 100755
--- a/examples/pytorch/text-classification/run_xnli.py
+++ b/examples/pytorch/text-classification/run_xnli.py
@@ -84,17 +84,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -221,8 +221,8 @@ def main():
         label_list = eval_dataset.features["label"].names
 
     if training_args.do_predict:
-        test_dataset = load_dataset("xnli", model_args.language, split="test", cache_dir=model_args.cache_dir)
-        label_list = test_dataset.features["label"].names
+        predict_dataset = load_dataset("xnli", model_args.language, split="test", cache_dir=model_args.cache_dir)
+        label_list = predict_dataset.features["label"].names
 
     # Labels
     num_labels = len(label_list)
@@ -286,8 +286,8 @@ def preprocess_function(examples):
             logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")
 
     if training_args.do_eval:
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         eval_dataset = eval_dataset.map(
             preprocess_function,
             batched=True,
@@ -295,9 +295,9 @@ def preprocess_function(examples):
         )
 
     if training_args.do_predict:
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
-        test_dataset = test_dataset.map(
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
+        predict_dataset = predict_dataset.map(
             preprocess_function,
             batched=True,
             load_from_cache_file=not data_args.overwrite_cache,
@@ -360,8 +360,8 @@ def compute_metrics(p: EvalPrediction):
         logger.info("*** Evaluate ***")
         metrics = trainer.evaluate(eval_dataset=eval_dataset)
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
@@ -369,18 +369,20 @@ def compute_metrics(p: EvalPrediction):
     # Prediction
     if training_args.do_predict:
         logger.info("*** Predict ***")
-        predictions, labels, metrics = trainer.predict(test_dataset)
+        predictions, labels, metrics = trainer.predict(predict_dataset)
 
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        max_predict_samples = (
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
+        )
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
         predictions = np.argmax(predictions, axis=1)
-        output_test_file = os.path.join(training_args.output_dir, "test_predictions.txt")
+        output_predict_file = os.path.join(training_args.output_dir, "predictions.txt")
         if trainer.is_world_process_zero():
-            with open(output_test_file, "w") as writer:
+            with open(output_predict_file, "w") as writer:
                 writer.write("index\tprediction\n")
                 for index, item in enumerate(predictions):
                     item = label_list[item]
diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py
index 2f31b1f64da5..3e3825d419ce 100755
--- a/examples/pytorch/token-classification/run_ner.py
+++ b/examples/pytorch/token-classification/run_ner.py
@@ -128,17 +128,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -363,8 +363,8 @@ def tokenize_and_align_labels(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         eval_dataset = eval_dataset.map(
             tokenize_and_align_labels,
             batched=True,
@@ -375,10 +375,10 @@ def tokenize_and_align_labels(examples):
     if training_args.do_predict:
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_dataset = datasets["test"]
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
-        test_dataset = test_dataset.map(
+        predict_dataset = datasets["test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
+        predict_dataset = predict_dataset.map(
             tokenize_and_align_labels,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
@@ -462,8 +462,8 @@ def compute_metrics(p):
 
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
@@ -472,7 +472,7 @@ def compute_metrics(p):
     if training_args.do_predict:
         logger.info("*** Predict ***")
 
-        predictions, labels, metrics = trainer.predict(test_dataset)
+        predictions, labels, metrics = trainer.predict(predict_dataset)
         predictions = np.argmax(predictions, axis=2)
 
         # Remove ignored index (special tokens)
@@ -481,13 +481,13 @@ def compute_metrics(p):
             for prediction, label in zip(predictions, labels)
         ]
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
         # Save predictions
-        output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
+        output_predictions_file = os.path.join(training_args.output_dir, "predictions.txt")
         if trainer.is_world_process_zero():
-            with open(output_test_predictions_file, "w") as writer:
+            with open(output_predictions_file, "w") as writer:
                 for prediction in true_predictions:
                     writer.write(" ".join(prediction) + "\n")
 
diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py
index e42dce9cf626..5a6bad474379 100755
--- a/examples/pytorch/translation/run_translation.py
+++ b/examples/pytorch/translation/run_translation.py
@@ -167,17 +167,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -432,8 +432,8 @@ def preprocess_function(examples):
         if "validation" not in datasets:
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         eval_dataset = eval_dataset.map(
             preprocess_function,
             batched=True,
@@ -446,10 +446,10 @@ def preprocess_function(examples):
         max_target_length = data_args.val_max_target_length
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_dataset = datasets["test"]
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
-        test_dataset = test_dataset.map(
+        predict_dataset = datasets["test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
+        predict_dataset = predict_dataset.map(
             preprocess_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
@@ -539,37 +539,39 @@ def compute_metrics(eval_preds):
         metrics = trainer.evaluate(
             max_length=data_args.val_max_target_length, num_beams=data_args.num_beams, metric_key_prefix="eval"
         )
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
 
     if training_args.do_predict:
-        logger.info("*** Test ***")
+        logger.info("*** Predict ***")
 
-        test_results = trainer.predict(
-            test_dataset,
+        predict_results = trainer.predict(
+            predict_dataset,
             metric_key_prefix="test",
             max_length=data_args.val_max_target_length,
             num_beams=data_args.num_beams,
         )
-        metrics = test_results.metrics
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        metrics = predict_results.metrics
+        max_predict_samples = (
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
+        )
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
         if trainer.is_world_process_zero():
             if training_args.predict_with_generate:
-                test_preds = tokenizer.batch_decode(
-                    test_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
+                predictions = tokenizer.batch_decode(
+                    predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
                 )
-                test_preds = [pred.strip() for pred in test_preds]
-                output_test_preds_file = os.path.join(training_args.output_dir, "test_generations.txt")
-                with open(output_test_preds_file, "w") as writer:
-                    writer.write("\n".join(test_preds))
+                predictions = [pred.strip() for pred in predictions]
+                output_prediction_file = os.path.join(training_args.output_dir, "predictions.txt")
+                with open(output_prediction_file, "w") as writer:
+                    writer.write("\n".join(predictions))
 
     return results
 
diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index dc39cae0fe48..ade89772e831 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -676,43 +676,43 @@ def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoa
             pin_memory=self.args.dataloader_pin_memory,
         )
 
-    def get_test_dataloader(self, test_dataset: Dataset) -> DataLoader:
+    def get_predict_dataloader(self, predict_dataset: Dataset) -> DataLoader:
         """
-        Returns the test :class:`~torch.utils.data.DataLoader`.
+        Returns the predict :class:`~torch.utils.data.DataLoader`.
 
         Subclass and override this method if you want to inject some custom behavior.
 
         Args:
-            test_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`):
-                The test dataset to use. If it is an :obj:`datasets.Dataset`, columns not accepted by the
+            predict_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`):
+                The predict dataset to use. If it is an :obj:`datasets.Dataset`, columns not accepted by the
                 ``model.forward()`` method are automatically removed. It must implement :obj:`__len__`.
         """
-        if is_datasets_available() and isinstance(test_dataset, datasets.Dataset):
-            test_dataset = self._remove_unused_columns(test_dataset, description="test")
+        if is_datasets_available() and isinstance(predict_dataset, datasets.Dataset):
+            predict_dataset = self._remove_unused_columns(predict_dataset, description="prediction")
 
-        if isinstance(test_dataset, torch.utils.data.dataset.IterableDataset):
+        if isinstance(predict_dataset, torch.utils.data.dataset.IterableDataset):
             if self.args.world_size > 1:
-                test_dataset = IterableDatasetShard(
-                    test_dataset,
+                predict_dataset = IterableDatasetShard(
+                    predict_dataset,
                     batch_size=self.args.eval_batch_size,
                     drop_last=self.args.dataloader_drop_last,
                     num_processes=self.args.world_size,
                     process_index=self.args.process_index,
                 )
             return DataLoader(
-                test_dataset,
+                predict_dataset,
                 batch_size=self.args.eval_batch_size,
                 collate_fn=self.data_collator,
                 num_workers=self.args.dataloader_num_workers,
                 pin_memory=self.args.dataloader_pin_memory,
             )
 
-        test_sampler = self._get_eval_sampler(test_dataset)
+        predict_sampler = self._get_eval_sampler(predict_dataset)
 
         # We use the same batch_size as for eval.
         return DataLoader(
-            test_dataset,
-            sampler=test_sampler,
+            predict_dataset,
+            sampler=predict_sampler,
             batch_size=self.args.eval_batch_size,
             collate_fn=self.data_collator,
             drop_last=self.args.dataloader_drop_last,
@@ -1896,24 +1896,24 @@ def evaluate(
         return output.metrics
 
     def predict(
-        self, test_dataset: Dataset, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "test"
+        self, predict_dataset: Dataset, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "predict"
     ) -> PredictionOutput:
         """
         Run prediction and returns predictions and potential metrics.
 
-        Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method
+        Depending on the dataset and your use case, your predict dataset may contain labels. In that case, this method
         will also return metrics, like in :obj:`evaluate()`.
 
         Args:
-            test_dataset (:obj:`Dataset`):
+            predict_dataset (:obj:`Dataset`):
                 Dataset to run the predictions on. If it is an :obj:`datasets.Dataset`, columns not accepted by the
                 ``model.forward()`` method are automatically removed. Has to implement the method :obj:`__len__`
             ignore_keys (:obj:`Lst[str]`, `optional`):
                 A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                 gathering predictions.
-            metric_key_prefix (:obj:`str`, `optional`, defaults to :obj:`"test"`):
+            metric_key_prefix (:obj:`str`, `optional`, defaults to :obj:`"predict"`):
                 An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named
-                "test_bleu" if the prefix is "test" (default)
+                "predict_bleu" if the prefix is "predict" (default)
 
         .. note::
 
@@ -1923,7 +1923,7 @@ def predict(
 
         Returns: `NamedTuple` A namedtuple with the following keys:
 
-            - predictions (:obj:`np.ndarray`): The predictions on :obj:`test_dataset`.
+            - predictions (:obj:`np.ndarray`): The predictions on :obj:`predict_dataset`.
             - label_ids (:obj:`np.ndarray`, `optional`): The labels (if the dataset contained some).
             - metrics (:obj:`Dict[str, float]`, `optional`): The potential dictionary of metrics (if the dataset
               contained labels).
@@ -1931,12 +1931,12 @@ def predict(
         # memory metrics - must set up as early as possible
         self._memory_tracker.start()
 
-        test_dataloader = self.get_test_dataloader(test_dataset)
+        predict_dataloader = self.get_predict_dataloader(predict_dataset)
         start_time = time.time()
 
         eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         output = eval_loop(
-            test_dataloader, description="Prediction", ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix
+            predict_dataloader, description="Prediction", ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix
         )
         output.metrics.update(speed_metrics(metric_key_prefix, start_time, output.num_samples))
 
diff --git a/src/transformers/trainer_pt_utils.py b/src/transformers/trainer_pt_utils.py
index 0b58904c00fd..1c9096ae8c95 100644
--- a/src/transformers/trainer_pt_utils.py
+++ b/src/transformers/trainer_pt_utils.py
@@ -822,7 +822,7 @@ def log_metrics(self, split, metrics):
 
     Args:
         split (:obj:`str`):
-            Mode/split name: one of ``train``, ``eval``, ``test``
+            Mode/split name: one of ``train``, ``eval``, ``predict``
         metrics (:obj:`Dict[str, float]`):
             The metrics returned from train/evaluate/predictmetrics: metrics dict
 
@@ -911,7 +911,7 @@ def save_metrics(self, split, metrics, combined=True):
 
     Args:
         split (:obj:`str`):
-            Mode/split name: one of ``train``, ``eval``, ``test``, ``all``
+            Mode/split name: one of ``train``, ``eval``, ``predict``, ``all``
         metrics (:obj:`Dict[str, float]`):
             The metrics returned from train/evaluate/predict
         combined (:obj:`bool`, `optional`, defaults to :obj:`True`):
diff --git a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
index d40328b925cf..48590fe16712 100755
--- a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
+++ b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
@@ -157,17 +157,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
             "value if set."
         },
     )
@@ -379,8 +379,8 @@ def tokenize_function(examples):
             raise ValueError("--do_eval requires a validation dataset")
         eval_dataset = datasets["validation"]
         # Selecting samples from dataset
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
         # tokenize validation dataset
         eval_dataset = eval_dataset.map(
             tokenize_function,
@@ -393,12 +393,12 @@ def tokenize_function(examples):
     if training_args.do_predict:
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_dataset = datasets["test"]
+        predict_dataset = datasets["test"]
         # Selecting samples from dataset
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
-        # tokenize test dataset
-        test_dataset = test_dataset.map(
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
+        # tokenize predict dataset
+        predict_dataset = predict_dataset.map(
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
@@ -455,8 +455,8 @@ def tokenize_function(examples):
 
         metrics = trainer.evaluate()
 
-        max_val_samples = data_args.max_val_samples if data_args.max_val_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_val_samples, len(eval_dataset))
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
 
         trainer.log_metrics("eval", metrics)
         trainer.save_metrics("eval", metrics)
@@ -464,13 +464,13 @@ def tokenize_function(examples):
     # Prediction
     if training_args.do_predict:
         logger.info("*** Predict ***")
-        predictions, labels, metrics = trainer.predict(test_dataset)
+        predictions, labels, metrics = trainer.predict(predict_dataset)
 
-        max_test_samples = data_args.max_test_samples if data_args.max_test_samples is not None else len(test_dataset)
-        metrics["test_samples"] = min(max_test_samples, len(test_dataset))
+        max_predict_samples = data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
         
         # write custom code for saving predictions according to task
 

From 6665ca401bb6d46822690509f7d576cf46111ebf Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Wed, 21 Apr 2021 23:52:30 +0530
Subject: [PATCH 02/12] modified files

---
 examples/pytorch/question-answering/trainer_qa.py  | 14 +++++++-------
 .../pytorch/summarization/run_summarization.py     | 12 ++++++------
 examples/pytorch/translation/run_translation.py    |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/examples/pytorch/question-answering/trainer_qa.py b/examples/pytorch/question-answering/trainer_qa.py
index 41699cd1dfae..565724a8c656 100644
--- a/examples/pytorch/question-answering/trainer_qa.py
+++ b/examples/pytorch/question-answering/trainer_qa.py
@@ -66,16 +66,16 @@ def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None):
         self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
         return metrics
 
-    def predict(self, test_dataset, test_examples, ignore_keys=None):
-        test_dataloader = self.get_test_dataloader(test_dataset)
+    def predict(self, predict_dataset, predict_examples, ignore_keys=None):
+        predict_dataloader = self.get_predict_dataloader(predict_dataset)
 
         # Temporarily disable metric computation, we will do it in the loop here.
         compute_metrics = self.compute_metrics
         self.compute_metrics = None
         try:
             output = self.prediction_loop(
-                test_dataloader,
-                description="Evaluation",
+                predict_dataloader,
+                description="Prediction",
                 # No point gathering the predictions if there are no metrics, otherwise we defer to
                 # self.args.prediction_loss_only
                 prediction_loss_only=True if compute_metrics is None else None,
@@ -87,7 +87,7 @@ def predict(self, test_dataset, test_examples, ignore_keys=None):
         if self.post_process_function is None or self.compute_metrics is None:
             return output
 
-        eval_preds = self.post_process_function(test_examples, test_dataset, output.predictions, "test")
-        metrics = self.compute_metrics(eval_preds)
+        predictions = self.post_process_function(test_examples, test_dataset, output.predictions, "predict")
+        metrics = self.compute_metrics(predictions)
 
-        return PredictionOutput(predictions=eval_preds.predictions, label_ids=eval_preds.label_ids, metrics=metrics)
+        return PredictionOutput(predictions=predictions.predictions, label_ids=eval_preds.label_ids, metrics=metrics)
diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py
index aa457576101e..be454ecc155e 100755
--- a/examples/pytorch/summarization/run_summarization.py
+++ b/examples/pytorch/summarization/run_summarization.py
@@ -558,7 +558,7 @@ def compute_metrics(eval_preds):
 
         predict_results = trainer.predict(
             predict_dataset,
-            metric_key_prefix="test",
+            metric_key_prefix="predict",
             max_length=data_args.val_max_target_length,
             num_beams=data_args.num_beams,
         )
@@ -573,12 +573,12 @@ def compute_metrics(eval_preds):
 
         if trainer.is_world_process_zero():
             if training_args.predict_with_generate:
-                test_preds = tokenizer.batch_decode(
-                    test_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
+                predictions = tokenizer.batch_decode(
+                    predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
                 )
-                test_preds = [pred.strip() for pred in test_preds]
-                output_test_preds_file = os.path.join(training_args.output_dir, "predict_generations.txt")
-                with open(output_test_preds_file, "w") as writer:
+                predictions = [pred.strip() for pred in predictions]
+                output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt")
+                with open(output_prediction_file, "w") as writer:
                     writer.write("\n".join(test_preds))
 
     return results
diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py
index 5a6bad474379..4d6ab868b480 100755
--- a/examples/pytorch/translation/run_translation.py
+++ b/examples/pytorch/translation/run_translation.py
@@ -569,7 +569,7 @@ def compute_metrics(eval_preds):
                     predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True
                 )
                 predictions = [pred.strip() for pred in predictions]
-                output_prediction_file = os.path.join(training_args.output_dir, "predictions.txt")
+                output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt")
                 with open(output_prediction_file, "w") as writer:
                     writer.write("\n".join(predictions))
 

From c1ec9b5cd40eef3a8bc6b34acae7ef4d1df01f4f Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Wed, 21 Apr 2021 23:56:24 +0530
Subject: [PATCH 03/12] corrected typo

---
 examples/pytorch/text-classification/run_glue.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py
index d1dc7fa89d30..2777b49d7a06 100755
--- a/examples/pytorch/text-classification/run_glue.py
+++ b/examples/pytorch/text-classification/run_glue.py
@@ -504,7 +504,7 @@ def compute_metrics(p: EvalPrediction):
         for predict_dataset, task in zip(predict_datasets, tasks):
             # Removing the `label` columns because it contains -1 and Trainer won't like that.
             predict_dataset.remove_columns_("label")
-            predictions = trainer.predict(test_dataset=predict_dataset).predictions
+            predictions = trainer.predict(predict_dataset=predict_dataset).predictions
             predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
 
             output_predict_file = os.path.join(training_args.output_dir, f"predict_results_{task}.txt")

From f878629094f0b8805a7ce3d9138f6498802f20c7 Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Thu, 22 Apr 2021 10:18:11 +0530
Subject: [PATCH 04/12] fixed qa scripts

---
 .../question-answering/run_qa_beam_search.py   | 18 +++++++++---------
 .../pytorch/question-answering/trainer_qa.py   |  4 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/pytorch/question-answering/run_qa_beam_search.py b/examples/pytorch/question-answering/run_qa_beam_search.py
index 0d1d14845cd1..2c322a95b923 100755
--- a/examples/pytorch/question-answering/run_qa_beam_search.py
+++ b/examples/pytorch/question-answering/run_qa_beam_search.py
@@ -522,12 +522,12 @@ def prepare_validation_features(examples):
     if training_args.do_predict:
         if "test" not in datasets:
             raise ValueError("--do_predict requires a test dataset")
-        test_examples = datasets["test"]
+        predict_examples = datasets["test"]
         if data_args.max_predict_samples is not None:
             # We will select sample from whole data
-            test_examples = test_examples.select(range(data_args.max_predict_samples))
+            predict_examples = predict_examples.select(range(data_args.max_predict_samples))
         # Test Feature Creation
-        test_dataset = test_examples.map(
+        predict_dataset = predict_examples.map(
             prepare_validation_features,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
@@ -536,7 +536,7 @@ def prepare_validation_features(examples):
         )
         if data_args.max_predict_samples is not None:
             # During Feature creation dataset samples might increase, we will select required samples again
-            test_dataset = test_dataset.select(range(data_args.max_predict_samples))
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
 
     # Data collator
     # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data
@@ -629,16 +629,16 @@ def compute_metrics(p: EvalPrediction):
     # Prediction
     if training_args.do_predict:
         logger.info("*** Predict ***")
-        results = trainer.predict(test_dataset, test_examples)
+        results = trainer.predict(predict_dataset, predict_examples)
         metrics = results.metrics
 
         max_predict_samples = (
-            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(test_dataset)
+            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)
         )
-        metrics["test_samples"] = min(max_predict_samples, len(test_dataset))
+        metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset))
 
-        trainer.log_metrics("test", metrics)
-        trainer.save_metrics("test", metrics)
+        trainer.log_metrics("predict", metrics)
+        trainer.save_metrics("predict", metrics)
 
 
 def _mp_fn(index):
diff --git a/examples/pytorch/question-answering/trainer_qa.py b/examples/pytorch/question-answering/trainer_qa.py
index 565724a8c656..95ec9f94fe3b 100644
--- a/examples/pytorch/question-answering/trainer_qa.py
+++ b/examples/pytorch/question-answering/trainer_qa.py
@@ -87,7 +87,7 @@ def predict(self, predict_dataset, predict_examples, ignore_keys=None):
         if self.post_process_function is None or self.compute_metrics is None:
             return output
 
-        predictions = self.post_process_function(test_examples, test_dataset, output.predictions, "predict")
+        predictions = self.post_process_function(predict_examples, predict_dataset, output.predictions, "predict")
         metrics = self.compute_metrics(predictions)
 
-        return PredictionOutput(predictions=predictions.predictions, label_ids=eval_preds.label_ids, metrics=metrics)
+        return PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)

From 3f6273e5f28b131c1931d0883462657b14d8e6ef Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Thu, 22 Apr 2021 10:34:33 +0530
Subject: [PATCH 05/12] fix typos

---
 examples/pytorch/summarization/run_summarization.py | 2 +-
 examples/pytorch/translation/run_translation.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py
index be454ecc155e..0acdc3f68f44 100755
--- a/examples/pytorch/summarization/run_summarization.py
+++ b/examples/pytorch/summarization/run_summarization.py
@@ -579,7 +579,7 @@ def compute_metrics(eval_preds):
                 predictions = [pred.strip() for pred in predictions]
                 output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt")
                 with open(output_prediction_file, "w") as writer:
-                    writer.write("\n".join(test_preds))
+                    writer.write("\n".join(predictions))
 
     return results
 
diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py
index 4d6ab868b480..3f1d5cff1909 100755
--- a/examples/pytorch/translation/run_translation.py
+++ b/examples/pytorch/translation/run_translation.py
@@ -550,7 +550,7 @@ def compute_metrics(eval_preds):
 
         predict_results = trainer.predict(
             predict_dataset,
-            metric_key_prefix="test",
+            metric_key_prefix="predict",
             max_length=data_args.val_max_target_length,
             num_beams=data_args.num_beams,
         )

From 06124d5b496fb711714a3f78afae018f1cd9a0de Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Thu, 22 Apr 2021 18:02:26 +0530
Subject: [PATCH 06/12] fixed predict typo in qa no trainer

---
 examples/pytorch/question-answering/run_qa_no_trainer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py
index 6bdda4816a84..97e2c8b431d0 100755
--- a/examples/pytorch/question-answering/run_qa_no_trainer.py
+++ b/examples/pytorch/question-answering/run_qa_no_trainer.py
@@ -510,7 +510,7 @@ def prepare_validation_features(examples):
             # We will select sample from whole data
             predict_examples = predict_examples.select(range(args.max_predict_samples))
         # Predict Feature Creation
-        predict_examples = predict_examples.map(
+        predict_dataset = predict_examples.map(
             prepare_validation_features,
             batched=True,
             num_proc=args.preprocessing_num_workers,
@@ -739,7 +739,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len):
         # Now we need to add extra columns which we removed for post processing
         predict_dataset.set_format(type=None, columns=list(predict_dataset.features.keys()))
         outputs_numpy = (start_logits_concat, end_logits_concat)
-        prediction = post_processing_function(test_examples, predict_dataset, outputs_numpy)
+        prediction = post_processing_function(predict_examples, predict_dataset, outputs_numpy)
         predict_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)
         logger.info(f"Predict metrics: {predict_metric}")
 

From 11e30e53e62e177bb9a97c647782120e9ff2081f Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Thu, 22 Apr 2021 18:19:57 +0530
Subject: [PATCH 07/12] fixed test file

---
 tests/extended/test_trainer_ext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/extended/test_trainer_ext.py b/tests/extended/test_trainer_ext.py
index 563ea8f059bf..bae358740034 100644
--- a/tests/extended/test_trainer_ext.py
+++ b/tests/extended/test_trainer_ext.py
@@ -191,7 +191,7 @@ def run_trainer(
             --output_dir {output_dir}
             --overwrite_output_dir
             --max_train_samples 8
-            --max_val_samples 8
+            --max_eval_samples 8
             --max_source_length {max_len}
             --max_target_length {max_len}
             --val_max_target_length {max_len}

From 9fb473b82f6c1b22e7dc42090fffb656af06dc01 Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Thu, 22 Apr 2021 23:35:43 +0530
Subject: [PATCH 08/12] reverted trainer changes

---
 .../pytorch/text-classification/run_glue.py   |  2 +-
 .../pytorch/token-classification/run_ner.py   |  2 +-
 src/transformers/trainer.py                   | 42 +++++++++----------
 src/transformers/trainer_pt_utils.py          |  4 +-
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py
index 2777b49d7a06..e2c4394c1f4b 100755
--- a/examples/pytorch/text-classification/run_glue.py
+++ b/examples/pytorch/text-classification/run_glue.py
@@ -504,7 +504,7 @@ def compute_metrics(p: EvalPrediction):
         for predict_dataset, task in zip(predict_datasets, tasks):
             # Removing the `label` columns because it contains -1 and Trainer won't like that.
             predict_dataset.remove_columns_("label")
-            predictions = trainer.predict(predict_dataset=predict_dataset).predictions
+            predictions = trainer.predict(predict_dataset, metric_key_prefix="predict").predictions
             predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
 
             output_predict_file = os.path.join(training_args.output_dir, f"predict_results_{task}.txt")
diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py
index 3e3825d419ce..91a3ddc92afe 100755
--- a/examples/pytorch/token-classification/run_ner.py
+++ b/examples/pytorch/token-classification/run_ner.py
@@ -472,7 +472,7 @@ def compute_metrics(p):
     if training_args.do_predict:
         logger.info("*** Predict ***")
 
-        predictions, labels, metrics = trainer.predict(predict_dataset)
+        predictions, labels, metrics = trainer.predict(predict_dataset, metric_key_prefix="predict")
         predictions = np.argmax(predictions, axis=2)
 
         # Remove ignored index (special tokens)
diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index ade89772e831..dc39cae0fe48 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -676,43 +676,43 @@ def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoa
             pin_memory=self.args.dataloader_pin_memory,
         )
 
-    def get_predict_dataloader(self, predict_dataset: Dataset) -> DataLoader:
+    def get_test_dataloader(self, test_dataset: Dataset) -> DataLoader:
         """
-        Returns the predict :class:`~torch.utils.data.DataLoader`.
+        Returns the test :class:`~torch.utils.data.DataLoader`.
 
         Subclass and override this method if you want to inject some custom behavior.
 
         Args:
-            predict_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`):
-                The predict dataset to use. If it is an :obj:`datasets.Dataset`, columns not accepted by the
+            test_dataset (:obj:`torch.utils.data.dataset.Dataset`, `optional`):
+                The test dataset to use. If it is an :obj:`datasets.Dataset`, columns not accepted by the
                 ``model.forward()`` method are automatically removed. It must implement :obj:`__len__`.
         """
-        if is_datasets_available() and isinstance(predict_dataset, datasets.Dataset):
-            predict_dataset = self._remove_unused_columns(predict_dataset, description="prediction")
+        if is_datasets_available() and isinstance(test_dataset, datasets.Dataset):
+            test_dataset = self._remove_unused_columns(test_dataset, description="test")
 
-        if isinstance(predict_dataset, torch.utils.data.dataset.IterableDataset):
+        if isinstance(test_dataset, torch.utils.data.dataset.IterableDataset):
             if self.args.world_size > 1:
-                predict_dataset = IterableDatasetShard(
-                    predict_dataset,
+                test_dataset = IterableDatasetShard(
+                    test_dataset,
                     batch_size=self.args.eval_batch_size,
                     drop_last=self.args.dataloader_drop_last,
                     num_processes=self.args.world_size,
                     process_index=self.args.process_index,
                 )
             return DataLoader(
-                predict_dataset,
+                test_dataset,
                 batch_size=self.args.eval_batch_size,
                 collate_fn=self.data_collator,
                 num_workers=self.args.dataloader_num_workers,
                 pin_memory=self.args.dataloader_pin_memory,
             )
 
-        predict_sampler = self._get_eval_sampler(predict_dataset)
+        test_sampler = self._get_eval_sampler(test_dataset)
 
         # We use the same batch_size as for eval.
         return DataLoader(
-            predict_dataset,
-            sampler=predict_sampler,
+            test_dataset,
+            sampler=test_sampler,
             batch_size=self.args.eval_batch_size,
             collate_fn=self.data_collator,
             drop_last=self.args.dataloader_drop_last,
@@ -1896,24 +1896,24 @@ def evaluate(
         return output.metrics
 
     def predict(
-        self, predict_dataset: Dataset, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "predict"
+        self, test_dataset: Dataset, ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "test"
     ) -> PredictionOutput:
         """
         Run prediction and returns predictions and potential metrics.
 
-        Depending on the dataset and your use case, your predict dataset may contain labels. In that case, this method
+        Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method
         will also return metrics, like in :obj:`evaluate()`.
 
         Args:
-            predict_dataset (:obj:`Dataset`):
+            test_dataset (:obj:`Dataset`):
                 Dataset to run the predictions on. If it is an :obj:`datasets.Dataset`, columns not accepted by the
                 ``model.forward()`` method are automatically removed. Has to implement the method :obj:`__len__`
             ignore_keys (:obj:`Lst[str]`, `optional`):
                 A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                 gathering predictions.
-            metric_key_prefix (:obj:`str`, `optional`, defaults to :obj:`"predict"`):
+            metric_key_prefix (:obj:`str`, `optional`, defaults to :obj:`"test"`):
                 An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named
-                "predict_bleu" if the prefix is "predict" (default)
+                "test_bleu" if the prefix is "test" (default)
 
         .. note::
 
@@ -1923,7 +1923,7 @@ def predict(
 
         Returns: `NamedTuple` A namedtuple with the following keys:
 
-            - predictions (:obj:`np.ndarray`): The predictions on :obj:`predict_dataset`.
+            - predictions (:obj:`np.ndarray`): The predictions on :obj:`test_dataset`.
             - label_ids (:obj:`np.ndarray`, `optional`): The labels (if the dataset contained some).
             - metrics (:obj:`Dict[str, float]`, `optional`): The potential dictionary of metrics (if the dataset
               contained labels).
@@ -1931,12 +1931,12 @@ def predict(
         # memory metrics - must set up as early as possible
         self._memory_tracker.start()
 
-        predict_dataloader = self.get_predict_dataloader(predict_dataset)
+        test_dataloader = self.get_test_dataloader(test_dataset)
         start_time = time.time()
 
         eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         output = eval_loop(
-            predict_dataloader, description="Prediction", ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix
+            test_dataloader, description="Prediction", ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix
         )
         output.metrics.update(speed_metrics(metric_key_prefix, start_time, output.num_samples))
 
diff --git a/src/transformers/trainer_pt_utils.py b/src/transformers/trainer_pt_utils.py
index 1c9096ae8c95..0b58904c00fd 100644
--- a/src/transformers/trainer_pt_utils.py
+++ b/src/transformers/trainer_pt_utils.py
@@ -822,7 +822,7 @@ def log_metrics(self, split, metrics):
 
     Args:
         split (:obj:`str`):
-            Mode/split name: one of ``train``, ``eval``, ``predict``
+            Mode/split name: one of ``train``, ``eval``, ``test``
         metrics (:obj:`Dict[str, float]`):
             The metrics returned from train/evaluate/predictmetrics: metrics dict
 
@@ -911,7 +911,7 @@ def save_metrics(self, split, metrics, combined=True):
 
     Args:
         split (:obj:`str`):
-            Mode/split name: one of ``train``, ``eval``, ``predict``, ``all``
+            Mode/split name: one of ``train``, ``eval``, ``test``, ``all``
         metrics (:obj:`Dict[str, float]`):
             The metrics returned from train/evaluate/predict
         combined (:obj:`bool`, `optional`, defaults to :obj:`True`):

From bdaf493e595bdc9181fa1c5a913a1f7a4d291d49 Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Fri, 23 Apr 2021 00:06:35 +0530
Subject: [PATCH 09/12] reverted trainer changes in custom exmaples

---
 examples/pytorch/question-answering/trainer_qa.py | 2 +-
 examples/pytorch/text-classification/run_xnli.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/pytorch/question-answering/trainer_qa.py b/examples/pytorch/question-answering/trainer_qa.py
index 95ec9f94fe3b..36e2e544a7ac 100644
--- a/examples/pytorch/question-answering/trainer_qa.py
+++ b/examples/pytorch/question-answering/trainer_qa.py
@@ -67,7 +67,7 @@ def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None):
         return metrics
 
     def predict(self, predict_dataset, predict_examples, ignore_keys=None):
-        predict_dataloader = self.get_predict_dataloader(predict_dataset)
+        predict_dataloader = self.get_test_dataloader(predict_dataset)
 
         # Temporarily disable metric computation, we will do it in the loop here.
         compute_metrics = self.compute_metrics
diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py
index 142d921f78c1..c1d8522c8d04 100755
--- a/examples/pytorch/text-classification/run_xnli.py
+++ b/examples/pytorch/text-classification/run_xnli.py
@@ -369,7 +369,7 @@ def compute_metrics(p: EvalPrediction):
     # Prediction
     if training_args.do_predict:
         logger.info("*** Predict ***")
-        predictions, labels, metrics = trainer.predict(predict_dataset)
+        predictions, labels, metrics = trainer.predict(predict_dataset, metric_key_prefix="predict")
 
         max_predict_samples = (
             data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)

From 0792746ec985bcc9efe2aee98ddb92583188856e Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Fri, 23 Apr 2021 10:00:55 +0530
Subject: [PATCH 10/12] updated readme

---
 examples/pytorch/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/pytorch/README.md b/examples/pytorch/README.md
index c01ba6749db6..389ca48563b5 100644
--- a/examples/pytorch/README.md
+++ b/examples/pytorch/README.md
@@ -50,8 +50,8 @@ For example here is how to truncate all three splits to just 50 samples each:
 ```
 examples/pytorch/token-classification/run_ner.py \
 --max_train_samples 50 \
---max_val_samples 50 \
---max_test_samples 50 \
+--max_eval_samples 50 \
+--max_predict_samples 50 \
 [...]
 ```
 

From 1a90c3dcc71f131f26ef51b6faa9447f059bd193 Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Sat, 24 Apr 2021 10:54:50 +0530
Subject: [PATCH 11/12] added changes in deepspeed test

---
 tests/deepspeed/test_deepspeed.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/deepspeed/test_deepspeed.py b/tests/deepspeed/test_deepspeed.py
index 9868966a5a32..26a026cfb707 100644
--- a/tests/deepspeed/test_deepspeed.py
+++ b/tests/deepspeed/test_deepspeed.py
@@ -578,7 +578,7 @@ def run_trainer(
             args.extend(
                 """
             --do_eval
-            --max_val_samples 100
+            --max_eval_samples 100
             --per_device_eval_batch_size 2
             """.split()
             )
@@ -620,7 +620,7 @@ def test_clm(self, stage):
             --do_train
             --do_eval
             --max_train_samples 10
-            --max_val_samples 10
+            --max_eval_samples 10
             --per_device_train_batch_size 5
             --per_device_eval_batch_size 5
             --num_train_epochs 1

From 4702d63bcbebf805c9da1a8ec897dbe6ac3f788c Mon Sep 17 00:00:00 2001
From: bhadreshpsavani <bhadreshpsavani@gmail.com>
Date: Mon, 26 Apr 2021 15:18:51 +0530
Subject: [PATCH 12/12] added changes for predict and eval

---
 .../run_text_classification.py                | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/examples/tensorflow/text-classification/run_text_classification.py b/examples/tensorflow/text-classification/run_text_classification.py
index 3c9e2600970d..f725f1c930f0 100644
--- a/examples/tensorflow/text-classification/run_text_classification.py
+++ b/examples/tensorflow/text-classification/run_text_classification.py
@@ -164,17 +164,17 @@ class DataTrainingArguments:
             "value if set."
         },
     )
-    max_val_samples: Optional[int] = field(
+    max_eval_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
             "value if set."
         },
     )
-    max_test_samples: Optional[int] = field(
+    max_predict_samples: Optional[int] = field(
         default=None,
         metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of test examples to this "
+            "help": "For debugging purposes or quicker training, truncate the number of predict examples to this "
             "value if set."
         },
     )
@@ -468,13 +468,13 @@ def preprocess_function(examples):
 
     if "validation" in datasets:
         eval_dataset = datasets["validation"]
-        if data_args.max_val_samples is not None:
-            eval_dataset = eval_dataset.select(range(data_args.max_val_samples))
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
 
     if "test" in datasets:
-        test_dataset = datasets["test"]
-        if data_args.max_test_samples is not None:
-            test_dataset = test_dataset.select(range(data_args.max_test_samples))
+        predict_dataset = datasets["test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
 
     # endregion
 
@@ -513,15 +513,15 @@ def preprocess_function(examples):
 
     # region Prediction
     if "test" in datasets:
-        logger.info("Doing predictions on test dataset...")
+        logger.info("Doing predictions on Predict dataset...")
 
-        test_dataset = DataSequence(
-            test_dataset, non_label_column_names, batch_size=training_args.per_device_eval_batch_size, labels=False
+        predict_dataset = DataSequence(
+            predict_dataset, non_label_column_names, batch_size=training_args.per_device_eval_batch_size, labels=False
         )
-        predictions = model.predict(test_dataset)["logits"]
+        predictions = model.predict(predict_dataset)["logits"]
         predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
-        output_test_file = os.path.join(training_args.output_dir, "test_results.txt")
-        with open(output_test_file, "w") as writer:
+        output_predict_file = os.path.join(training_args.output_dir, "predict_results.txt")
+        with open(output_predict_file, "w") as writer:
             writer.write("index\tprediction\n")
             for index, item in enumerate(predictions):
                 if is_regression:
@@ -529,7 +529,7 @@ def preprocess_function(examples):
                 else:
                     item = model.config.id2label[item]
                     writer.write(f"{index}\t{item}\n")
-        logger.info(f"Wrote predictions to {output_test_file}!")
+        logger.info(f"Wrote predictions to {output_predict_file}!")
     # endregion