From d289f0af7583bb71b23af53653e16a40fcc13cde Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 12:45:55 -0400 Subject: [PATCH 1/9] glue example --- examples/pytorch/test_accelerate_examples.py | 548 ++++++++++++++++++ .../run_glue_no_trainer.py | 18 +- 2 files changed, 562 insertions(+), 4 deletions(-) create mode 100644 examples/pytorch/test_accelerate_examples.py diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py new file mode 100644 index 000000000000..8c22017bade1 --- /dev/null +++ b/examples/pytorch/test_accelerate_examples.py @@ -0,0 +1,548 @@ +# coding=utf-8 +# Copyright 2018 HuggingFace Inc.. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import json +import logging +import os +import sys +import unittest +from unittest.mock import patch + +import torch + +from transformers import ViTMAEForPreTraining, Wav2Vec2ForPreTraining +from transformers.testing_utils import CaptureLogger, TestCasePlus, get_gpu_count, slow, torch_device +from transformers.utils import is_apex_available + + +SRC_DIRS = [ + os.path.join(os.path.dirname(__file__), dirname) + for dirname in [ + "text-generation", + "text-classification", + "token-classification", + "language-modeling", + "multiple-choice", + "question-answering", + "summarization", + "translation", + "image-classification", + "speech-recognition", + "audio-classification", + "speech-pretraining", + "image-pretraining", + ] +] +sys.path.extend(SRC_DIRS) + + +if SRC_DIRS is not None: + # import run_audio_classification_no_trainer + import run_clm_no_trainer + + # import run_generation_no_trainer + import run_glue_no_trainer + + # import run_image_classification_no_trainer + # import run_mae_no_trainer + import run_mlm_no_trainer + import run_ner_no_trainer + import run_qa_no_trainer as run_squad + + # import run_seq2seq_qa_no_trainer as run_squad_seq2seq + # import run_speech_recognition_ctc_no_trainer + # import run_speech_recognition_seq2seq_no_trainer + import run_summarization_no_trainer + import run_swag_no_trainer + import run_translation_no_trainer + + # import run_wav2vec2_pretraining_no_trainer_no_trainer + + +logging.basicConfig(level=logging.DEBUG) + +logger = logging.getLogger() + + +def get_setup_file(): + parser = argparse.ArgumentParser() + parser.add_argument("-f") + args = parser.parse_args() + return args.f + + +def get_results(output_dir): + results = {} + path = os.path.join(output_dir, "all_results.json") + if os.path.exists(path): + with open(path, "r") as f: + results = json.load(f) + else: + raise ValueError(f"can't find {path}") + return results + + +def is_cuda_and_apex_available(): + is_using_cuda = torch.cuda.is_available() and torch_device == "cuda" + return is_using_cuda and is_apex_available() + + +class ExamplesTests(TestCasePlus): + def test_run_glue(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_glue_no_trainer.py + --model_name_or_path distilbert-base-uncased + --output_dir {tmp_dir} + --train_file ./tests/fixtures/tests_samples/MRPC/train.csv + --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --learning_rate=1e-4 + --seed=42 + --checkpointing_steps=2 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_glue_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.75) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + def test_run_clm(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_clm_no_trainer.py + --model_name_or_path distilgpt2 + --train_file ./tests/fixtures/sample_text.txt + --validation_file ./tests/fixtures/sample_text.txt + --do_train + --do_eval + --block_size 128 + --per_device_train_batch_size 5 + --per_device_eval_batch_size 5 + --num_train_epochs 2 + --output_dir {tmp_dir} + --overwrite_output_dir + --checkpointing_steps=2 + """.split() + + if torch.cuda.device_count() > 1: + # Skipping because there are not enough batches to train the model + would need a drop_last to work. + return + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_clm.main() + result = get_results(tmp_dir) + self.assertLess(result["perplexity"], 100) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + def test_run_mlm(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_mlm_no_trainer.py + --model_name_or_path distilroberta-base + --train_file ./tests/fixtures/sample_text.txt + --validation_file ./tests/fixtures/sample_text.txt + --output_dir {tmp_dir} + --overwrite_output_dir + --do_train + --do_eval + --prediction_loss_only + --num_train_epochs=1 + --checkpointing_steps=2 + """.split() + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_mlm.main() + result = get_results(tmp_dir) + self.assertLess(result["perplexity"], 42) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + def test_run_ner(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu + epochs = 7 if get_gpu_count() > 1 else 2 + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_ner_no_trainer.py + --model_name_or_path bert-base-uncased + --train_file tests/fixtures/tests_samples/conll/sample.json + --validation_file tests/fixtures/tests_samples/conll/sample.json + --output_dir {tmp_dir} + --overwrite_output_dir + --do_train + --do_eval + --warmup_steps=2 + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=2 + --num_train_epochs={epochs} + --seed 7 + --checkpointing_steps=2 + """.split() + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_ner.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.75) + self.assertLess(result["eval_loss"], 0.5) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + def test_run_squad(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_qa_no_trainer.py + --model_name_or_path bert-base-uncased + --version_2_with_negative + --train_file tests/fixtures/tests_samples/SQUAD/sample.json + --validation_file tests/fixtures/tests_samples/SQUAD/sample.json + --output_dir {tmp_dir} + --overwrite_output_dir + --max_steps=10 + --warmup_steps=2 + --do_train + --do_eval + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_squad.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_f1"], 30) + self.assertGreaterEqual(result["eval_exact"], 30) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + def test_run_swag(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_swag_no_trainer.py + --model_name_or_path bert-base-uncased + --train_file tests/fixtures/tests_samples/swag/sample.json + --validation_file tests/fixtures/tests_samples/swag/sample.json + --output_dir {tmp_dir} + --overwrite_output_dir + --max_steps=20 + --warmup_steps=2 + --do_train + --do_eval + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_swag.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.8) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + @slow + def test_run_summarization(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_summarization_no_trainer.py + --model_name_or_path t5-small + --train_file tests/fixtures/tests_samples/xsum/sample.json + --validation_file tests/fixtures/tests_samples/xsum/sample.json + --output_dir {tmp_dir} + --overwrite_output_dir + --max_steps=50 + --warmup_steps=8 + --do_train + --do_eval + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --predict_with_generate + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_summarization.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_rouge1"], 10) + self.assertGreaterEqual(result["eval_rouge2"], 2) + self.assertGreaterEqual(result["eval_rougeL"], 7) + self.assertGreaterEqual(result["eval_rougeLsum"], 7) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + @slow + def test_run_translation(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_translation_no_trainer.py + --model_name_or_path sshleifer/student_marian_en_ro_6_1 + --source_lang en + --target_lang ro + --train_file tests/fixtures/tests_samples/wmt16/sample.json + --validation_file tests/fixtures/tests_samples/wmt16/sample.json + --output_dir {tmp_dir} + --overwrite_output_dir + --max_steps=50 + --warmup_steps=8 + --do_train + --do_eval + --learning_rate=3e-3 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --predict_with_generate + --source_lang en_XX + --target_lang ro_RO + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_translation.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_bleu"], 30) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) + + @unittest.skip("This is currently broken.") + def test_run_image_classification(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_image_classification.py + --output_dir {tmp_dir} + --model_name_or_path google/vit-base-patch16-224-in21k + --dataset_name hf-internal-testing/cats_vs_dogs_sample + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --dataloader_num_workers 16 + --metric_for_best_model accuracy + --max_steps 10 + --train_val_split 0.1 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_image_classification.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.8) + + def test_run_speech_recognition_ctc(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_speech_recognition_ctc.py + --output_dir {tmp_dir} + --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 + --dataset_name hf-internal-testing/librispeech_asr_dummy + --dataset_config_name clean + --train_split_name validation + --eval_split_name validation + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --preprocessing_num_workers 16 + --max_steps 10 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_speech_recognition_ctc.main() + result = get_results(tmp_dir) + self.assertLess(result["eval_loss"], result["train_loss"]) + + def test_run_speech_recognition_seq2seq(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_speech_recognition_seq2seq.py + --output_dir {tmp_dir} + --model_name_or_path hf-internal-testing/tiny-random-speech-encoder-decoder + --dataset_name hf-internal-testing/librispeech_asr_dummy + --dataset_config_name clean + --train_split_name validation + --eval_split_name validation + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 4 + --remove_unused_columns False + --overwrite_output_dir True + --preprocessing_num_workers 16 + --max_steps 10 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_speech_recognition_seq2seq.main() + result = get_results(tmp_dir) + self.assertLess(result["eval_loss"], result["train_loss"]) + + def test_run_audio_classification(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_audio_classification.py + --output_dir {tmp_dir} + --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 + --dataset_name anton-l/superb_demo + --dataset_config_name ks + --train_split_name test + --eval_split_name test + --audio_column_name audio + --label_column_name label + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --num_train_epochs 10 + --max_steps 50 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_audio_classification.main() + result = get_results(tmp_dir) + self.assertLess(result["eval_loss"], result["train_loss"]) + + def test_run_wav2vec2_pretraining(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_wav2vec2_pretraining_no_trainer.py + --output_dir {tmp_dir} + --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 + --dataset_name hf-internal-testing/librispeech_asr_dummy + --dataset_config_names clean + --dataset_split_names validation + --learning_rate 1e-4 + --per_device_train_batch_size 4 + --per_device_eval_batch_size 4 + --preprocessing_num_workers 16 + --max_train_steps 2 + --validation_split_percentage 5 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_wav2vec2_pretraining_no_trainer.main() + model = Wav2Vec2ForPreTraining.from_pretrained(tmp_dir) + self.assertIsNotNone(model) + + @unittest.skip("This is currently broken.") + def test_run_vit_mae_pretraining(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_mae.py + --output_dir {tmp_dir} + --dataset_name hf-internal-testing/cats_vs_dogs_sample + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --dataloader_num_workers 16 + --metric_for_best_model accuracy + --max_steps 10 + --train_val_split 0.1 + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_mae.main() + model = ViTMAEForPreTraining.from_pretrained(tmp_dir) + self.assertIsNotNone(model) diff --git a/examples/pytorch/text-classification/run_glue_no_trainer.py b/examples/pytorch/text-classification/run_glue_no_trainer.py index 5bd1d1fa1e5c..e240635044ee 100644 --- a/examples/pytorch/text-classification/run_glue_no_trainer.py +++ b/examples/pytorch/text-classification/run_glue_no_trainer.py @@ -14,6 +14,7 @@ # limitations under the License. """ Finetuning a 🤗 Transformers model for sequence classification on GLUE.""" import argparse +import json import logging import math import os @@ -150,7 +151,6 @@ def parse_args(): "--hub_model_id", type=str, help="The name of the repository to keep in sync with the local `output_dir`." ) parser.add_argument("--hub_token", type=str, help="The token to use to push to the Model Hub.") - parser.add_argument("--hub_token", type=str, help="The token to use to push to the Model Hub.") parser.add_argument( "--checkpointing_steps", type=str, @@ -488,7 +488,10 @@ def preprocess_function(examples): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -526,7 +529,10 @@ def preprocess_function(examples): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -557,6 +563,10 @@ def preprocess_function(examples): eval_metric = metric.compute() logger.info(f"mnli-mm: {eval_metric}") + if args.output_dir is not None: + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"eval_accuracy": eval_metric["accuracy"]}, f) + if __name__ == "__main__": - main() + main() \ No newline at end of file From dfcc307cec48cd9f69a7092d37424ba1a9935ba0 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 12:47:34 -0400 Subject: [PATCH 2/9] Keep it small for review --- examples/pytorch/test_accelerate_examples.py | 420 +------------------ 1 file changed, 1 insertion(+), 419 deletions(-) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 8c22017bade1..5daf668e00cc 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -127,422 +127,4 @@ def test_run_glue(self): run_glue_no_trainer.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) - - def test_run_clm(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_clm_no_trainer.py - --model_name_or_path distilgpt2 - --train_file ./tests/fixtures/sample_text.txt - --validation_file ./tests/fixtures/sample_text.txt - --do_train - --do_eval - --block_size 128 - --per_device_train_batch_size 5 - --per_device_eval_batch_size 5 - --num_train_epochs 2 - --output_dir {tmp_dir} - --overwrite_output_dir - --checkpointing_steps=2 - """.split() - - if torch.cuda.device_count() > 1: - # Skipping because there are not enough batches to train the model + would need a drop_last to work. - return - - if torch_device != "cuda": - testargs.append("--no_cuda") - - with patch.object(sys, "argv", testargs): - run_clm.main() - result = get_results(tmp_dir) - self.assertLess(result["perplexity"], 100) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - def test_run_mlm(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_mlm_no_trainer.py - --model_name_or_path distilroberta-base - --train_file ./tests/fixtures/sample_text.txt - --validation_file ./tests/fixtures/sample_text.txt - --output_dir {tmp_dir} - --overwrite_output_dir - --do_train - --do_eval - --prediction_loss_only - --num_train_epochs=1 - --checkpointing_steps=2 - """.split() - - if torch_device != "cuda": - testargs.append("--no_cuda") - - with patch.object(sys, "argv", testargs): - run_mlm.main() - result = get_results(tmp_dir) - self.assertLess(result["perplexity"], 42) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - def test_run_ner(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu - epochs = 7 if get_gpu_count() > 1 else 2 - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_ner_no_trainer.py - --model_name_or_path bert-base-uncased - --train_file tests/fixtures/tests_samples/conll/sample.json - --validation_file tests/fixtures/tests_samples/conll/sample.json - --output_dir {tmp_dir} - --overwrite_output_dir - --do_train - --do_eval - --warmup_steps=2 - --learning_rate=2e-4 - --per_device_train_batch_size=2 - --per_device_eval_batch_size=2 - --num_train_epochs={epochs} - --seed 7 - --checkpointing_steps=2 - """.split() - - if torch_device != "cuda": - testargs.append("--no_cuda") - - with patch.object(sys, "argv", testargs): - run_ner.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_accuracy"], 0.75) - self.assertLess(result["eval_loss"], 0.5) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - def test_run_squad(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_qa_no_trainer.py - --model_name_or_path bert-base-uncased - --version_2_with_negative - --train_file tests/fixtures/tests_samples/SQUAD/sample.json - --validation_file tests/fixtures/tests_samples/SQUAD/sample.json - --output_dir {tmp_dir} - --overwrite_output_dir - --max_steps=10 - --warmup_steps=2 - --do_train - --do_eval - --learning_rate=2e-4 - --per_device_train_batch_size=2 - --per_device_eval_batch_size=1 - --checkpointing_steps=2 - """.split() - - with patch.object(sys, "argv", testargs): - run_squad.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_f1"], 30) - self.assertGreaterEqual(result["eval_exact"], 30) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - def test_run_swag(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_swag_no_trainer.py - --model_name_or_path bert-base-uncased - --train_file tests/fixtures/tests_samples/swag/sample.json - --validation_file tests/fixtures/tests_samples/swag/sample.json - --output_dir {tmp_dir} - --overwrite_output_dir - --max_steps=20 - --warmup_steps=2 - --do_train - --do_eval - --learning_rate=2e-4 - --per_device_train_batch_size=2 - --per_device_eval_batch_size=1 - --checkpointing_steps=2 - """.split() - - with patch.object(sys, "argv", testargs): - run_swag.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_accuracy"], 0.8) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - @slow - def test_run_summarization(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_summarization_no_trainer.py - --model_name_or_path t5-small - --train_file tests/fixtures/tests_samples/xsum/sample.json - --validation_file tests/fixtures/tests_samples/xsum/sample.json - --output_dir {tmp_dir} - --overwrite_output_dir - --max_steps=50 - --warmup_steps=8 - --do_train - --do_eval - --learning_rate=2e-4 - --per_device_train_batch_size=2 - --per_device_eval_batch_size=1 - --predict_with_generate - --checkpointing_steps=2 - """.split() - - with patch.object(sys, "argv", testargs): - run_summarization.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_rouge1"], 10) - self.assertGreaterEqual(result["eval_rouge2"], 2) - self.assertGreaterEqual(result["eval_rougeL"], 7) - self.assertGreaterEqual(result["eval_rougeLsum"], 7) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - @slow - def test_run_translation(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_translation_no_trainer.py - --model_name_or_path sshleifer/student_marian_en_ro_6_1 - --source_lang en - --target_lang ro - --train_file tests/fixtures/tests_samples/wmt16/sample.json - --validation_file tests/fixtures/tests_samples/wmt16/sample.json - --output_dir {tmp_dir} - --overwrite_output_dir - --max_steps=50 - --warmup_steps=8 - --do_train - --do_eval - --learning_rate=3e-3 - --per_device_train_batch_size=2 - --per_device_eval_batch_size=1 - --predict_with_generate - --source_lang en_XX - --target_lang ro_RO - --checkpointing_steps=2 - """.split() - - with patch.object(sys, "argv", testargs): - run_translation.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_bleu"], 30) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_0"))) - - @unittest.skip("This is currently broken.") - def test_run_image_classification(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_image_classification.py - --output_dir {tmp_dir} - --model_name_or_path google/vit-base-patch16-224-in21k - --dataset_name hf-internal-testing/cats_vs_dogs_sample - --do_train - --do_eval - --learning_rate 1e-4 - --per_device_train_batch_size 2 - --per_device_eval_batch_size 1 - --remove_unused_columns False - --overwrite_output_dir True - --dataloader_num_workers 16 - --metric_for_best_model accuracy - --max_steps 10 - --train_val_split 0.1 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_image_classification.main() - result = get_results(tmp_dir) - self.assertGreaterEqual(result["eval_accuracy"], 0.8) - - def test_run_speech_recognition_ctc(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_speech_recognition_ctc.py - --output_dir {tmp_dir} - --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 - --dataset_name hf-internal-testing/librispeech_asr_dummy - --dataset_config_name clean - --train_split_name validation - --eval_split_name validation - --do_train - --do_eval - --learning_rate 1e-4 - --per_device_train_batch_size 2 - --per_device_eval_batch_size 1 - --remove_unused_columns False - --overwrite_output_dir True - --preprocessing_num_workers 16 - --max_steps 10 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_speech_recognition_ctc.main() - result = get_results(tmp_dir) - self.assertLess(result["eval_loss"], result["train_loss"]) - - def test_run_speech_recognition_seq2seq(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_speech_recognition_seq2seq.py - --output_dir {tmp_dir} - --model_name_or_path hf-internal-testing/tiny-random-speech-encoder-decoder - --dataset_name hf-internal-testing/librispeech_asr_dummy - --dataset_config_name clean - --train_split_name validation - --eval_split_name validation - --do_train - --do_eval - --learning_rate 1e-4 - --per_device_train_batch_size 2 - --per_device_eval_batch_size 4 - --remove_unused_columns False - --overwrite_output_dir True - --preprocessing_num_workers 16 - --max_steps 10 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_speech_recognition_seq2seq.main() - result = get_results(tmp_dir) - self.assertLess(result["eval_loss"], result["train_loss"]) - - def test_run_audio_classification(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_audio_classification.py - --output_dir {tmp_dir} - --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 - --dataset_name anton-l/superb_demo - --dataset_config_name ks - --train_split_name test - --eval_split_name test - --audio_column_name audio - --label_column_name label - --do_train - --do_eval - --learning_rate 1e-4 - --per_device_train_batch_size 2 - --per_device_eval_batch_size 1 - --remove_unused_columns False - --overwrite_output_dir True - --num_train_epochs 10 - --max_steps 50 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_audio_classification.main() - result = get_results(tmp_dir) - self.assertLess(result["eval_loss"], result["train_loss"]) - - def test_run_wav2vec2_pretraining(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_wav2vec2_pretraining_no_trainer.py - --output_dir {tmp_dir} - --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 - --dataset_name hf-internal-testing/librispeech_asr_dummy - --dataset_config_names clean - --dataset_split_names validation - --learning_rate 1e-4 - --per_device_train_batch_size 4 - --per_device_eval_batch_size 4 - --preprocessing_num_workers 16 - --max_train_steps 2 - --validation_split_percentage 5 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_wav2vec2_pretraining_no_trainer.main() - model = Wav2Vec2ForPreTraining.from_pretrained(tmp_dir) - self.assertIsNotNone(model) - - @unittest.skip("This is currently broken.") - def test_run_vit_mae_pretraining(self): - stream_handler = logging.StreamHandler(sys.stdout) - logger.addHandler(stream_handler) - - tmp_dir = self.get_auto_remove_tmp_dir() - testargs = f""" - run_mae.py - --output_dir {tmp_dir} - --dataset_name hf-internal-testing/cats_vs_dogs_sample - --do_train - --do_eval - --learning_rate 1e-4 - --per_device_train_batch_size 2 - --per_device_eval_batch_size 1 - --remove_unused_columns False - --overwrite_output_dir True - --dataloader_num_workers 16 - --metric_for_best_model accuracy - --max_steps 10 - --train_val_split 0.1 - --seed 42 - """.split() - - if is_cuda_and_apex_available(): - testargs.append("--fp16") - - with patch.object(sys, "argv", testargs): - run_mae.main() - model = ViTMAEForPreTraining.from_pretrained(tmp_dir) - self.assertIsNotNone(model) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) \ No newline at end of file From b41ba8b7cab63d8968390fe89f989055fcb0795d Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 12:50:13 -0400 Subject: [PATCH 3/9] Style --- examples/pytorch/test_accelerate_examples.py | 2 +- examples/pytorch/text-classification/run_glue_no_trainer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 5daf668e00cc..9e98667b82f4 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -127,4 +127,4 @@ def test_run_glue(self): run_glue_no_trainer.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) \ No newline at end of file + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) diff --git a/examples/pytorch/text-classification/run_glue_no_trainer.py b/examples/pytorch/text-classification/run_glue_no_trainer.py index e240635044ee..2c7fa186d0e0 100644 --- a/examples/pytorch/text-classification/run_glue_no_trainer.py +++ b/examples/pytorch/text-classification/run_glue_no_trainer.py @@ -569,4 +569,4 @@ def preprocess_function(examples): if __name__ == "__main__": - main() \ No newline at end of file + main() From 9b1536c6b632ecf5cb80711330c649c69387bfd7 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 12:59:53 -0400 Subject: [PATCH 4/9] Unused imports --- examples/pytorch/test_accelerate_examples.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 9e98667b82f4..610267184e72 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -52,23 +52,23 @@ if SRC_DIRS is not None: # import run_audio_classification_no_trainer - import run_clm_no_trainer + # import run_clm_no_trainer # import run_generation_no_trainer import run_glue_no_trainer # import run_image_classification_no_trainer # import run_mae_no_trainer - import run_mlm_no_trainer - import run_ner_no_trainer - import run_qa_no_trainer as run_squad + # import run_mlm_no_trainer + # import run_ner_no_trainer + # import run_qa_no_trainer as run_squad # import run_seq2seq_qa_no_trainer as run_squad_seq2seq # import run_speech_recognition_ctc_no_trainer # import run_speech_recognition_seq2seq_no_trainer - import run_summarization_no_trainer - import run_swag_no_trainer - import run_translation_no_trainer + # import run_summarization_no_trainer + # import run_swag_no_trainer + # import run_translation_no_trainer # import run_wav2vec2_pretraining_no_trainer_no_trainer From a9d6e8b7827f49e4264ee94fba85b21b1c3f29e1 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 15:46:49 -0400 Subject: [PATCH 5/9] Four written tests --- .../language-modeling/run_clm_no_trainer.py | 15 +- .../language-modeling/run_mlm_no_trainer.py | 22 ++- .../multiple-choice/run_swag_no_trainer.py | 14 +- .../question-answering/run_qa_no_trainer.py | 14 +- examples/pytorch/test_accelerate_examples.py | 150 +++++++++++++++++- .../run_ner_no_trainer.py | 15 +- 6 files changed, 211 insertions(+), 19 deletions(-) diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index 76eca5486939..a32172a16093 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -23,6 +23,7 @@ # You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -537,7 +538,10 @@ def group_texts(examples): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -581,7 +585,10 @@ def group_texts(examples): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -592,6 +599,10 @@ def group_texts(examples): if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"perplexity": perplexity}, f) + + if __name__ == "__main__": main() diff --git a/examples/pytorch/language-modeling/run_mlm_no_trainer.py b/examples/pytorch/language-modeling/run_mlm_no_trainer.py index 6a3b48c3b1c2..2634cc25e5b2 100755 --- a/examples/pytorch/language-modeling/run_mlm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_mlm_no_trainer.py @@ -23,6 +23,7 @@ # You can also adapt this script on your own mlm task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -457,9 +458,11 @@ def group_texts(examples): train_dataset = tokenized_datasets["train"] eval_dataset = tokenized_datasets["validation"] - # Log a few random samples from the training set: - for index in random.sample(range(len(train_dataset)), 3): - logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") + # Conditional for small test subsets + if len(train_dataset) > 3: + # Log a few random samples from the training set: + for index in random.sample(range(len(train_dataset)), 3): + logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") # Data collator # This one will take care of randomly masking the tokens. @@ -581,7 +584,10 @@ def group_texts(examples): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -625,7 +631,10 @@ def group_texts(examples): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -636,6 +645,9 @@ def group_texts(examples): if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"perplexity": perplexity}, f) + if __name__ == "__main__": main() diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index d97fb71f395c..88e17b4dfa0d 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -19,6 +19,7 @@ # You can also adapt this script on your own multiple choice task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -540,7 +541,10 @@ def preprocess_function(examples): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -577,6 +581,11 @@ def preprocess_function(examples): repo.push_to_hub( commit_message=f"Training in progress epoch {epoch}", blocking=False, auto_lfs_prune=True ) + if args.checkpointing_steps == "epoch": + output_dir = f"epoch_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -586,7 +595,8 @@ def preprocess_function(examples): tokenizer.save_pretrained(args.output_dir) if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) - + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"eval_accuracy":eval_metric["accuracy"]}, f) if __name__ == "__main__": main() diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index 08f8339036c2..d806cd95ec50 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -19,6 +19,7 @@ # You can also adapt this script on your own question answering task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -783,7 +784,10 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -880,7 +884,10 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): accelerator.log(log, step=completed_steps) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -890,6 +897,9 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): tokenizer.save_pretrained(args.output_dir) if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + logger.info(eval_metric) + json.dump({"eval_f1":eval_metric['f1'], "eval_exact":eval_metric['exact']}, f) if __name__ == "__main__": diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 610267184e72..10211d2f1773 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -52,22 +52,22 @@ if SRC_DIRS is not None: # import run_audio_classification_no_trainer - # import run_clm_no_trainer + import run_clm_no_trainer # import run_generation_no_trainer import run_glue_no_trainer # import run_image_classification_no_trainer # import run_mae_no_trainer - # import run_mlm_no_trainer - # import run_ner_no_trainer - # import run_qa_no_trainer as run_squad + import run_mlm_no_trainer + import run_ner_no_trainer + import run_qa_no_trainer as run_squad_no_trainer # import run_seq2seq_qa_no_trainer as run_squad_seq2seq # import run_speech_recognition_ctc_no_trainer # import run_speech_recognition_seq2seq_no_trainer # import run_summarization_no_trainer - # import run_swag_no_trainer + import run_swag_no_trainer # import run_translation_no_trainer # import run_wav2vec2_pretraining_no_trainer_no_trainer @@ -128,3 +128,143 @@ def test_run_glue(self): result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + def test_run_clm(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_clm_no_trainer.py + --model_name_or_path distilgpt2 + --train_file ./tests/fixtures/sample_text.txt + --validation_file ./tests/fixtures/sample_text.txt + --block_size 128 + --per_device_train_batch_size 5 + --per_device_eval_batch_size 5 + --num_train_epochs 2 + --output_dir {tmp_dir} + --checkpointing_steps=2 + """.split() + + if torch.cuda.device_count() > 1: + # Skipping because there are not enough batches to train the model + would need a drop_last to work. + return + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_clm_no_trainer.main() + result = get_results(tmp_dir) + self.assertLess(result["perplexity"], 100) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + def test_run_mlm(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_mlm_no_trainer.py + --model_name_or_path distilroberta-base + --train_file ./tests/fixtures/sample_text.txt + --validation_file ./tests/fixtures/sample_text.txt + --output_dir {tmp_dir} + --num_train_epochs=1 + --checkpointing_steps epoch + """.split() + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_mlm_no_trainer.main() + result = get_results(tmp_dir) + self.assertLess(result["perplexity"], 42) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) + + def test_run_ner(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu + epochs = 7 if get_gpu_count() > 1 else 2 + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_ner_no_trainer.py + --model_name_or_path bert-base-uncased + --train_file tests/fixtures/tests_samples/conll/sample.json + --validation_file tests/fixtures/tests_samples/conll/sample.json + --output_dir {tmp_dir} + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=2 + --num_train_epochs={epochs} + --seed 7 + --checkpointing_steps=2 + """.split() + + if torch_device != "cuda": + testargs.append("--no_cuda") + + with patch.object(sys, "argv", testargs): + run_ner_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.75) + self.assertLess(result["train_loss"], 0.5) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + def test_run_squad(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_qa_no_trainer.py + --model_name_or_path bert-base-uncased + --version_2_with_negative=False + --train_file tests/fixtures/tests_samples/SQUAD/sample.json + --validation_file tests/fixtures/tests_samples/SQUAD/sample.json + --output_dir {tmp_dir} + --max_train_steps=10 + --num_warmup_steps=2 + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_squad_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_f1"], 30) + self.assertGreaterEqual(result["eval_exact"], 30) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + def test_run_swag(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_swag_no_trainer.py + --model_name_or_path bert-base-uncased + --train_file tests/fixtures/tests_samples/swag/sample.json + --validation_file tests/fixtures/tests_samples/swag/sample.json + --output_dir {tmp_dir} + --max_train_steps=20 + --num_warmup_steps=2 + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_swag_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_accuracy"], 0.8) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index 57d3ceee905d..e36bd54aa693 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -19,6 +19,7 @@ """ import argparse +import json import logging import math import os @@ -639,7 +640,10 @@ def compute_metrics(): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -662,7 +666,6 @@ def compute_metrics(): references=refs, ) # predictions and preferences are expected to be a nested list of labels, not label_ids - # eval_metric = metric.compute() eval_metric = compute_metrics() accelerator.print(f"epoch {epoch}:", eval_metric) if args.with_tracking: @@ -686,7 +689,10 @@ def compute_metrics(): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -697,6 +703,9 @@ def compute_metrics(): if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"eval_accuracy":eval_metric["accuracy"], "train_loss":float(loss.cpu().detach().numpy())}, f) + if __name__ == "__main__": main() From a8c0b4d876601e5d37b70dccee4dde2ff2d967d9 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 16:24:21 -0400 Subject: [PATCH 6/9] Finish all example tests --- .circleci/config.yml | 1 + .../language-modeling/run_clm_no_trainer.py | 1 - .../multiple-choice/run_swag_no_trainer.py | 3 +- .../question-answering/run_qa_no_trainer.py | 10 +-- .../run_summarization_no_trainer.py | 21 +++++- examples/pytorch/test_accelerate_examples.py | 66 +++++++++++++++++-- .../run_ner_no_trainer.py | 2 +- .../translation/run_translation_no_trainer.py | 13 +++- setup.py | 2 +- utils/tests_fetcher.py | 1 + 10 files changed, 102 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 856211e280cb..1bfe5d29f7f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -587,6 +587,7 @@ jobs: - run: pip install --upgrade pip - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech] - run: pip install -r examples/pytorch/_tests_requirements.txt + - run: pip install git+https://github.com/huggingface/accelerate - save_cache: key: v0.4-torch_examples-{{ checksum "setup.py" }} paths: diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index a32172a16093..247ba09d54ab 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -601,7 +601,6 @@ def group_texts(examples): with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: json.dump({"perplexity": perplexity}, f) - if __name__ == "__main__": diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index 88e17b4dfa0d..7b691459bce7 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -596,7 +596,8 @@ def preprocess_function(examples): if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: - json.dump({"eval_accuracy":eval_metric["accuracy"]}, f) + json.dump({"eval_accuracy": eval_metric["accuracy"]}, f) + if __name__ == "__main__": main() diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index d806cd95ec50..2f5a22238175 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -884,10 +884,10 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): accelerator.log(log, step=completed_steps) if args.checkpointing_steps == "epoch": - output_dir = f"epoch_{epoch}" - if args.output_dir is not None: - output_dir = os.path.join(args.output_dir, output_dir) - accelerator.save_state(output_dir) + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -899,7 +899,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: logger.info(eval_metric) - json.dump({"eval_f1":eval_metric['f1'], "eval_exact":eval_metric['exact']}, f) + json.dump({"eval_f1": eval_metric["f1"], "eval_exact": eval_metric["exact"]}, f) if __name__ == "__main__": diff --git a/examples/pytorch/summarization/run_summarization_no_trainer.py b/examples/pytorch/summarization/run_summarization_no_trainer.py index fd2bb2cc8162..adc9e616dda0 100644 --- a/examples/pytorch/summarization/run_summarization_no_trainer.py +++ b/examples/pytorch/summarization/run_summarization_no_trainer.py @@ -19,6 +19,7 @@ # You can also adapt this script on your own summarization task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -602,7 +603,10 @@ def postprocess_text(preds, labels): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -669,7 +673,10 @@ def postprocess_text(preds, labels): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -679,6 +686,16 @@ def postprocess_text(preds, labels): tokenizer.save_pretrained(args.output_dir) if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump( + { + "eval_rouge1": result["rouge1"], + "eval_rouge2": result["rouge2"], + "eval_rougeL": result["rougeL"], + "eval_rougeLsum": result["rougeLsum"], + }, + f, + ) if __name__ == "__main__": diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 10211d2f1773..7a19659d08f4 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -19,13 +19,11 @@ import logging import os import sys -import unittest from unittest.mock import patch import torch -from transformers import ViTMAEForPreTraining, Wav2Vec2ForPreTraining -from transformers.testing_utils import CaptureLogger, TestCasePlus, get_gpu_count, slow, torch_device +from transformers.testing_utils import TestCasePlus, get_gpu_count, slow, torch_device from transformers.utils import is_apex_available @@ -66,9 +64,9 @@ # import run_seq2seq_qa_no_trainer as run_squad_seq2seq # import run_speech_recognition_ctc_no_trainer # import run_speech_recognition_seq2seq_no_trainer - # import run_summarization_no_trainer + import run_summarization_no_trainer import run_swag_no_trainer - # import run_translation_no_trainer + import run_translation_no_trainer # import run_wav2vec2_pretraining_no_trainer_no_trainer @@ -268,3 +266,61 @@ def test_run_swag(self): self.assertGreaterEqual(result["eval_accuracy"], 0.8) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + @slow + def test_run_summarization(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_summarization_no_trainer.py + --model_name_or_path t5-small + --train_file tests/fixtures/tests_samples/xsum/sample.json + --validation_file tests/fixtures/tests_samples/xsum/sample.json + --output_dir {tmp_dir} + --max_train_steps=50 + --num_warmup_steps=8 + --learning_rate=2e-4 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_summarization_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_rouge1"], 10) + self.assertGreaterEqual(result["eval_rouge2"], 2) + self.assertGreaterEqual(result["eval_rougeL"], 7) + self.assertGreaterEqual(result["eval_rougeLsum"], 7) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + + @slow + def test_run_translation(self): + stream_handler = logging.StreamHandler(sys.stdout) + logger.addHandler(stream_handler) + + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_translation_no_trainer.py + --model_name_or_path sshleifer/student_marian_en_ro_6_1 + --source_lang en + --target_lang ro + --train_file tests/fixtures/tests_samples/wmt16/sample.json + --validation_file tests/fixtures/tests_samples/wmt16/sample.json + --output_dir {tmp_dir} + --max_train_steps=50 + --num_warmup_steps=8 + --learning_rate=3e-3 + --per_device_train_batch_size=2 + --per_device_eval_batch_size=1 + --source_lang en_XX + --target_lang ro_RO + --checkpointing_steps=2 + """.split() + + with patch.object(sys, "argv", testargs): + run_translation_no_trainer.main() + result = get_results(tmp_dir) + self.assertGreaterEqual(result["eval_bleu"], 30) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index e36bd54aa693..ab9fcce6df95 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -704,7 +704,7 @@ def compute_metrics(): repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: - json.dump({"eval_accuracy":eval_metric["accuracy"], "train_loss":float(loss.cpu().detach().numpy())}, f) + json.dump({"eval_accuracy": eval_metric["accuracy"], "train_loss": float(loss.cpu().detach().numpy())}, f) if __name__ == "__main__": diff --git a/examples/pytorch/translation/run_translation_no_trainer.py b/examples/pytorch/translation/run_translation_no_trainer.py index bf7e15ae4dd1..034387582b84 100644 --- a/examples/pytorch/translation/run_translation_no_trainer.py +++ b/examples/pytorch/translation/run_translation_no_trainer.py @@ -19,6 +19,7 @@ # You can also adapt this script on your own text translation task. Pointers for this are left as comments. import argparse +import json import logging import math import os @@ -586,7 +587,10 @@ def postprocess_text(preds, labels): if isinstance(checkpointing_steps, int): if completed_steps % checkpointing_steps == 0: - accelerator.save_state(f"step_{completed_steps}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if completed_steps >= args.max_train_steps: break @@ -653,7 +657,10 @@ def postprocess_text(preds, labels): ) if args.checkpointing_steps == "epoch": - accelerator.save_state(f"epoch_{epoch}") + output_dir = f"step_{completed_steps}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) if args.output_dir is not None: accelerator.wait_for_everyone() @@ -663,6 +670,8 @@ def postprocess_text(preds, labels): tokenizer.save_pretrained(args.output_dir) if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) + with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + json.dump({"eval_bleu": eval_metric["score"]}, f) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 4d386ae00825..5fb6dba5841d 100644 --- a/setup.py +++ b/setup.py @@ -284,7 +284,7 @@ def run(self): "rouge-score", "nltk", "GitPython", - "hf-doc-builder", + "hf-doc-builder" ) + extras["retrieval"] + extras["modelcreation"] diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 5ad7b4b1f788..16bf6348d387 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -466,6 +466,7 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None): # Example files are tested separately elif f.startswith("examples/pytorch"): test_files_to_run.append("examples/pytorch/test_pytorch_examples.py") + test_files_to_run.append("examples/pytorch/test_accelerate_examples.py") elif f.startswith("examples/flax"): test_files_to_run.append("examples/flax/test_flax_examples.py") else: From c49aea5a012514be74cf6814fb09739f090cb60a Mon Sep 17 00:00:00 2001 From: muellerzr Date: Thu, 7 Apr 2022 16:46:22 -0400 Subject: [PATCH 7/9] Fixup args --- examples/pytorch/test_accelerate_examples.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 7a19659d08f4..b9ecee88c41c 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -149,9 +149,6 @@ def test_run_clm(self): # Skipping because there are not enough batches to train the model + would need a drop_last to work. return - if torch_device != "cuda": - testargs.append("--no_cuda") - with patch.object(sys, "argv", testargs): run_clm_no_trainer.main() result = get_results(tmp_dir) @@ -173,9 +170,6 @@ def test_run_mlm(self): --checkpointing_steps epoch """.split() - if torch_device != "cuda": - testargs.append("--no_cuda") - with patch.object(sys, "argv", testargs): run_mlm_no_trainer.main() result = get_results(tmp_dir) @@ -204,9 +198,6 @@ def test_run_ner(self): --checkpointing_steps=2 """.split() - if torch_device != "cuda": - testargs.append("--no_cuda") - with patch.object(sys, "argv", testargs): run_ner_no_trainer.main() result = get_results(tmp_dir) From 7c7cc41140b7775d68c6b57cf3860794aab3fb49 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Fri, 8 Apr 2022 08:35:14 -0400 Subject: [PATCH 8/9] Add comma to setup, NoTrainer and no_trainer suffixes --- examples/pytorch/test_accelerate_examples.py | 31 ++++++-------------- setup.py | 2 +- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index b9ecee88c41c..565b7e0d1270 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -49,28 +49,15 @@ if SRC_DIRS is not None: - # import run_audio_classification_no_trainer import run_clm_no_trainer - - # import run_generation_no_trainer import run_glue_no_trainer - - # import run_image_classification_no_trainer - # import run_mae_no_trainer import run_mlm_no_trainer import run_ner_no_trainer import run_qa_no_trainer as run_squad_no_trainer - - # import run_seq2seq_qa_no_trainer as run_squad_seq2seq - # import run_speech_recognition_ctc_no_trainer - # import run_speech_recognition_seq2seq_no_trainer import run_summarization_no_trainer import run_swag_no_trainer import run_translation_no_trainer - # import run_wav2vec2_pretraining_no_trainer_no_trainer - - logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger() @@ -99,8 +86,8 @@ def is_cuda_and_apex_available(): return is_using_cuda and is_apex_available() -class ExamplesTests(TestCasePlus): - def test_run_glue(self): +class ExamplesTestsNoTrainer(TestCasePlus): + def test_run_glue_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -127,7 +114,7 @@ def test_run_glue(self): self.assertGreaterEqual(result["eval_accuracy"], 0.75) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) - def test_run_clm(self): + def test_run_clm_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -155,7 +142,7 @@ def test_run_clm(self): self.assertLess(result["perplexity"], 100) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) - def test_run_mlm(self): + def test_run_mlm_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -176,7 +163,7 @@ def test_run_mlm(self): self.assertLess(result["perplexity"], 42) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) - def test_run_ner(self): + def test_run_ner_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -205,7 +192,7 @@ def test_run_ner(self): self.assertLess(result["train_loss"], 0.5) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) - def test_run_squad(self): + def test_run_squad_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -232,7 +219,7 @@ def test_run_squad(self): self.assertGreaterEqual(result["eval_exact"], 30) self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) - def test_run_swag(self): + def test_run_swag_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -258,7 +245,7 @@ def test_run_swag(self): self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) @slow - def test_run_summarization(self): + def test_run_summarization_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) @@ -287,7 +274,7 @@ def test_run_summarization(self): self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) @slow - def test_run_translation(self): + def test_run_translation_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stream_handler) diff --git a/setup.py b/setup.py index 5fb6dba5841d..4d386ae00825 100644 --- a/setup.py +++ b/setup.py @@ -284,7 +284,7 @@ def run(self): "rouge-score", "nltk", "GitPython", - "hf-doc-builder" + "hf-doc-builder", ) + extras["retrieval"] + extras["modelcreation"] From 68c9a0c9b628cf5a6cad3cf5b2770f686059b272 Mon Sep 17 00:00:00 2001 From: muellerzr Date: Fri, 8 Apr 2022 09:54:25 -0400 Subject: [PATCH 9/9] Wrap up tests --- .../multiple-choice/run_swag_no_trainer.py | 3 ++- .../question-answering/run_qa_no_trainer.py | 13 +++++----- examples/pytorch/test_accelerate_examples.py | 26 +++++++++---------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index 7b691459bce7..a575644130f0 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -581,8 +581,9 @@ def preprocess_function(examples): repo.push_to_hub( commit_message=f"Training in progress epoch {epoch}", blocking=False, auto_lfs_prune=True ) + if args.checkpointing_steps == "epoch": - output_dir = f"epoch_{completed_steps}" + output_dir = f"epoch_{epoch}" if args.output_dir is not None: output_dir = os.path.join(args.output_dir, output_dir) accelerator.save_state(output_dir) diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index 2f5a22238175..6da75822398c 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -792,6 +792,12 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): if completed_steps >= args.max_train_steps: break + if args.checkpointing_steps == "epoch": + output_dir = f"epoch_{epoch}" + if args.output_dir is not None: + output_dir = os.path.join(args.output_dir, output_dir) + accelerator.save_state(output_dir) + if args.push_to_hub and epoch < args.num_train_epochs - 1: accelerator.wait_for_everyone() unwrapped_model = accelerator.unwrap_model(model) @@ -883,12 +889,6 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): accelerator.log(log, step=completed_steps) - if args.checkpointing_steps == "epoch": - output_dir = f"epoch_{epoch}" - if args.output_dir is not None: - output_dir = os.path.join(args.output_dir, output_dir) - accelerator.save_state(output_dir) - if args.output_dir is not None: accelerator.wait_for_everyone() unwrapped_model = accelerator.unwrap_model(model) @@ -898,7 +898,6 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): if args.push_to_hub: repo.push_to_hub(commit_message="End of training", auto_lfs_prune=True) with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: - logger.info(eval_metric) json.dump({"eval_f1": eval_metric["f1"], "eval_exact": eval_metric["exact"]}, f) diff --git a/examples/pytorch/test_accelerate_examples.py b/examples/pytorch/test_accelerate_examples.py index 565b7e0d1270..883dc434deb7 100644 --- a/examples/pytorch/test_accelerate_examples.py +++ b/examples/pytorch/test_accelerate_examples.py @@ -102,7 +102,7 @@ def test_run_glue_no_trainer(self): --per_device_eval_batch_size=1 --learning_rate=1e-4 --seed=42 - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() if is_cuda_and_apex_available(): @@ -112,7 +112,7 @@ def test_run_glue_no_trainer(self): run_glue_no_trainer.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) def test_run_clm_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) @@ -129,7 +129,7 @@ def test_run_clm_no_trainer(self): --per_device_eval_batch_size 5 --num_train_epochs 2 --output_dir {tmp_dir} - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() if torch.cuda.device_count() > 1: @@ -140,7 +140,7 @@ def test_run_clm_no_trainer(self): run_clm_no_trainer.main() result = get_results(tmp_dir) self.assertLess(result["perplexity"], 100) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) def test_run_mlm_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) @@ -182,7 +182,7 @@ def test_run_ner_no_trainer(self): --per_device_eval_batch_size=2 --num_train_epochs={epochs} --seed 7 - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() with patch.object(sys, "argv", testargs): @@ -190,7 +190,7 @@ def test_run_ner_no_trainer(self): result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.75) self.assertLess(result["train_loss"], 0.5) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) def test_run_squad_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) @@ -209,7 +209,7 @@ def test_run_squad_no_trainer(self): --learning_rate=2e-4 --per_device_train_batch_size=2 --per_device_eval_batch_size=1 - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() with patch.object(sys, "argv", testargs): @@ -217,7 +217,7 @@ def test_run_squad_no_trainer(self): result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_f1"], 30) self.assertGreaterEqual(result["eval_exact"], 30) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) def test_run_swag_no_trainer(self): stream_handler = logging.StreamHandler(sys.stdout) @@ -235,14 +235,12 @@ def test_run_swag_no_trainer(self): --learning_rate=2e-4 --per_device_train_batch_size=2 --per_device_eval_batch_size=1 - --checkpointing_steps=2 """.split() with patch.object(sys, "argv", testargs): run_swag_no_trainer.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_accuracy"], 0.8) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) @slow def test_run_summarization_no_trainer(self): @@ -261,7 +259,7 @@ def test_run_summarization_no_trainer(self): --learning_rate=2e-4 --per_device_train_batch_size=2 --per_device_eval_batch_size=1 - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() with patch.object(sys, "argv", testargs): @@ -271,7 +269,7 @@ def test_run_summarization_no_trainer(self): self.assertGreaterEqual(result["eval_rouge2"], 2) self.assertGreaterEqual(result["eval_rougeL"], 7) self.assertGreaterEqual(result["eval_rougeLsum"], 7) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0"))) @slow def test_run_translation_no_trainer(self): @@ -294,11 +292,11 @@ def test_run_translation_no_trainer(self): --per_device_eval_batch_size=1 --source_lang en_XX --target_lang ro_RO - --checkpointing_steps=2 + --checkpointing_steps epoch """.split() with patch.object(sys, "argv", testargs): run_translation_no_trainer.main() result = get_results(tmp_dir) self.assertGreaterEqual(result["eval_bleu"], 30) - self.assertTrue(os.path.exists(os.path.join(tmp_dir, "step_2"))) + self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))