From bc0630d44642956980b0fafa9cebe742adab4df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:23:53 +0200 Subject: [PATCH 1/6] update tests with EarlyStopping default --- tests/backends/test_ddp_spawn.py | 2 +- tests/callbacks/test_early_stopping.py | 2 +- tests/models/test_gpu.py | 2 +- tests/models/test_tpu.py | 4 ++-- .../test_trainer_steps_result_return.py | 10 +++++----- .../test_validation_steps_result_return.py | 8 ++++---- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/backends/test_ddp_spawn.py b/tests/backends/test_ddp_spawn.py index 0c5db6b1a0b8b..c718b6ab25694 100644 --- a/tests/backends/test_ddp_spawn.py +++ b/tests/backends/test_ddp_spawn.py @@ -15,7 +15,7 @@ def test_multi_gpu_early_stop_ddp_spawn(tmpdir): trainer_options = dict( default_root_dir=tmpdir, - early_stop_callback=True, + callbacks=[EarlyStopping()], max_epochs=50, limit_train_batches=10, limit_val_batches=10, diff --git a/tests/callbacks/test_early_stopping.py b/tests/callbacks/test_early_stopping.py index 98ff939ae6ca3..8a1daaf695a2f 100644 --- a/tests/callbacks/test_early_stopping.py +++ b/tests/callbacks/test_early_stopping.py @@ -77,7 +77,7 @@ def test_early_stopping_no_extraneous_invocations(tmpdir): expected_count = 4 trainer = Trainer( default_root_dir=tmpdir, - early_stop_callback=True, + callbacks=[EarlyStopping()], val_check_interval=1.0, max_epochs=expected_count, ) diff --git a/tests/models/test_gpu.py b/tests/models/test_gpu.py index 8c2be4cabc594..2737e3ea7a15b 100644 --- a/tests/models/test_gpu.py +++ b/tests/models/test_gpu.py @@ -25,7 +25,7 @@ def test_multi_gpu_early_stop_dp(tmpdir): trainer_options = dict( default_root_dir=tmpdir, - early_stop_callback=True, + callbacks=[EarlyStopping()], max_epochs=50, limit_train_batches=10, limit_val_batches=10, diff --git a/tests/models/test_tpu.py b/tests/models/test_tpu.py index ef82cf4e46f56..71260e3829f6f 100644 --- a/tests/models/test_tpu.py +++ b/tests/models/test_tpu.py @@ -155,7 +155,7 @@ def test_model_tpu_early_stop(tmpdir): """Test if single TPU core training works""" model = EvalModelTemplate() trainer = Trainer( - early_stop_callback=True, + callbacks=[EarlyStopping()], default_root_dir=tmpdir, progress_bar_refresh_rate=0, max_epochs=50, @@ -261,7 +261,7 @@ def test_result_obj_on_tpu(tmpdir): trainer_options = dict( default_root_dir=tmpdir, max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=batches, weights_summary=None, diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py index 40236deef7a1e..4329cb4dd8299 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py @@ -420,7 +420,7 @@ def test_no_auto_callbacks_with_train_loop_only(tmpdir): trainer = Trainer( default_root_dir=tmpdir, - early_stop_callback=True, + callbacks=[EarlyStopping()], max_epochs=epochs, row_log_interval=1, limit_train_batches=batches, @@ -475,7 +475,7 @@ def test_use_callbacks_with_train_loop_only(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=1, limit_train_batches=batches, weights_summary=None, @@ -531,7 +531,7 @@ def test_full_train_loop_with_results_obj_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=batches, weights_summary=None, @@ -572,7 +572,7 @@ def test_loop_steps_only_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=batches, weights_summary=None, @@ -612,7 +612,7 @@ def test_result_monitor_warnings(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=2, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=2, weights_summary=None, diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py index e7e305adc576d..d554d01f8a339 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py @@ -39,7 +39,7 @@ def test_val_step_result_callbacks(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=1, limit_train_batches=batches, weights_summary=None, @@ -87,7 +87,7 @@ def test_val_step_using_train_callbacks(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=1, limit_train_batches=batches, weights_summary=None, @@ -420,7 +420,7 @@ def test_val_step_full_loop_result_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=batches, weights_summary=None, @@ -474,7 +474,7 @@ def test_full_loop_result_cpu(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - early_stop_callback=True, + callbacks=[EarlyStopping()], row_log_interval=2, limit_train_batches=batches, weights_summary=None, From 26b7e52d5248ed05048772a70b9006fe9847e584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:25:54 +0200 Subject: [PATCH 2/6] imports --- tests/backends/test_ddp_spawn.py | 1 + tests/models/test_gpu.py | 1 + tests/models/test_tpu.py | 1 + .../test_validation_steps_result_return.py | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/backends/test_ddp_spawn.py b/tests/backends/test_ddp_spawn.py index c718b6ab25694..a1573b69ed207 100644 --- a/tests/backends/test_ddp_spawn.py +++ b/tests/backends/test_ddp_spawn.py @@ -3,6 +3,7 @@ import tests.base.develop_pipelines as tpipes import tests.base.develop_utils as tutils +from pytorch_lightning.callbacks import EarlyStopping from tests.base import EvalModelTemplate from pytorch_lightning.core import memory from pytorch_lightning.trainer import Trainer diff --git a/tests/models/test_gpu.py b/tests/models/test_gpu.py index 2737e3ea7a15b..4b3d95c254eec 100644 --- a/tests/models/test_gpu.py +++ b/tests/models/test_gpu.py @@ -8,6 +8,7 @@ import tests.base.develop_pipelines as tpipes import tests.base.develop_utils as tutils from pytorch_lightning import Trainer +from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.core import memory from pytorch_lightning.utilities import device_parser from pytorch_lightning.utilities.exceptions import MisconfigurationException diff --git a/tests/models/test_tpu.py b/tests/models/test_tpu.py index 71260e3829f6f..cddc3db78ac4e 100644 --- a/tests/models/test_tpu.py +++ b/tests/models/test_tpu.py @@ -6,6 +6,7 @@ import tests.base.develop_pipelines as tpipes from pytorch_lightning import Trainer, seed_everything from pytorch_lightning.accelerators import TPUBackend +from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.base import EvalModelTemplate from tests.base.datasets import TrialMNIST diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py index d554d01f8a339..9a667b69eb2d6 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py @@ -7,6 +7,7 @@ import torch from pytorch_lightning import Trainer +from pytorch_lightning.callbacks import EarlyStopping from tests.base import EvalModelTemplate from tests.base.deterministic_model import DeterministicModel from pytorch_lightning import seed_everything From 40d83bda1f8d78b53a8d5d783ec64d9b0892c96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:50:57 +0200 Subject: [PATCH 3/6] revert legacy tests --- .../test_trainer_steps_result_return.py | 2 +- .../test_validation_steps_result_return.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py index 4329cb4dd8299..5ae1ea17e6df7 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py @@ -420,7 +420,7 @@ def test_no_auto_callbacks_with_train_loop_only(tmpdir): trainer = Trainer( default_root_dir=tmpdir, - callbacks=[EarlyStopping()], + early_stop_callback=True, max_epochs=epochs, row_log_interval=1, limit_train_batches=batches, diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py index 9a667b69eb2d6..491a2c911515e 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py @@ -421,7 +421,7 @@ def test_val_step_full_loop_result_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=2, limit_train_batches=batches, weights_summary=None, @@ -475,7 +475,7 @@ def test_full_loop_result_cpu(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=2, limit_train_batches=batches, weights_summary=None, From 7a8fe83d7bc93b1a77f8c4c84471a5c32db22ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:51:26 +0200 Subject: [PATCH 4/6] fix test --- pytorch_lightning/callbacks/early_stopping.py | 2 +- pytorch_lightning/utilities/debugging.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/callbacks/early_stopping.py b/pytorch_lightning/callbacks/early_stopping.py index 2330e69ecd72c..3177c9300efb3 100644 --- a/pytorch_lightning/callbacks/early_stopping.py +++ b/pytorch_lightning/callbacks/early_stopping.py @@ -181,7 +181,7 @@ def _run_early_stopping_check(self, trainer, pl_module): current = logs.get(self.monitor) # when in dev debugging - trainer.dev_debugger.track_early_stopping_history(current) + trainer.dev_debugger.track_early_stopping_history(self, current) if not isinstance(current, torch.Tensor): current = torch.tensor(current, device=pl_module.device) diff --git a/pytorch_lightning/utilities/debugging.py b/pytorch_lightning/utilities/debugging.py index 79a35cba4781b..242f3105d780c 100644 --- a/pytorch_lightning/utilities/debugging.py +++ b/pytorch_lightning/utilities/debugging.py @@ -154,15 +154,14 @@ def track_pbar_metrics_history(self, metrics): self.pbar_added_metrics.append(metrics) @enabled_only - def track_early_stopping_history(self, current): - es = self.trainer.early_stop_callback + def track_early_stopping_history(self, callback, current): debug_dict = { 'epoch': self.trainer.current_epoch, 'global_step': self.trainer.global_step, 'rank': self.trainer.global_rank, 'current': current, - 'best': es.best_score, - 'patience': es.wait_count + 'best': callback.best_score, + 'patience': callback.wait_count } self.early_stopping_history.append(debug_dict) From e204c4e8af37e1cf520e12bdd182ed958ae8285b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:57:21 +0200 Subject: [PATCH 5/6] revert --- .../test_trainer_steps_result_return.py | 4 ++-- .../test_validation_steps_result_return.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py index 5ae1ea17e6df7..a60ae959f1b26 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py @@ -475,7 +475,7 @@ def test_use_callbacks_with_train_loop_only(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=1, limit_train_batches=batches, weights_summary=None, @@ -572,7 +572,7 @@ def test_loop_steps_only_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=2, limit_train_batches=batches, weights_summary=None, diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py index 491a2c911515e..e8e2d63b6a16a 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py @@ -40,7 +40,7 @@ def test_val_step_result_callbacks(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=1, limit_train_batches=batches, weights_summary=None, @@ -88,7 +88,7 @@ def test_val_step_using_train_callbacks(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=1, limit_train_batches=batches, weights_summary=None, From 55d9466a1be98bb3e91b40d3f21cad340a98a9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 4 Oct 2020 19:58:44 +0200 Subject: [PATCH 6/6] revert --- .../test_trainer_steps_result_return.py | 4 ++-- .../test_validation_steps_result_return.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py index a60ae959f1b26..40236deef7a1e 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_trainer_steps_result_return.py @@ -531,7 +531,7 @@ def test_full_train_loop_with_results_obj_dp(tmpdir): distributed_backend='dp', gpus=[0, 1], max_epochs=epochs, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=2, limit_train_batches=batches, weights_summary=None, @@ -612,7 +612,7 @@ def test_result_monitor_warnings(tmpdir): trainer = Trainer( default_root_dir=tmpdir, max_epochs=2, - callbacks=[EarlyStopping()], + early_stop_callback=True, row_log_interval=2, limit_train_batches=2, weights_summary=None, diff --git a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py index e8e2d63b6a16a..e7e305adc576d 100644 --- a/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py +++ b/tests/trainer/legacy_deprecate_flow_log_tests/test_validation_steps_result_return.py @@ -7,7 +7,6 @@ import torch from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import EarlyStopping from tests.base import EvalModelTemplate from tests.base.deterministic_model import DeterministicModel from pytorch_lightning import seed_everything