Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test_cpu and test_gpu EvalModelTemplate deprecation #4820

Merged
Merged
Changes from 47 commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
bb37edc
test_cpu refactoring - BoringModel and checkpoints; test_gpu refactor…
gianscarpe Nov 23, 2020
186747d
Removed EvalModelTemplate import from test_cpu and test_gpu
gianscarpe Nov 23, 2020
dddd8e1
Reverting unintended changes
gianscarpe Nov 23, 2020
4348287
Issues with checkpointing
gianscarpe Nov 23, 2020
3861630
Fixed tests for logging and checkpointing
gianscarpe Nov 23, 2020
d974232
Fix for dispatcher
gianscarpe Nov 23, 2020
d078240
Fixed acc check for stocasticity of seeds
gianscarpe Nov 24, 2020
41dd1b0
test_cpu refactoring - BoringModel and checkpoints; test_gpu refactor…
gianscarpe Nov 23, 2020
67c3bcb
Reverting unintended changes
gianscarpe Nov 23, 2020
dba0ab8
Issues with checkpointing
gianscarpe Nov 23, 2020
5955112
Fixed tests for logging and checkpointing
gianscarpe Nov 23, 2020
a5c4920
Fixed according to @borda suggestions
gianscarpe Nov 24, 2020
8c7b802
Hparams for boring_model
gianscarpe Nov 24, 2020
a6af6a3
Deprecated RuntimeParamChagneModelAssing (functionality is tested in …
gianscarpe Nov 24, 2020
ed0cf40
Reduced boring_model parameters to just in and out features, test_cpu…
gianscarpe Nov 24, 2020
00a7f88
Fix PEP8
gianscarpe Nov 25, 2020
b36e5c5
Merged test_early_stopping with all_features; added TODO for self.log
gianscarpe Nov 26, 2020
eaf51f7
Update tests/base/develop_pipelines.py
gianscarpe Nov 25, 2020
e318eb1
Update tests/base/boring_model.py
gianscarpe Nov 25, 2020
2d2c0b8
Update tests/base/develop_pipelines.py
gianscarpe Nov 25, 2020
15050c3
Update tests/models/test_cpu.py
gianscarpe Nov 25, 2020
b775c39
Update tests/models/test_cpu.py
gianscarpe Nov 26, 2020
aa9c1b5
Fixed test_all_features trainer options
gianscarpe Nov 28, 2020
ecca297
Ready for review!
gianscarpe Nov 30, 2020
8275444
Update tests/models/test_cpu.py
gianscarpe Dec 1, 2020
90bf714
added optimizer_name, lr, and batch_size as hparams for save_hparamet…
gianscarpe Dec 1, 2020
3c73a65
Update tests/models/test_cpu.py
gianscarpe Dec 1, 2020
6fec1be
Update tests/models/test_cpu.py
gianscarpe Dec 1, 2020
50a2c9b
Fixes for reducing PR size
gianscarpe Dec 1, 2020
5c9290b
Reverse test_hparams (removed DEPRECATED test for hparams direct assi…
gianscarpe Dec 1, 2020
7757c82
Changes for in_features
gianscarpe Dec 3, 2020
95f5766
Fixed hparams
gianscarpe Dec 3, 2020
9b729e3
Fixed parameters for boring_model
gianscarpe Dec 5, 2020
37f7573
Update tests/models/test_cpu.py
gianscarpe Dec 15, 2020
bd7e8cf
Update tests/models/test_cpu.py
gianscarpe Dec 15, 2020
6fecc09
Update tests/models/test_cpu.py
gianscarpe Dec 15, 2020
6527e3f
Add Google Colab badges (#5111)
shacharmirkin Dec 14, 2020
0a7ba63
fix for pep8
gianscarpe Dec 15, 2020
717ca2f
Fixed run_predction and TODO
gianscarpe Dec 18, 2020
8ee5c99
fix min acc for darwin/windows without pl_opt
gianscarpe Dec 19, 2020
8750fbd
eval as DEFAULT run_prediction strategy
gianscarpe Dec 19, 2020
879f11a
Updated val_dataloader for running_test_no_val
gianscarpe Dec 21, 2020
b899e9c
rebased
gianscarpe Jan 5, 2021
a9589ee
Revert "Updated val_dataloader for running_test_no_val"
gianscarpe Jan 5, 2021
6423611
Revert "Revert "Updated val_dataloader for running_test_no_val""
gianscarpe Jan 5, 2021
e7e5f67
reverting unwanted modifications
gianscarpe Jan 5, 2021
87ef88f
unwanted changes
gianscarpe Jan 5, 2021
1a9525c
Merge branch 'release/1.2-dev' into issue3827_test_gpu_and_cpu
s-rog Jan 6, 2021
707d820
Merge branch 'release/1.2-dev' into issue3827_test_gpu_and_cpu
Borda Jan 7, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 32 additions & 14 deletions tests/base/develop_pipelines.py
Original file line number Diff line number Diff line change
@@ -14,8 +14,8 @@
import torch

from pytorch_lightning import Trainer
from tests.base.develop_utils import load_model_from_checkpoint, get_default_logger, \
reset_seed
from tests.base import BoringModel
from tests.base.develop_utils import get_default_logger, load_model_from_checkpoint, reset_seed


def run_model_test_without_loggers(trainer_options, model, min_acc: float = 0.50):
@@ -31,6 +31,7 @@ def run_model_test_without_loggers(trainer_options, model, min_acc: float = 0.50
pretrained_model = load_model_from_checkpoint(
trainer.logger,
trainer.checkpoint_callback.best_model_path,
type(model)
)

# test new model accuracy
@@ -39,15 +40,16 @@ def run_model_test_without_loggers(trainer_options, model, min_acc: float = 0.50
test_loaders = [test_loaders]

for dataloader in test_loaders:
run_prediction(dataloader, pretrained_model, min_acc=min_acc)
run_prediction(pretrained_model, dataloader, min_acc=min_acc)

if trainer.use_ddp:
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()


def run_model_test(trainer_options, model, on_gpu: bool = True, version=None, with_hpc: bool = True):
def run_model_test(trainer_options, model, on_gpu: bool = True, version=None,
with_hpc: bool = True, min_acc: float = 0.25):
gianscarpe marked this conversation as resolved.
Show resolved Hide resolved

reset_seed()
save_dir = trainer_options['default_root_dir']
@@ -56,35 +58,34 @@ def run_model_test(trainer_options, model, on_gpu: bool = True, version=None, wi
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)

if 'checkpoint_callback' not in trainer_options:
trainer_options.update(checkpoint_callback=True)

trainer = Trainer(**trainer_options)
initial_values = torch.tensor([torch.sum(torch.abs(x)) for x in model.parameters()])
result = trainer.fit(model)
post_train_values = torch.tensor([torch.sum(torch.abs(x)) for x in model.parameters()])

assert result == 1, 'trainer failed'
# Check that the model is actually changed post-training
assert torch.norm(initial_values - post_train_values) > 0.1
change_ratio = torch.norm(initial_values - post_train_values)
assert change_ratio > 0.1, f"the model is changed of {change_ratio}"

# test model loading
pretrained_model = load_model_from_checkpoint(logger, trainer.checkpoint_callback.best_model_path)
pretrained_model = load_model_from_checkpoint(logger, trainer.checkpoint_callback.best_model_path, type(model))

# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]

for dataloader in test_loaders:
run_prediction(dataloader, pretrained_model)
run_prediction(pretrained_model, dataloader, min_acc=min_acc)

if with_hpc:
if trainer.use_ddp or trainer.use_ddp2:
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = \
trainer.init_optimizers(pretrained_model)
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = trainer.init_optimizers(
pretrained_model
)

# test HPC saving
trainer.checkpoint_connector.hpc_save(save_dir, logger)
@@ -93,7 +94,14 @@ def run_model_test(trainer_options, model, on_gpu: bool = True, version=None, wi
trainer.checkpoint_connector.hpc_load(checkpoint_path, on_gpu=on_gpu)


def run_prediction(dataloader, trained_model, dp=False, min_acc=0.50):
def run_prediction(trained_model, dataloader, dp=False, min_acc=0.25):
if isinstance(trained_model, BoringModel):
return _boring_model_run_prediction(trained_model, dataloader, dp, min_acc)
else:
return _eval_model_template_run_prediction(trained_model, dataloader, dp, min_acc)


def _eval_model_template_run_prediction(trained_model, dataloader, dp=False, min_acc=0.50):
# run prediction on 1 batch
batch = next(iter(dataloader))
x, y = batch
@@ -102,7 +110,7 @@ def run_prediction(dataloader, trained_model, dp=False, min_acc=0.50):
if dp:
with torch.no_grad():
output = trained_model(batch, 0)
acc = output['val_acc']
acc = output['val_acc']
acc = torch.mean(acc).item()

else:
@@ -119,3 +127,13 @@ def run_prediction(dataloader, trained_model, dp=False, min_acc=0.50):
acc = acc.item()

assert acc >= min_acc, f"This model is expected to get > {min_acc} in test set (it got {acc})"


def _boring_model_run_prediction(trained_model, dataloader, dp=False, min_acc=0.25):
# run prediction on 1 batch
batch = next(iter(dataloader))
with torch.no_grad():
gianscarpe marked this conversation as resolved.
Show resolved Hide resolved
output = trained_model(batch)
acc = trained_model.loss(batch, output)

assert acc >= min_acc, f"This model is expected to get, {min_acc} in test set but got {acc}"
2 changes: 1 addition & 1 deletion tests/models/data/horovod/train_default_model.py
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ def run_test_from_config(trainer_options):
test_loaders = [test_loaders]

for dataloader in test_loaders:
run_prediction(dataloader, pretrained_model)
run_prediction(pretrained_model, dataloader)

# test HPC saving
trainer.checkpoint_connector.hpc_save(ckpt_path, trainer.logger)
Loading