From 6372037ec4eb56b609a184712e5ec993a21faad0 Mon Sep 17 00:00:00 2001 From: vishaalkapoor <40836875+vishaalkapoor@users.noreply.github.com> Date: Sun, 29 Jul 2018 12:40:32 -0700 Subject: [PATCH] [MXAPPS-581] Fixes for broken Straight Dope tests. (#11923) * Update relative paths pointing to the data directory to point to the correct place in the testing temporary folder. * Enable the notebooks that were previously broken because of relative file paths not pointing to the correct place. * Move some notebooks we do not plan to test to the whitelist. These notebooks are not published in the Straight Dope book. * Clean-up: Convert print statements to info/warn/error logging statements. Add some logging statements for better status. --- .../straight_dope/straight_dope_test_utils.py | 39 ++++++-- .../test_notebooks_single_gpu.py | 94 +++++-------------- tests/utils/notebook_test/__init__.py | 7 +- 3 files changed, 61 insertions(+), 79 deletions(-) diff --git a/tests/nightly/straight_dope/straight_dope_test_utils.py b/tests/nightly/straight_dope/straight_dope_test_utils.py index bb64f37fe5f5..ee499a56fbd3 100644 --- a/tests/nightly/straight_dope/straight_dope_test_utils.py +++ b/tests/nightly/straight_dope/straight_dope_test_utils.py @@ -24,6 +24,7 @@ the notebook. e.g: `export MXNET_TEST_KERNEL=python2` """ import io +import logging import os import re import shutil @@ -40,6 +41,7 @@ GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope' KERNEL = os.getenv('MXNET_TEST_KERNEL', None) NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook') +RELATIVE_DATA_PATH_REGEX = r'\.\.\/data\/' # Regular expression to match the relative data path. def _test_notebook(notebook, override_epochs=True): """Run Jupyter notebook to catch any execution error. @@ -47,13 +49,18 @@ def _test_notebook(notebook, override_epochs=True): Args: notebook : string notebook name in folder/notebook format - epochs : boolean + override_epochs : boolean whether or not to override the number of epochs to 1 + Returns: True if the notebook runs without warning or error. """ + # Some notebooks will fail to run without error if we do not override the data path. + _override_data_path(notebook) + if override_epochs: _override_epochs(notebook) + return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR) @@ -63,15 +70,14 @@ def _override_epochs(notebook): Args: notebook : string notebook name in folder/notebook format - """ notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb" - # Read the notebook and set epochs to num_epochs + # Read the notebook and set epochs to num_epochs. with io.open(notebook_path, 'r', encoding='utf-8') as f: notebook = f.read() - # Set number of epochs to 1 + # Set number of epochs to 1. modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook) # Replace the original notebook with the modified one. @@ -79,13 +85,34 @@ def _override_epochs(notebook): f.write(modified_notebook) +def _override_data_path(notebook): + """Overrides the relative path for the data directory to point to the right place. This is + required as we run the notebooks in a different directory hierarchy more suitable for testing. + + Args: + notebook : string + notebook name in folder/notebook format + """ + notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb" + + # Read the notebook and set epochs to num_epochs. + with io.open(notebook_path, 'r', encoding='utf-8') as f: + notebook = f.read() + + # Update the location for the data directory. + modified_notebook = re.sub(RELATIVE_DATA_PATH_REGEX, NOTEBOOKS_DIR + '/data/', notebook) + + # Replace the original notebook with the modified one. + with io.open(notebook_path, 'w', encoding='utf-8') as f: + f.write(modified_notebook) + def _download_straight_dope_notebooks(): """Downloads the Straight Dope Notebooks. Returns: True if it succeeds in downloading the notebooks without error. """ - print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR)) + logging.info('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR)) shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True) cmd = [GIT_PATH, @@ -98,7 +125,7 @@ def _download_straight_dope_notebooks(): if proc.returncode != 0: err_msg = 'Error downloading Straight Dope notebooks.\n' err_msg += msg - print(err_msg) + logging.error(err_msg) return False return True diff --git a/tests/nightly/straight_dope/test_notebooks_single_gpu.py b/tests/nightly/straight_dope/test_notebooks_single_gpu.py index b87d16cb0a0d..ee7c94c80afc 100644 --- a/tests/nightly/straight_dope/test_notebooks_single_gpu.py +++ b/tests/nightly/straight_dope/test_notebooks_single_gpu.py @@ -32,9 +32,19 @@ 'chapter01_crashcourse/introduction', 'chapter01_crashcourse/chapter-one-problem-set', 'chapter02_supervised-learning/environment', + 'chapter03_deep-neural-networks/kaggle-gluon-kfold', 'chapter07_distributed-learning/multiple-gpus-scratch', 'chapter07_distributed-learning/multiple-gpus-gluon', - 'chapter07_distributed-learning/training-with-multiple-machines' + 'chapter07_distributed-learning/training-with-multiple-machines', + 'chapter12_time-series/intro-forecasting-gluon', + 'chapter12_time-series/intro-forecasting-2-gluon', + 'chapter13_unsupervised-learning/vae-gluon', + 'chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn', + 'chapter17_deep-reinforcement-learning/DQN', + 'chapter17_deep-reinforcement-learning/DDQN', + 'chapter19_graph-neural-networks/Graph-Neural-Networks', + 'chapter16_tensor_methods/tensor_basics', + 'cheatsheets/kaggle-gluon-kfold' ] @@ -91,10 +101,8 @@ def test_linear_regression_scratch(self): def test_linear_regression_gluon(self): assert _test_notebook('chapter02_supervised-learning/linear-regression-gluon') - # TODO(vishaalk): There is a relative file path needs to be fixed so that the - # python code can be run from another directory. - #def test_logistic_regression_gluon(self): - # assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon') + def test_logistic_regression_gluon(self): + assert _test_notebook('chapter02_supervised-learning/logistic-regression-gluon') def test_softmax_regression_scratch(self): assert _test_notebook('chapter02_supervised-learning/softmax-regression-scratch') @@ -132,9 +140,6 @@ def test_plumbing(self): def test_custom_layer(self): assert _test_notebook('chapter03_deep-neural-networks/custom-layer') - #def test_kaggle_gluon_kfold(self): - # assert _test_notebook('chapter03_deep-neural-networks/kaggle-gluon-kfold') - # TODO(vishaalk): Load params and Save params are deprecated warning. #def test_serialization(self): # assert _test_notebook('chapter03_deep-neural-networks/serialization') @@ -162,20 +167,14 @@ def test_cnn_batch_norm_gluon(self): # Chapter 5 - # TODO(vishaalk): There is a relative file path needs to be fixed so that the - # python code can be run from another directory. - #def test_simple_rnn(self): - # assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn') + def test_simple_rnn(self): + assert _test_notebook('chapter05_recurrent-neural-networks/simple-rnn') - # TODO(vishaalk): There is a relative file path needs to be fixed so that the - # python code can be run from another directory. - #def test_lstm_scratch(self): - # assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch') + def test_lstm_scratch(self): + assert _test_notebook('chapter05_recurrent-neural-networks/lstm-scratch') - # TODO(vishaalk): There is a relative file path needs to be fixed so that the - # python code can be run from another directory. - #def test_gru_scratch(self): - # assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch') + def test_gru_scratch(self): + assert _test_notebook('chapter05_recurrent-neural-networks/gru-scratch') #def test_rnns_gluon(self): # assert _test_notebook('chapter05_recurrent-neural-networks/rnns-gluon') @@ -263,19 +262,6 @@ def test_lds_scratch(self): #def test_issm_scratch(self): # assert _test_notebook('chapter12_time-series/issm-scratch') - # TODO(vishaalk): Error: sequential1_batchnorm0_running_mean' has not been initialized - # def test_intro_forecasting_gluon(self): - # assert _test_notebook('chapter12_time-series/intro-forecasting-gluon') - - #def test_intro_forecasting_2_gluon(self): - # assert _test_notebook('chapter12_time-series/intro-forecasting-2-gluon') - - # Chapter 13 - - # TODO(vishaalk): Load params and Save params are deprecated warning. - #def test_vae_gluon(self): - # assert _test_notebook('chapter13_unsupervised-learning/vae-gluon') - # Chapter 14 def test_igan_intro(self): @@ -287,46 +273,14 @@ def test_dcgan(self): def test_generative_adversarial_networks(self): assert _test_notebook('chapter14_generative-adversarial-networks/conditional') - # Chapter 16 - - # TODO(vishaalk): Checked failed oshape.Size() != dshape.Size() - #def test_tensor_basics(self): - # assert _test_notebook('chapter16_tensor_methods/tensor_basics') - # TODO(vishaalk): Notebook does not appear to be valid JSON. #def test_pixel2pixel(self): # assert _test_notebook('chapter14_generative-adversarial-networks/pixel2pixel') - # Chapter 17 - - # TODO(vishaalk): Requires OpenAI Gym. Also uses deprecated load_params. - #def test_dqn(self): -# assert _test_notebook('chapter17_deep-reinforcement-learning/DQN') - -#def test_ddqn(self): -# assert _test_notebook('chapter17_deep-reinforcement-learning/DDQN') - -# Chapter 18 - -#def test_bayes_by_backprop(self): -# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop') - -#def test_bayes_by_backprop_gluon(self): -# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon') - -# TODO(vishaalk): AttributeError: 'list' object has no attribute 'keys' -#def test_bayes_by_backprop_rnn(self): -# assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-rnn') - -# Chapter 19 - -# TODO(vishaalk): Requires deepchem -#def test_graph_neural_networks(self): -# assert _test_notebook('chapter19_graph-neural-networks/Graph-Neural-Networks') + # Chapter 18 -# Cheatsheets + #def test_bayes_by_backprop(self): + # assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop') -# TODO(vishaalk): There is a relative file path needs to be fixed so that the -# python code can be run from another directory. -#def test_kaggle_gluon_kfold(self): -# assert _test_notebook('cheatsheets/kaggle-gluon-kfold') + #def test_bayes_by_backprop_gluon(self): + # assert _test_notebook('chapter18_variational-methods-and-uncertainty/bayes-by-backprop-gluon') diff --git a/tests/utils/notebook_test/__init__.py b/tests/utils/notebook_test/__init__.py index cb5282fb4043..2cdb6134a604 100644 --- a/tests/utils/notebook_test/__init__.py +++ b/tests/utils/notebook_test/__init__.py @@ -21,6 +21,7 @@ warning or exception. """ import io +import logging import os import shutil import time @@ -57,12 +58,12 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir=' ------- Returns true if the workbook runs with no warning or exception. """ - + logging.info("Running notebook '{}'".format(notebook)) notebook_path = os.path.join(*([notebook_dir] + notebook.split('/'))) working_dir = os.path.join(*([temp_dir] + notebook.split('/'))) if no_cache == '1': - print("Cleaning and setting up temp directory '{}'".format(working_dir)) + logging.info("Cleaning and setting up temp directory '{}'".format(working_dir)) shutil.rmtree(temp_dir, ignore_errors=True) errors = [] @@ -92,6 +93,6 @@ def run_notebook(notebook, notebook_dir, kernel=None, no_cache=False, temp_dir=' if "Warning:" in line: errors.append("Warning:\n" + line) if len(errors) > 0: - print('\n'.join(errors)) + logging.error('\n'.join(errors)) return False return True