[MXAPPS-581] Download notebooks in test setup.

* Moving logic to download the Straight Dope notebooks to the test
harness.
* Remove cache logic as it is unnecessary.

ci/docker/runtime_functions.sh

@@ -896,36 +896,39 @@ nightly_test_javascript() {
 }
 
 # Nightly 'MXNet: The Straight Dope' Single-GPU Tests
-set_up_nightly_straight_dope_tests() {
+nightly_straight_dope_python2_single_gpu_tests() {
     set -ex
     cd /work/mxnet/tests/nightly/straight_dope
-    rm -rf ./straight_dope_book
-    git clone https://github.com/zackchase/mxnet-the-straight-dope straight_dope_book
     export PYTHONPATH=/work/mxnet/python/
-    export MXNET_TEST_KERNEL=${1}
-}
-
-nightly_straight_dope_python2_single_gpu_tests() {
-    set_up_nightly_straight_dope_tests python2
+    export MXNET_TEST_KERNEL=python2
     nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \
       test_notebooks_single_gpu.py --nologcapture
 }
 
 nightly_straight_dope_python3_single_gpu_tests() {
-    set_up_nightly_straight_dope_tests python3
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
     nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_single_gpu.xml \
       test_notebooks_single_gpu.py --nologcapture
 }
 
 # Nightly 'MXNet: The Straight Dope' Multi-GPU Tests
 nightly_straight_dope_python2_multi_gpu_tests() {
-    set_up_nightly_straight_dope_tests python2
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python2
     nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \
       test_notebooks_multi_gpu.py --nologcapture
 }
 
 nightly_straight_dope_python3_multi_gpu_tests() {
-    set_up_nightly_straight_dope_tests python3
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
     nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_multi_gpu.xml \
       test_notebooks_multi_gpu.py --nologcapture
 }

tests/nightly/straight_dope/straight_dope_test_utils.py

@@ -22,28 +22,22 @@
 
     env variable MXNET_TEST_KERNEL controls which kernel to use when running
     the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
-
-    env variable MXNET_TEST_NO_CACHE controls whether to clean the
-    temporary directory in which the notebook was run and re-download any
-    resource file. The default behaviour is to not clean the directory. Set to '1'
-    to force clean the directory. e.g: `export MXNET_TEST_NO_CACHE=1`
-    NB: in the real CI, the tests will re-download everything since they start from
-    a clean workspace.
 """
 import io
 import os
 import re
 import shutil
+import subprocess
 import sys
 
 #TODO(vishaalk): Find a cleaner way to import this notebook.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'utils'))
 from notebook_test import run_notebook
 
 EPOCHS_REGEX = r'epochs\s+=\s+[0-9]+'  # Regular expression that matches 'epochs = #'
+GIT_PATH = '/usr/bin/git'
 KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
-NO_CACHE = os.getenv('MXNET_TEST_NO_CACHE', False)
-NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'straight_dope_book')
+NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
 
 def _test_notebook(notebook, override_epochs=True):
     """Run Jupyter notebook to catch any execution error.
@@ -58,15 +52,16 @@ def _test_notebook(notebook, override_epochs=True):
     """
     if override_epochs:
         _override_epochs(notebook)
-    return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, no_cache=NO_CACHE)
+    return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR)
 
 
 def _override_epochs(notebook):
-    """Overrides the number of epochs in the notebook to 1 epoch.
+    """Overrides the number of epochs in the notebook to 1 epoch. Note this operation is idempotent.
 
     Args:
         notebook : string
             notebook name in folder/notebook format
+
     """
     notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"
 
@@ -76,7 +71,35 @@ def _override_epochs(notebook):
 
     # Set number of epochs to 1
     modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
-    
+
     # Replace the original notebook with the modified one.
     with io.open(notebook_path, 'w', encoding='utf-8') as f:
         f.write(modified_notebook)
+
+
+def _download_straight_dope_notebooks():
+    """Downloads the Straight Dope Notebooks.
+
+    Returns:
+        True if it succeeds in downloading the notebooks without error.
+    """
+    print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
+    shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
+
+    cmd = [GIT_PATH,
+           'clone',
+           'https://github.com/zackchase/mxnet-the-straight-dope',
+           NOTEBOOKS_DIR]
+
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    (out, _) = proc.communicate()
+
+    if proc.returncode != 0:
+        msg = 'Error downloading Straight Dope notebooks.\n'
+        msg += out.decode('utf-8')
+        print(msg)
+        return False
+    return True

tests/nightly/straight_dope/test_notebooks_multi_gpu.py

@@ -20,19 +20,26 @@
     This file tests that the notebooks requiring multi GPUs run without
     warning or exception.
 """
+import unittest
 from straight_dope_test_utils import _test_notebook
+from straight_dope_test_utils import _download_straight_dope_notebooks
 
-# Chapter 7
+class StraightDopeMultiGpuTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        assert _download_straight_dope_notebooks()
 
-# TODO(vishaalk): module 'mxnet.gluon' has no attribute 'autograd'
-#def test_multiple_gpus_scratch():
-#    assert _test_notebook('chapter07_distributed-learning/multiple-gpus-scratch')
+    # Chapter 7
 
-def test_multiple_gpus_gluon():
-    assert _test_notebook('chapter07_distributed-learning/multiple-gpus-gluon')
+    # TODO(vishaalk): module 'mxnet.gluon' has no attribute 'autograd'
+    #def test_multiple_gpus_scratch(self):
+    #    assert _test_notebook('chapter07_distributed-learning/multiple-gpus-scratch')
 
-# Chapter 8
+    def test_multiple_gpus_gluon(self):
+        assert _test_notebook('chapter07_distributed-learning/multiple-gpus-gluon')
 
-# TODO(vishaalk): Module skimage needs to be added to docker image.
-# def test_fine_tuning():
-#    assert _test_notebook('chapter08_computer-vision/fine-tuning')
+    # Chapter 8
+
+    # TODO(vishaalk): Module skimage needs to be added to docker image.
+    # def test_fine_tuning(self):
+    #    assert _test_notebook('chapter08_computer-vision/fine-tuning')