[MXAPPS-581] Nightly Straight Dope tests. (#11814)

* [MXAPPS-581] Nightly Straight Dope tests.

The Straight Dope notebooks will retrieved from the Github repo, run and
scanned for warnings and errors. Because we are not checking accuracy of
the training, we set the number of epochs to 1 to reduce the integration
test run time.
* Common functionality for running and testing notebooks has been
  factored into a common test util module.
* Support for running UTF-8 notebooks added (Python2 and 3 compatible).
* Notebooks requiring a single GPU and multi GPUs have been split
  into two different test suites so that they can be run on different
  hardware.
* Add test to make sure that all notebooks are tested.
* Comment out broken notebooks while they are being fixed (I will
  uncomment them in a follow up PR).

* [MXAPPS-581] Download notebooks in test setup.

* Moving logic to download the Straight Dope notebooks to the test
harness.
* Remove cache logic as it is unnecessary.

* [MXAPPS-581] Add a timeout for download of notebooks.

* [MXAPPS-581] Move notebooks requiring multi-gpus.

Move two notebooks requiring multi-GPUs out of the single GPU test suite.

ci/docker/runtime_functions.sh

@@ -895,6 +895,44 @@ nightly_test_javascript() {
     make -C /work/mxnet/amalgamation libmxnet_predict.js MIN=1 EMCC=/work/deps/emscripten/emcc
 }
 
+# Nightly 'MXNet: The Straight Dope' Single-GPU Tests
+nightly_straight_dope_python2_single_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python2
+    nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \
+      test_notebooks_single_gpu.py --nologcapture
+}
+
+nightly_straight_dope_python3_single_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
+    nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_single_gpu.xml \
+      test_notebooks_single_gpu.py --nologcapture
+}
+
+# Nightly 'MXNet: The Straight Dope' Multi-GPU Tests
+nightly_straight_dope_python2_multi_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python2
+    nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \
+      test_notebooks_multi_gpu.py --nologcapture
+}
+
+nightly_straight_dope_python3_multi_gpu_tests() {
+    set -ex
+    cd /work/mxnet/tests/nightly/straight_dope
+    export PYTHONPATH=/work/mxnet/python/
+    export MXNET_TEST_KERNEL=python3
+    nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_multi_gpu.xml \
+      test_notebooks_multi_gpu.py --nologcapture
+}
+
 # Deploy
 
 deploy_docs() {

tests/nightly/JenkinsfileForBinaries

@@ -98,6 +98,42 @@ try {
           docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true) 
         }
       }
+    },
+    'StraightDope: Python2 Single-GPU': {
+      node('mxnetlinux-gpu-p3') {
+        ws('workspace/straight_dope-single_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_single_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python2 Multi-GPU': {
+      node('mxnetlinux-gpu') {
+        ws('workspace/straight_dope-multi_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python2_multi_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python3 Single-GPU': {
+      node('mxnetlinux-gpu-p3') {
+        ws('workspace/straight_dope-single_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_single_gpu_tests', true)
+        }
+      }
+    },
+    'StraightDope: Python3 Multi-GPU': {
+      node('mxnetlinux-gpu') {
+        ws('workspace/straight_dope-multi_gpu') {
+          init_git()
+          unpack_lib('gpu', mx_lib)
+          docker_run('ubuntu_nightly_gpu', 'nightly_straight_dope_python3_multi_gpu_tests', true)
+        }
+      }
     }
   }
 } catch (caughtError) {

tests/nightly/straight_dope/README.md

@@ -0,0 +1,7 @@
+# Nightly Tests for MXNet: The Straight Dope
+
+These are some longer running tests that are scheduled to run every night. 
+
+### Description
+These tests verify that the straight dope tutorials run without error. They are
+run on both single and multi-gpu configurations.

tests/nightly/straight_dope/straight_dope_test_utils.py

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name
+"""
+    This file tests and ensures that all straight dope notebooks run
+    without warning or exception.
+
+    env variable MXNET_TEST_KERNEL controls which kernel to use when running
+    the notebook. e.g: `export MXNET_TEST_KERNEL=python2`
+"""
+import io
+import os
+import re
+import shutil
+import subprocess
+import sys
+from time import sleep
+
+#TODO(vishaalk): Find a cleaner way to import this notebook.
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'utils'))
+from notebook_test import run_notebook
+
+EPOCHS_REGEX = r'epochs\s+=\s+[0-9]+'  # Regular expression that matches 'epochs = #'
+GIT_PATH = '/usr/bin/git'
+GIT_REPO = 'https://github.com/zackchase/mxnet-the-straight-dope'
+KERNEL = os.getenv('MXNET_TEST_KERNEL', None)
+NOTEBOOKS_DIR = os.path.join(os.path.dirname(__file__), 'tmp_notebook')
+
+def _test_notebook(notebook, override_epochs=True):
+    """Run Jupyter notebook to catch any execution error.
+
+    Args:
+        notebook : string
+            notebook name in folder/notebook format
+        epochs : boolean
+            whether or not to override the number of epochs to 1
+    Returns:
+        True if the notebook runs without warning or error.
+    """
+    if override_epochs:
+        _override_epochs(notebook)
+    return run_notebook(notebook, NOTEBOOKS_DIR, kernel=KERNEL, temp_dir=NOTEBOOKS_DIR)
+
+
+def _override_epochs(notebook):
+    """Overrides the number of epochs in the notebook to 1 epoch. Note this operation is idempotent.
+
+    Args:
+        notebook : string
+            notebook name in folder/notebook format
+
+    """
+    notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"
+
+    # Read the notebook and set epochs to num_epochs
+    with io.open(notebook_path, 'r', encoding='utf-8') as f:
+        notebook = f.read()
+
+    # Set number of epochs to 1
+    modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)
+
+    # Replace the original notebook with the modified one.
+    with io.open(notebook_path, 'w', encoding='utf-8') as f:
+        f.write(modified_notebook)
+
+
+def _download_straight_dope_notebooks():
+    """Downloads the Straight Dope Notebooks.
+
+    Returns:
+        True if it succeeds in downloading the notebooks without error.
+    """
+    print('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
+    shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)
+
+    cmd = [GIT_PATH,
+           'clone',
+           GIT_REPO,
+           NOTEBOOKS_DIR]
+
+    proc, msg = _run_command(cmd)
+
+    if proc.returncode != 0:
+        err_msg = 'Error downloading Straight Dope notebooks.\n'
+        err_msg += msg
+        print(err_msg)
+        return False
+    return True
+
+def _run_command(cmd, timeout_secs=300):
+    """ Runs a command with a specified timeout.
+
+    Args:
+        cmd : list of string
+            The command with arguments to run.
+        timeout_secs: integer
+            The timeout in seconds
+
+    Returns:
+        Returns the process and the output as a pair.
+    """
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+
+    for i in range(timeout_secs):
+        sleep(1)
+        if proc.poll() is not None:
+            (out, _) = proc.communicate()
+            return proc, out.decode('utf-8')
+
+    proc.kill()
+    return proc, "Timeout of %s secs exceeded." % timeout_secs
+

tests/nightly/straight_dope/test_notebooks_multi_gpu.py

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name, missing-docstring
+"""
+    This file tests that the notebooks requiring multi GPUs run without
+    warning or exception.
+"""
+import unittest
+from straight_dope_test_utils import _test_notebook
+from straight_dope_test_utils import _download_straight_dope_notebooks
+
+class StraightDopeMultiGpuTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        assert _download_straight_dope_notebooks()
+
+    # Chapter 7
+
+    # TODO(vishaalk): module 'mxnet.gluon' has no attribute 'autograd'
+    #def test_multiple_gpus_scratch(self):
+    #    assert _test_notebook('chapter07_distributed-learning/multiple-gpus-scratch')
+
+    def test_multiple_gpus_gluon(self):
+        assert _test_notebook('chapter07_distributed-learning/multiple-gpus-gluon')
+
+    # TODO(vishaalk): Do a dry run, and then enable.
+    #def test_training_with_multiple_machines(self):
+    #    assert _test_notebook('chapter07_distributed-learning/training-with-multiple-machines')
+
+    # Chapter 8
+
+    # TODO(vishaalk): Module skimage needs to be added to docker image.
+    # def test_fine_tuning(self):
+    #    assert _test_notebook('chapter08_computer-vision/fine-tuning')