[DEV] switch nose with pytest (apache#18025)

* switch nose with pytest

* switch centos python to 3.6

* disable dist kvstore tests

* skip hanging test

.github/workflows/os_x_staticbuild.yml

@@ -10,16 +10,16 @@ jobs:
         uses: actions/checkout@v2
       - name: Install Dependencies
         run: |
-          brew install nasm automake ninja libtool cmake pkgconfig protobuf
+          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib
+          python3 -m pip install --user -r ci/docker/install/requirements
       - name: Build project
         run: |
           git --version
           clang --version
           CMAKE_STATICBUILD=1 ./tools/staticbuild/build.sh cpu
       - name: Setup Python
         run: |
-          python3 -m pip install --user nose nose-timer nose-exclude numpy scipy
           python3 -m pip install --user -e python
       - name: Test project
         run: |
-          python3 -m nose --with-timer --verbose tests/python/unittest/ --exclude-test=test_extensions.test_subgraph --exclude-test=test_extensions.test_custom_op --exclude-test=test_gluon_data.test_recordimage_dataset_with_data_loader_multiworker --exclude-test=test_gluon_data.test_multi_worker --exclude-test=test_gluon_data.test_multi_worker_shape --exclude-test=test_gluon_data.test_multi_worker_forked_data_loader --exclude-test=test_gluon_data.test_multi_worker_dataloader_release_pool
+          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)'

3rdparty/mshadow/mshadow/random.h

@@ -336,8 +336,10 @@ class Random<gpu, DType> {
    * \brief get a set of random integers
    */
   inline void GetRandInt(const Tensor<gpu, 1, unsigned>& dst) {
-    curandStatus_t status = curandGenerate(gen_, dst.dptr_, dst.size(0));
-    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << "CURAND Gen rand ints failed.";
+    curandStatus_t status;
+    status = curandGenerate(gen_, dst.dptr_, dst.size(0));
+    CHECK_EQ(status, CURAND_STATUS_SUCCESS) << "CURAND Gen rand ints failed."
+                                            << " size = " << dst.size(0);
   }
   /*!
    * \brief generate data from uniform [a,b)

cd/python/docker/Dockerfile.test

@@ -24,7 +24,7 @@ ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 
 # Install test dependencies
-RUN pip install nose
+RUN pip install pytest
 
 ARG USER_ID=1001
 ARG GROUP_ID=1001

ci/dev_menu.py

@@ -105,8 +105,8 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
         # Install MXNet python bindigs
         check_call([pip, 'install', '--upgrade', '--force-reinstall', '-e', 'python'])
         # Install test dependencies
-        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r', os.path.join('tests',
-            'requirements.txt')])
+        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r',
+                    os.path.join('ci', 'docker', 'install', 'requirements')])
     else:
         logging.warn("Can't find pip: '%s' not found", pip)
 
@@ -119,7 +119,7 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
         provision_virtualenv,
     ]),
     ('[Local] Python Unit tests',
-        "./py3_venv/bin/nosetests -v tests/python/unittest/"
+        "pytest -v tests/python/unittest/"
     ),
     ('[Docker] Build the MXNet binary - outputs to "lib/"',
         "ci/build.py --platform ubuntu_cpu_lite /work/runtime_functions.sh build_ubuntu_cpu_docs"),

ci/docker/Dockerfile.build.centos7

@@ -51,7 +51,7 @@ RUN yum -y check-update || true && \
         protobuf-devel \
         # CentOS Software Collections https://www.softwarecollections.org
         devtoolset-7 \
-        rh-python35 \
+        rh-python36 \
         rh-maven35 \
         # Libraries
         # Provide clbas headerfiles
@@ -71,7 +71,7 @@ RUN yum -y check-update || true && \
 
 # Make GCC7, Python 3.5 and Maven 3.3 Software Collections available by default
 # during build and runtime of this container
-SHELL [ "/usr/bin/scl", "enable", "devtoolset-7", "rh-python35", "rh-maven35" ]
+SHELL [ "/usr/bin/scl", "enable", "devtoolset-7", "rh-python36", "rh-maven35" ]
 
 # Install minimum required cmake version
 RUN cd /usr/local/src && \
@@ -93,7 +93,16 @@ RUN cd /usr/local/src && \
 
 # Python dependencies
 RUN pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir nose pylint cython numpy nose-timer requests h5py scipy==1.2.3 wheel
+    pip3 install --no-cache-dir pylint cython numpy requests h5py scipy==1.2.3 wheel \
+    pytest==5.3.5 \
+    pytest-env==0.6.2 \
+    pytest-cov==2.8.1 \
+    pytest-xdist==1.31.0 \
+    pytest-timeout==1.3.4 \
+    mock==2.0.0 \
+    onnx==1.5.0 \
+    protobuf==3.5.2 \
+    tabulate==0.7.5
 
 
 ARG USER_ID=0

ci/docker/Dockerfile.build.test.armv7

@@ -27,15 +27,24 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3-pip \
     python3-numpy \
     python3-scipy \
-    python3-nose \
-    python3-nose-timer \
     python3-requests \
  && rm -rf /var/lib/apt/lists/*
 
+
+# Python dependencies
+RUN pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir \
+    pytest==5.3.5 \
+    pytest-env==0.6.2 \
+    pytest-cov==2.8.1 \
+    pytest-xdist==1.31.0 \
+    pytest-timeout==1.3.4 \
+    mock==2.0.0
+
 ARG USER_ID=0
 ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.test.armv8

@@ -27,15 +27,22 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3-pip \
     python3-numpy \
     python3-scipy \
-    python3-nose \
-    python3-nose-timer \
     python3-requests \
  && rm -rf /var/lib/apt/lists/*
 
+RUN pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir \
+    pytest==5.3.5 \
+    pytest-env==0.6.2 \
+    pytest-cov==2.8.1 \
+    pytest-xdist==1.31.0 \
+    pytest-timeout==1.3.4 \
+    mock==2.0.0
+
 ARG USER_ID=0
 ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -54,12 +54,6 @@ RUN /work/ubuntu_gcc8.sh
 COPY install/ubuntu_mkl.sh /work/
 RUN /work/ubuntu_mkl.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_cpu_julia

@@ -54,12 +54,6 @@ RUN /work/ubuntu_gcc8.sh
 COPY install/ubuntu_mkl.sh /work/
 RUN /work/ubuntu_mkl.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_cpu_python

@@ -32,9 +32,6 @@ COPY install/ubuntu_python.sh /work/
 COPY install/requirements /work/
 RUN /work/ubuntu_python.sh
 
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_docs.sh /work/
 RUN /work/ubuntu_docs.sh
 
@@ -46,4 +43,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.ubuntu_cpu_scala

@@ -50,4 +50,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.ubuntu_gpu_cu101

@@ -51,12 +51,6 @@ RUN /work/ubuntu_tvm.sh
 COPY install/ubuntu_llvm.sh /work/
 RUN /work/ubuntu_llvm.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_docs.sh /work/
 COPY install/requirements /work/
 RUN /work/ubuntu_docs.sh

ci/docker/Dockerfile.build.ubuntu_nightly_cpu

@@ -45,12 +45,6 @@ RUN /work/ubuntu_clang.sh
 COPY install/ubuntu_gcc8.sh /work/
 RUN /work/ubuntu_gcc8.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_nightly_gpu

@@ -51,12 +51,6 @@ RUN /work/ubuntu_tvm.sh
 COPY install/ubuntu_llvm.sh /work/
 RUN /work/ubuntu_llvm.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/install/requirements

@@ -18,17 +18,36 @@
 # build and install are separated so changes to build don't invalidate
 # the whole docker cache for the image
 
-boto3==1.9.229
-cpplint==1.3.0
+# Required dependencies
+numpy>=1.17
+requests>=2.20.0,<3
+graphviz<0.9.0,>=0.8.1
+
+# Optional dependencies
+onnx==1.5.0
+# protobuf version frozen due to ps-lite
+protobuf==3.5.2
+scipy==1.4.1
+tabulate==0.7.5
 Cython==0.29.7
-decorator==4.4.0
-h5py==2.8.0rc1
-mock==2.0.0
-nose==1.3.7
-nose-timer==0.7.3
-numpy>1.16.0,<2.0.0
+
+# Development dependencies
+cpplint==1.3.0
 pylint==2.3.1; python_version >= '3.0'
-requests<2.19.0,>=2.18.4
-scipy==1.2.1
-six==1.11.0
+pytest==5.3.5
+pytest-env==0.6.2
+pytest-cov==2.8.1
+pytest-xdist==1.31.0
+pytest-timeout==1.3.4
 setuptools
+mock==2.0.0
+
+# TVM dependencies
+decorator==4.4.0
+
+# Used in examples
+boto3==1.9.229
+h5py==2.10.0
+# TODO(szha): remove once clean-up for py2 is complete
+six==1.11.0
+Pillow<6

ci/docker/install/ubuntu_python.sh

@@ -24,6 +24,7 @@ set -ex
 # install libraries for mxnet's python package on ubuntu
 apt-get update || true
 apt-get install -y python-dev python3-dev virtualenv wget
+apt-get install -y libprotobuf-dev protobuf-compiler
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py