Merge branch 'upstream' into onnx_add_deconvolution_crop

CMakeLists.txt

@@ -224,7 +224,7 @@ if(ENABLE_TESTCOVERAGE)
 endif()
 
 if(USE_MKLDNN)
-  include(cmake/MklDnn.cmake)
+  include(cmake/DownloadMKLML.cmake)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)
     set(ARCH_OPT_FLAGS "-mtune=generic")

CONTRIBUTORS.md

@@ -190,10 +190,18 @@ List of Contributors
 * [Denisa Roberts](https://github.com/D-Roberts)
 * [Dick Carter](https://github.com/DickJC123)
 * [Rahul Padmanabhan](https://github.com/rahul3)
+* [Yuxi Hu](https://github.com/yuxihu)
+* [Harsh Patel](https://github.com/harshp8l)
 
 Label Bot
 ---------
 * [mxnet-label-bot](https://github.com/mxnet-label-bot)
-  - mxnet-label-bot provides users with the functionality to add labels from the repository to issues
-  - To use me, comment: @mxnet-label-bot add [specify label here]
+  - mxnet-label-bot provides users with the functionality to manage labels for Issues/Pull Requests on the repository
+  - To use me, comment:
+    - @mxnet-label-bot add [specify comma separated labels here]
+    - @mxnet-label-bot remove [specify comma separated labels here]
+    - @mxnet-label-bot update [specify comma separated labels here]  
+      (i.e. @mxnet-label-bot update [Bug, Python])
+
+  - Available label names which are supported: [Labels](https://github.com/apache/incubator-mxnet/labels)
   - For further details: [My Wiki Page](https://cwiki.apache.org/confluence/display/MXNET/Machine+Learning+Based+GitHub+Bot)

Jenkinsfile

@@ -22,6 +22,10 @@
 
 // mxnet libraries
 mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+
+// Python wheels
+mx_pip = 'build/*.whl'
+
 // for scala build, need to pass extra libs when run with dist_kvstore
 mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
@@ -148,7 +152,7 @@ core_logic: {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
-            utils.pack_lib('centos7_cpu', mx_lib, true)
+            utils.pack_lib('centos7_cpu', mx_dist_lib, true)
           }
         }
       }
@@ -414,6 +418,7 @@ core_logic: {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('armv7', 'build_armv7', false)
+            utils.pack_lib('armv7', mx_pip)
           }
         }
       }
@@ -693,6 +698,17 @@ core_logic: {
         }
       }
     },
+    'Scala: CentOS CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-scala-centos7-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
+            utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
+            utils.publish_test_coverage()
+          }
+        }
+      }
+    },
     'Clojure: CPU': {
       node(NODE_LINUX_CPU) {
         ws('workspace/ut-clojure-cpu') {
@@ -941,6 +957,16 @@ core_logic: {
           }
         }
       }
+    },
+    'ARMv7 QEMU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-armv7-qemu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('armv7', mx_pip)
+            sh "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
+          }
+        }
+      }
     }
   }
 

Makefile

@@ -597,7 +597,7 @@ rpkg:
 
 rpkgtest:
 	Rscript -e 'require(testthat);res<-test_dir("R-package/tests/testthat");if(!testthat:::all_passed(res)){stop("Test failures", call. = FALSE)}'
-	Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file("r-package_coverage.json");writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'
+	Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'
 
 scalaclean:
 	(cd $(ROOTDIR)/scala-package; \

ci/README.md

@@ -5,6 +5,8 @@ Docker containers
 
 You need docker and nvidia docker if you have a GPU.
 
+Also you need to run `pip3 install docker` as it uses the [docker python module](https://docker-py.readthedocs.io/en/stable/containers.html#)
+
 If you are in ubuntu an easy way to install Docker CE is executing the
 following script:
 

ci/build.py

@@ -101,6 +101,8 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
 
 def get_docker_tag(platform: str, registry: str) -> str:
     """:return: docker tag to be used for the container"""
+    if not registry:
+        registry = "mxnet_local"
     return "{0}/build.{1}".format(registry, platform)
 
 
@@ -112,14 +114,14 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
     return "nvidia-docker" if use_nvidia_docker else "docker"
 
 
-def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, use_cache: bool) -> str:
+def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str:
     """
     Build a container for the given platform
     :param platform: Platform
     :param docker_binary: docker binary to use (docker/nvidia-docker)
     :param registry: Dockerhub registry name
     :param num_retries: Number of retries to build the docker image
-    :param use_cache: will pass cache_from to docker to use the previously pulled tag
+    :param no_cache: pass no-cache to docker to rebuild the images
     :return: Id of the top level image
     """
     tag = get_docker_tag(platform=platform, registry=registry)
@@ -144,7 +146,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
            "-f", get_dockerfile(platform),
            "--build-arg", "USER_ID={}".format(os.getuid()),
            "--build-arg", "GROUP_ID={}".format(os.getgid())]
-    if use_cache:
+    if no_cache:
+        cmd.append("--no-cache")
+    elif registry:
         cmd.extend(["--cache-from", tag])
     cmd.extend(["-t", tag, get_dockerfiles_path()])
 
@@ -422,7 +426,7 @@ def main() -> int:
                         action='store_true')
 
     parser.add_argument("-d", "--docker-registry",
-                        help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'",
+                        help="Dockerhub registry name to retrieve cache from.",
                         default='mxnetci',
                         type=str)
 
@@ -431,10 +435,8 @@ def main() -> int:
                         default=1,
                         type=int)
 
-    parser.add_argument("-c", "--no-dockerhub-cache", action="store_true",
-                        help="Disables use of --cache-from option on docker build, allowing docker"
-                        " to use local layers for caching. If absent, we use the cache from dockerhub"
-                        " which is the default.")
+    parser.add_argument("--no-cache", action="store_true",
+                        help="passes --no-cache to docker build")
 
     parser.add_argument("command",
                         help="command to run in the container",
@@ -447,9 +449,6 @@ def main() -> int:
 
     args = parser.parse_args()
 
-    def use_cache():
-        return not args.no_dockerhub_cache or under_ci()
-
     command = list(chain(*args.command))
     docker_binary = get_docker_binary(args.nvidiadocker)
 
@@ -472,10 +471,10 @@ def signal_handler(signum, _):
     elif args.platform:
         platform = args.platform
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-        if use_cache():
+        if args.docker_registry:
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
         build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry,
-                     num_retries=args.docker_build_retries, use_cache=use_cache())
+                     num_retries=args.docker_build_retries, no_cache=args.no_cache)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
@@ -512,10 +511,9 @@ def signal_handler(signum, _):
         logging.info("Artifacts will be produced in the build/ directory.")
         for platform in platforms:
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-            if use_cache():
-                load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
             build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry,
-                         num_retries=args.docker_build_retries, use_cache=use_cache())
+                         num_retries=args.docker_build_retries, no_cache=args.no_cache)
             if args.build_only:
                 continue
             shutil.rmtree(buildir(), ignore_errors=True)

ci/docker/Dockerfile.build.centos7_cpu

@@ -28,6 +28,8 @@ COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/
 RUN /work/centos7_python.sh
+COPY install/centos7_scala.sh /work/
+RUN /work/centos7_scala.sh
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
 

ci/docker/install/centos7_scala.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+yum install -y java-1.8.0-openjdk-devel
+# Build from source with Maven
+wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+tar xzf apache-maven-3.3.9-bin.tar.gz
+mkdir /usr/local/maven
+mv apache-maven-3.3.9/ /usr/local/maven/
+alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1

ci/docker/install/ubuntu_caffe.sh

@@ -18,6 +18,7 @@
 # under the License.
 set -ex
 
+apt-get update || true
 apt-get install -y \
     libgflags-dev \
     libgoogle-glog-dev \

ci/docker/install/ubuntu_clang.sh

@@ -21,6 +21,8 @@
 # the whole docker cache for the image
 
 set -ex
+
+apt-get update || true
 # Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major release)
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \

ci/docker/install/ubuntu_core.sh

@@ -21,7 +21,7 @@
 # the whole docker cache for the image
 
 set -ex
-apt-get update
+apt-get update || true
 apt-get install -y \
     apt-transport-https \
     build-essential \

ci/docker/install/ubuntu_docs.sh

@@ -23,6 +23,7 @@
 set -ex
 # Install dependencies
 echo 'Installing dependencies...'
+apt-get update || true
 apt-get install -y \
     doxygen \
     pandoc

ci/docker/install/ubuntu_emscripten.sh

@@ -25,6 +25,7 @@
 
 set -ex
 
+apt-get update || true
 apt-get -y install nodejs
 
 git clone -b 1.38.6 https://github.com/kripken/emscripten.git

ci/docker/install/ubuntu_gcc8.sh

@@ -19,5 +19,5 @@
 
 sudo add-apt-repository ppa:jonathonf/gcc-8.0
 sudo add-apt-repository ppa:jonathonf/gcc-7.3
-sudo apt-get update
+sudo apt-get update || true
 sudo apt-get install -y gcc-8 g++-8

ci/docker/install/ubuntu_llvm.sh

@@ -23,4 +23,5 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+apt-get update || true
+apt-get install -y --force-yes llvm-5.0

ci/docker/install/ubuntu_mklml.sh

@@ -21,5 +21,5 @@
 # the whole docker cache for the image
 
 set -ex
-wget -q --no-check-certificate -O /tmp/mklml.tgz https://github.com/intel/mkl-dnn/releases/download/v0.14/mklml_lnx_2018.0.3.20180406.tgz
+wget -q --no-check-certificate -O /tmp/mklml.tgz https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz
 tar -zxf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*

ci/docker/install/ubuntu_nightly_tests.sh

@@ -25,7 +25,7 @@ set -ex
 # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays.
 for i in 1 2 3 4 5; do add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || sleep 300; done
 
-apt-get update
+apt-get update || true
 apt-get -y install time
 
 # Install for RAT License Check Nightly Test

ci/docker/install/ubuntu_npm_blc.sh

@@ -22,7 +22,7 @@
 
 set -ex
 echo 'Installing npm...'
-apt-get update
+apt-get update || true
 apt-get install -y npm
 
 echo "Obtaining NodeJS version 8.x"

ci/docker/install/ubuntu_nvidia.sh

@@ -18,6 +18,7 @@
 # under the License.
 
 set -ex
+apt-get update || true
 apt install -y software-properties-common
 
 # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays.

ci/docker/install/ubuntu_onnx.sh

@@ -27,6 +27,7 @@ set -e
 set -x
 
 echo "Installing libprotobuf-dev and protobuf-compiler ..."
+apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
 echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX and tabulate ..."

ci/docker/install/ubuntu_perl.sh

@@ -22,5 +22,6 @@
 
 set -ex
 # install libraries for mxnet's perl package on ubuntu
+apt-get update || true
 apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
 cpanm -q Function::Parameters Hash::Ordered PDL::CCS

ci/docker/install/ubuntu_python.sh

@@ -22,6 +22,7 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
+apt-get update || true
 apt-get install -y python-dev python3-dev virtualenv wget
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here

ci/docker/install/ubuntu_r.sh

@@ -34,7 +34,7 @@ apt-key add r.gpg
 # Installing the latest version (3.3+) that is compatible with MXNet
 add-apt-repository 'deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu xenial/'
 
-apt-get update
+apt-get update || true
 apt-get install -y --allow-unauthenticated \
     libcairo2-dev \
     libssl-dev \

ci/docker/install/ubuntu_rat.sh

@@ -20,7 +20,7 @@
 set -ex
 
 echo "Install dependencies"
-apt-get update
+apt-get update || true
 apt-get install -y subversion maven openjdk-8-jdk openjdk-8-jre
 
 echo "download RAT"

ci/docker/install/ubuntu_scala.sh

@@ -24,17 +24,13 @@ set -ex
 cd "$(dirname "$0")"
 # install libraries for mxnet's scala package on ubuntu
 echo 'Installing Scala...'
+apt-get update || true
 apt-get install -y software-properties-common
-apt-get update
+apt-get update || true
 apt-get install -y openjdk-8-jdk
 apt-get install -y openjdk-8-jre
 
-echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
-# ubuntu keyserver is very flaky
-#apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
-#apt-key adv --keyserver keys.gnupg.net --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
-apt-key add sbt.gpg
-apt-get update && apt-get install -y \
+apt-get update || true
+apt-get install -y \
     maven \
-    sbt \
     scala