Jenkins nightly maven with static build script and gpu (#13767)

.gitignore

@@ -146,6 +146,7 @@ tools/pip_package/mxnet
 
 # temporary path for building dependencies when building wheel
 ./deps/
+./staticdeps/
 bld
 ./tmp/*
 *.jar

ci/docker/Dockerfile.publish.ubuntu1604_cpu

@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib

ci/docker/Dockerfile.publish.ubuntu1604_gpu

@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib

ci/docker/install/ubuntu_base.sh

@@ -33,4 +33,8 @@ apt-get install -y \
     software-properties-common \
     sudo \
     unzip \
+    expect \
+    gnupg \
+    gnupg2 \
+    gnupg-agent \
     wget

ci/docker/install/ubuntu_scala.sh

@@ -36,9 +36,6 @@ apt-get install -y \
     openjdk-8-jdk \
     openjdk-8-jre \
     software-properties-common \
-    gnupg \
-    gnupg2 \
-    gnupg-agent \
     scala
 
 # Ubuntu 14.04

ci/publish/Jenkinsfile

@@ -21,7 +21,7 @@
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
 //mxnet libraries
-mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**'
+mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**'
 
 // timeout in minutes
 max_time = 120
@@ -34,8 +34,9 @@ node('restricted-utility') {
 utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
 
 // CPU and GPU. OSX nodes are not currently supported by Jenkins
-def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
 def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']
 
 def wrapStep(nodeToRun, workspaceName, step) {
   return {
@@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) {
 }
 
 def toBuild = [:]
-def labels = ['cpu'] // , 'gpu']
+def labels = ['cpu', 'gpu']
 for (x in labels) {
   def label = x // Required due to language
-  toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
-    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+  toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") {
+    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
       utils.init_git()
-      utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+      utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant')
       utils.pack_lib("scala_${label}", mx_scala_pub, false)
     }
   }
@@ -69,19 +70,21 @@ for (x in labels) {
   for (y in systems) {
     def system = y // Required due to language
     toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
-      utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
-      utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu')
+      withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) {
+        utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+        utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant')
+      }
     }
   }
 }
 
 def toDeploy = [:]
 for (x in labels) {
   def label = x // Required due to language
-  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
-    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") {
+    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
       utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
-      utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+      utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
     }
   }
 }
@@ -101,7 +104,7 @@ core_logic: {
 ,
 failure_handler: {
   if (currentBuild.result == "FAILURE") {
-    // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
   }
 }
 )

ci/publish/scala/build.sh

@@ -22,7 +22,9 @@ set -ex
 # MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
 # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
 
-bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+source tools/staticbuild/build.sh $mxnet_variant maven
+
+set -ex
 
 # Compile tests for discovery later
 cd scala-package

ci/publish/scala/deploy.sh

@@ -18,12 +18,10 @@
 
 set -ex
 
-# Setup Environment Variables
-# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
-# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-
-# Run python to configure keys
-python3 ci/publish/scala/buildkey.py
+# On Jenkins, run python script to configure keys
+if [[ $BUILD_ID ]]; then
+    python3 ci/publish/scala/buildkey.py
+fi
 
 # Updating cache
 mkdir -p ~/.gnupg
@@ -37,5 +35,7 @@ cd scala-package
 
 mvn -B deploy -Pnightly
 
-# Clear all password .xml files, exp files, and gpg key files
-rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+# On Jenkins, clear all password .xml files, exp files, and gpg key files
+if [[ $BUILD_ID ]]; then
+    rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+fi

ci/publish/scala/test.sh

@@ -24,5 +24,10 @@ fi
 
 # Test
 cd scala-package/packageTest
-# make testlocal CI=1
-make testsnapshot UNIT=1 CI=1
+
+if [[ $mxnet_variant == cu* ]]; then
+    export SCALA_TEST_ON_GPU=1
+    make testlocal USE_CUDA=1 CI=1
+else
+    make testlocal CI=1
+fi