Jenkins nightly maven with static build script and gpu

.gitignore

@@ -146,6 +146,7 @@ tools/pip_package/mxnet
 
 # temporary path for building dependencies when building wheel
 ./deps/
+./staticdeps/
 bld
 ./tmp/*
 *.jar

Makefile

@@ -667,14 +667,14 @@ scalarelease-perform:
 
 scaladeploy:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		mvn deploy $(MAVEN_ARGS) -Pdeploy,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scaladeploylocal:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		mvn deploy $(MAVEN_ARGS) -Pdeploy,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
 		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
 		  -Dgpg.skip \
 		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \

ci/Jenkinsfile_utils.groovy

@@ -18,25 +18,18 @@
 // under the License.
 
 // initialize source codes
-// Clean - 0: None, 1: xdf, 2: xdff
-def init_git(clean=2) {
+def init_git() {
   deleteDir()
   retry(5) {
     try {
       // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
       // retries as this will increase the amount of requests and worsen the throttling
       timeout(time: 15, unit: 'MINUTES') {
         checkout scm
-        if(clean == 0)
-          cleanCmd = 'git status'
-        else if(clean == 1)
-          cleanCmd = 'git clean -xdf'
-        else if(clean == 2)
-          cleanCmd = 'git clean -xdff'
-        sh cleanCmd
+        sh 'git clean -xdff'
         sh 'git reset --hard'
         sh 'git submodule update --init --recursive'
-        sh 'git submodule foreach --recursive ' + cleanCmd
+        sh 'git submodule foreach --recursive git clean -ffxd'
         sh 'git submodule foreach --recursive git reset --hard'
       }
     } catch (exc) {
@@ -87,8 +80,8 @@ return 0
 }
 
 // unpack libraries saved before
-def unpack_and_init(name, libs, include_gcov_data = false, clean = 2) {
-  init_git(clean)
+def unpack_and_init(name, libs, include_gcov_data = false) {
+  init_git()
   unstash name
   sh returnStatus: true, script: """
 set +e

ci/docker/Dockerfile.publish.test.ubuntu1404_gpu

@@ -18,7 +18,7 @@
 #
 # Dockerfile to run MXNet on Ubuntu 14.04 for GPU
 
-Use CPU with setup_gpu script instead
+# Use CPU with setup_gpu script instead
 # FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu14.04
 FROM ubuntu:14.04
 

ci/docker/Dockerfile.publish.ubuntu1604_cpu

@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib

ci/docker/Dockerfile.publish.ubuntu1604_gpu

@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib

ci/docker/install/ubuntu_base.sh

@@ -33,4 +33,8 @@ apt-get install -y \
     software-properties-common \
     sudo \
     unzip \
+    expect \
+    gnupg \
+    gnupg2 \
+    gnupg-agent \
     wget

ci/docker/install/ubuntu_scala.sh

@@ -36,9 +36,6 @@ apt-get install -y \
     openjdk-8-jdk \
     openjdk-8-jre \
     software-properties-common \
-    gnupg \
-    gnupg2 \
-    gnupg-agent \
     scala
 
 # Ubuntu 14.04

ci/publish/Jenkinsfile

@@ -21,7 +21,7 @@
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
 //mxnet libraries
-mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**'
+mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**'
 
 // timeout in minutes
 max_time = 120
@@ -34,8 +34,9 @@ node('restricted-utility') {
 utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
 
 // CPU and GPU. OSX nodes are not currently supported by Jenkins
-def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
 def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']
 
 def wrapStep(nodeToRun, workspaceName, step) {
   return {
@@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) {
 }
 
 def toBuild = [:]
-def labels = ['cpu'] // , 'gpu']
+def labels = ['cpu', 'gpu']
 for (x in labels) {
   def label = x // Required due to language
-  toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
-    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
-      utils.init_git(0)
-      utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+  toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") {
+    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
+      utils.init_git()
+      utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant')
       utils.pack_lib("scala_${label}", mx_scala_pub, false)
     }
   }
@@ -69,19 +70,21 @@ for (x in labels) {
   for (y in systems) {
     def system = y // Required due to language
     toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
-      utils.unpack_and_init("scala_${label}", mx_scala_pub, false, 0)
-      utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu')
+      withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) {
+        utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+        utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant')
+      }
     }
   }
 }
 
 def toDeploy = [:]
 for (x in labels) {
   def label = x // Required due to language
-  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
-    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
-      utils.unpack_and_init("scala_${label}", mx_scala_pub, false, 0)
-      utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") {
+    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
+      utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+      utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
     }
   }
 }
@@ -101,7 +104,7 @@ core_logic: {
 ,
 failure_handler: {
   if (currentBuild.result == "FAILURE") {
-    // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
   }
 }
 )

ci/publish/scala/build.sh

@@ -22,12 +22,12 @@ set -ex
 # MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
 # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
 
-bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+source tools/staticbuild/build.sh $mxnet_variant maven
+
+set -ex
 
 # Scala steps to deploy
 make scalapkg CI=1
-make scalaunittest CI=1
-make scalaintegrationtest CI=1
 
 # Compile tests for discovery later
 export GPG_TTY=$(tty)

ci/publish/scala/buildkey.py

@@ -76,22 +76,36 @@ def importASC(key, gpgPassphrase):
     filename = os.path.join(KEY_PATH, "key.asc")
     with open(filename, 'w') as f:
         f.write(key)
-    subprocess.run(['gpg2', '--batch', '--yes',
+    subprocess.check_output(['gpg2', '--batch', '--yes',
                     '--passphrase-fd', '0',
                     "--import", "{}".format(filename)],
                    input=str.encode(gpgPassphrase))
 
 
 def encryptMasterPSW(password):
-    result = subprocess.run(['mvn', '--encrypt-master-password'],
-                            stdout=subprocess.PIPE, input=str.encode(password))
-    return str(result.stdout)[2:-3]
+    filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp")
+    with open(filename, 'w') as f:
+        f.write('''
+        spawn mvn --encrypt-master-password
+        expect -exact "Master password: "
+        send -- "{}\r"
+        expect eof
+        '''.format(password))
+    result = subprocess.check_output(['expect', filename])
+    return str(result).split('\r\n')[-1][2:-3]
 
 
 def encryptPSW(password):
-    result = subprocess.run(['mvn', '--encrypt-password'],
-                            stdout=subprocess.PIPE, input=str.encode(password))
-    return str(result.stdout)[2:-3]
+    filename = os.path.join(KEY_PATH, "encryptPassword.exp")
+    with open(filename, 'w') as f:
+        f.write('''
+        spawn mvn --encrypt-password
+        expect -exact "Password: "
+        send -- "{}\r"
+        expect eof
+        '''.format(password))
+    result = subprocess.check_output(['expect', filename])
+    return str(result).split('\r\n')[-1][2:-3]
 
 
 def masterPSW(password):

ci/publish/scala/deploy.sh

@@ -18,22 +18,11 @@
 
 set -ex
 
-# Setup Environment Variables
-# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
-# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-
-
-if [[ $MAVEN_PUBLISH_OS_TYPE == "linux-x86_64-cpu" ]];
-then
-    MAKE_FLAGS="USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1"
-elif [[ $MAVEN_PUBLISH_OS_TYPE == "linux-x86_64-gpu" ]]
-then
-    MAKE_FLAGS="USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_ON_GPU=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1"
+# On Jenkins, run python script to configure keys
+if [[ $BUILD_ID ]]; then
+    python3 ci/publish/scala/buildkey.py
 fi
 
-# Run python to configure keys
-python3 ci/publish/scala/buildkey.py
-
 # Updating cache
 mkdir -p ~/.gnupg
 echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf
@@ -49,5 +38,7 @@ cd ..
 # echo "\n\n$VERSION\n" | make scalarelease-dryrun $MAKE_FLAGS
 make scaladeploy $MAKE_FLAGS CI=1
 
-# Clear all password .xml files, gpg key files, and all imported gpg secret keys
-rm -rf ~/.m2/*.xml ~/.m2/key.asc
+# On Jenkins, clear all password .xml files, exp files, and gpg key files
+if [[ $BUILD_ID ]]; then
+    rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+fi

ci/publish/scala/engine.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# Setup Environment Variables
+# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
+# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
+
+# bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+
+mxnet_variant=$1
+
+source tools/staticbuild/build.sh $mxnet_variant maven

ci/publish/scala/test.sh

@@ -24,5 +24,10 @@ fi
 
 # Test
 cd scala-package/packageTest
-# make testlocal CI=1
-make testsnapshot UNIT=1 CI=1
+
+if [[ $mxnet_variant == cu* ]]; then
+    export SCALA_TEST_ON_GPU=1
+    make testlocal USE_CUDA=1 CI=1
+else
+    make testlocal CI=1
+fi