Seperate Build, Test, and Deploy Stages with parallel

Makefile

@@ -454,6 +454,10 @@ else
 	CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0
 endif
 
+ifeq ($(CI), 1)
+	MAVEN_ARGS := -B
+endif
+
 # For quick compile test, used smaller subset
 ALLX_DEP= $(ALL_DEP)
 
@@ -618,27 +622,27 @@ scalatestcompile:
 
 scalapkg:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
+		mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dcurrent_libdir="$(ROOTDIR)/lib" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scalaunittest:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
+		mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalaintegrationtest:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
+		mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
 
 scalainstall:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
+		mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
@@ -663,14 +667,14 @@ scalarelease-perform:
 
 scaladeploy:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
 		    -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
 scaladeploylocal:
 	(cd $(ROOTDIR)/scala-package && \
-		mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
 		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
 		  -Dgpg.skip \
 		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \

ci/Jenkinsfile_utils.groovy

@@ -64,7 +64,7 @@ def init_git_win() {
 
 // pack libraries for later use
 def pack_lib(name, libs, include_gcov_data = false) {
-  sh """
+  sh returnStatus: true, script: """
 set +e
 echo "Packing ${libs} into ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
@@ -83,7 +83,7 @@ return 0
 def unpack_and_init(name, libs, include_gcov_data = false) {
   init_git()
   unstash name
-  sh """
+  sh returnStatus: true, script: """
 set +e
 echo "Unpacked ${libs} from ${name}"
 echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
@@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) {
 }
 
 
-def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
-  def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") {
+  def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+  command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
   command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
   command = command.replaceAll('%PLATFORM%', platform)
   command = command.replaceAll('%FUNCTION_NAME%', function_name)

ci/build.py

@@ -215,20 +215,21 @@ def container_run(platform: str,
                   local_ccache_dir: str,
                   command: List[str],
                   cleanup: Cleanup,
+                  environment: Dict[str, str],
                   dry_run: bool = False) -> int:
     """Run command in a container"""
     container_wait_s = 600
     #
     # Environment setup
     #
-    environment = {
+    environment.update({
         'CCACHE_MAXSIZE': '500G',
         'CCACHE_TEMPDIR': '/tmp/ccache',  # temp dir should be local and not shared
         'CCACHE_DIR': '/work/ccache',  # this path is inside the container as /work/ccache is
                                        # mounted
         'CCACHE_LOGFILE': '/tmp/ccache.log',  # a container-scoped log, useful for ccache
                                               # verification.
-    }
+    })
     # These variables are passed to the container to the process tree killer can find runaway
     # process inside the container
     # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
@@ -446,6 +447,10 @@ def main() -> int:
     parser.add_argument("--no-cache", action="store_true",
                         help="passes --no-cache to docker build")
 
+    parser.add_argument("-e", "--environment", nargs="*", default=[],
+                        help="Environment variables for the docker container. "
+                        "Specify with a list containing either names or name=value")
+
     parser.add_argument("command",
                         help="command to run in the container",
                         nargs='*', action='append', type=str)
@@ -474,6 +479,9 @@ def signal_handler(signum, _):
     signal.signal(signal.SIGTERM, signal_handler)
     signal.signal(signal.SIGINT, signal_handler)
 
+    environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
+                        for e in args.environment])
+
     if args.list:
         print(list_platforms())
     elif args.platform:
@@ -493,21 +501,21 @@ def signal_handler(signum, _):
             ret = container_run(
                 platform=platform, nvidia_runtime=args.nvidiadocker,
                 shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
-                local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+                local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
         elif args.print_docker_run:
             command = []
             ret = container_run(
                 platform=platform, nvidia_runtime=args.nvidiadocker,
                 shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
-                local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup)
+                local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment)
         else:
             # With no commands, execute a build function for the target platform
             command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
             logging.info("No command specified, trying default build: %s", ' '.join(command))
             ret = container_run(
                 platform=platform, nvidia_runtime=args.nvidiadocker,
                 shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
-                local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+                local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
 
         if ret != 0:
             logging.critical("Execution of %s failed with status: %d", command, ret)
@@ -535,7 +543,7 @@ def signal_handler(signum, _):
             container_run(
                 platform=platform, nvidia_runtime=args.nvidiadocker,
                 shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
-                local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+                local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
             shutil.move(buildir(), plat_buildir)
             logging.info("Built files left in: %s", plat_buildir)
 

ci/docker/install/ubuntu_scala.sh

@@ -33,4 +33,7 @@ apt-get install -y openjdk-8-jre
 apt-get update || true
 apt-get install -y \
     maven \
+    gnupg \
+    gnupg2 \
+    gnupg-agent \
     scala

ci/docker/runtime_functions.sh

@@ -36,6 +36,11 @@ clean_repo() {
     git submodule update --init --recursive
 }
 
+scala_prepare() {
+    # Clean up maven logs
+    export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+}
+
 build_ccache_wrappers() {
     set -ex
 
@@ -835,21 +840,24 @@ unittest_ubuntu_python3_quantization_gpu() {
 
 unittest_ubuntu_cpu_scala() {
     set -ex
-    make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
-    make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    scala_prepare
+    make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+    make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
 }
 
 unittest_centos7_cpu_scala() {
     set -ex
     cd /work/mxnet
+    scala_prepare
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }
 
 unittest_ubuntu_cpu_clojure() {
     set -ex
-    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
-    make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    scala_prepare
+    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+    make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
     ./contrib/clojure-package/ci-test.sh
 }
 
@@ -993,8 +1001,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
-    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
-    make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    scala_prepare
+    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
+    make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
 }
 
 integrationtest_ubuntu_gpu_dist_kvstore() {
@@ -1246,14 +1255,30 @@ deploy_jl_docs() {
     # ...
 }
 
-deploy_nightly_maven() {
+publish_scala_build() {
     set -ex
     pushd .
-    cd /work/mxnet
+    scala_prepare
     ./scala-package/dev/build.sh
     popd
 }
 
+publish_scala_test() {
+    set -ex
+    pushd .
+    scala_prepare
+    ./scala-package/dev/test.sh
+    popd
+}
+
+publish_scala_deploy() {
+    set -ex
+    pushd .
+    scala_prepare
+    ./scala-package/dev/deploy.sh
+    popd
+}
+
 # broken_link_checker
 
 broken_link_checker() {

ci/docker_cache.py

@@ -211,8 +211,7 @@ def _get_dockerhub_credentials():  # pragma: no cover
             logging.exception("The request was invalid due to:")
         elif client_error.response['Error']['Code'] == 'InvalidParameterException':
             logging.exception("The request had invalid params:")
-        else:
-            raise
+        raise
     else:
         secret = get_secret_value_response['SecretString']
         secret_dict = json.loads(secret)

ci/publish/Jenkinsfile

@@ -20,58 +20,86 @@
 // Jenkins pipeline
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
+//mxnet libraries
+mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**'
+
 // timeout in minutes
 max_time = 120
 
-node('restricted-mxnetlinux-cpu') {
+node('restricted-utility') {
   // Loading the utilities requires a node context unfortunately
   checkout scm
   utils = load('ci/Jenkinsfile_utils.groovy')
 }
-utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
+utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
 
-utils.main_wrapper(
-core_logic: {
-  stage('Deploy Nightly Maven') {
-    environment {
-      MVN_DEPLOY_USER = credentials('mvn-deploy-user')
-      MVN_DEPLOY_PASSWORD = credentials('mvn-deploy-password')
-      MVN_DEPLOY_PASSPHRASE = credentials('mvn-deploy-passphrase')
-      MVN_DEPLOY_MASTERPASS = credentials('mvn-deploy-masterpass')
-      MVN_DEPLOY_GPG_KEY = credentials('mvn-deploy-gpg-key')
-    }
-    parallel 'linux-cpu': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/linux-cpu') {
-          environment {
-            MVN_DEPLOY_OS_TYPE = linux-x86_64-cpu
-          }
-          utils.init_git()
-          timeout(time: max_time, unit: 'MINUTES') {
-              sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
-          }
-        }
-      }
-    },
-    'linux-gpu': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/linux-gpu') {
-          environment {
-            MVN_DEPLOY_OS_TYPE = linux-x86_64-gpu
-          }
-          utils.init_git()
-          timeout(time: max_time, unit: 'MINUTES') {
-              sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
-          }
+// CPU and GPU. OSX nodes are not currently supported by Jenkins
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+
+def wrapStep(nodeToRun, workspaceName, step) {
+  return {
+    node(nodeToRun) {
+      ws("workspace/${workspaceName}") {
+        timeout(time: max_time, unit: 'MINUTES') {
+          step()
         }
       }
     }
   }
 }
+
+def toBuild = [:]
+def labels = ['cpu'] // , 'gpu']
+for (x in labels) {
+  def label = x // Required due to language
+  toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
+    env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
+    utils.init_git()
+    utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+    utils.pack_lib("scala_${label}", mx_scala_pub, false)
+  }
+}
+
+def toTest = [:]
+def systems = ['ubuntu'] // , 'centos7']
+for (x in labels) {
+  def label = x // Required due to language
+  for (y in systems) {
+    def system = y // Required due to language
+    toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
+      utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+      utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu')
+    }
+  }
+}
+
+def toDeploy = [:]
+for (x in labels) {
+  def label = x // Required due to language
+  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
+    env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
+    utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+    utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+  }
+}
+
+utils.main_wrapper(
+core_logic: {
+  stage('Build Packages') {
+    parallel toBuild
+  }
+  stage('Test Packages') {
+    parallel toTest
+  }
+  stage('Deploy Packages') {
+    parallel toDeploy
+  }
+}
 ,
 failure_handler: {
   if (currentBuild.result == "FAILURE") {
-    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+    // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
   }
 }
 )