Adds dynamic libmxnet to CD pipeline (#16163)

* Adds dynamic libmxnet release pipeline

* Updates default variants on CD pipeline

cd/Jenkinsfile_cd_pipeline

@@ -48,10 +48,16 @@ pipeline {
         script {
           cd_utils.error_checked_parallel([
 
-            "Static libmxnet based Release": {
+            "Static libmxnet based release": {
               stage("Build") {
                 cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)    
               }
+            },
+
+            "Dynamic libmxnet based release": {
+              stage("Build") {
+                cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)    
+              }
             }
 
           ])

cd/Jenkinsfile_release_job

@@ -37,8 +37,8 @@ pipeline {
   parameters {
     // Release parameters
     string(defaultValue: "Generic release job", description: "Optional Job name", name: "RELEASE_JOB_NAME")
-    choice(choices: ["mxnet_lib/static"], description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
-    string(defaultValue: "cpu,mkl,cu80,cu80mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
+    choice(choices: ["mxnet_lib/static", "mxnet_lib/dynamic"], description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
+    string(defaultValue: "cpu,mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl,cu101,cu101mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
     booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD")
   }
 

cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

@@ -0,0 +1,57 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// NOTE: ci_utils is loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job
+
+// libmxnet location
+libmxnet = 'lib/libmxnet.so'
+
+// licenses
+licenses = 'licenses/*'
+
+// libmxnet dependencies
+mx_deps = ''
+mx_mkldnn_deps = 'lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so'
+
+// library type
+// either static or dynamic - depending on how it links to its dependencies
+libtype = 'dynamic'
+
+libmxnet_pipeline = load('cd/mxnet_lib/mxnet_lib_pipeline.groovy')
+
+// Builds the dynamic binary for the specified mxnet variant
+def build(mxnet_variant) {
+  node(NODE_LINUX_CPU) {
+    ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") {
+      def image = libmxnet_pipeline.get_environment(mxnet_variant)
+      ci_utils.init_git()
+      ci_utils.docker_run(image, "build_dynamic_libmxnet ${mxnet_variant}", false)
+      ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant))
+    }
+  }
+}
+
+def get_pipeline(mxnet_variant) {
+  return libmxnet_pipeline.get_pipeline(mxnet_variant, this.&build)
+}
+
+return this

ci/docker/runtime_functions.sh

@@ -146,6 +146,107 @@ build_wheel() {
 # Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
 # function here with the same suffix:
 
+gather_licenses() {
+    mkdir -p licenses
+
+    cp tools/dependencies/LICENSE.binary.dependencies licenses/
+    cp NOTICE licenses/
+    cp LICENSE licenses/
+    cp DISCLAIMER licenses/
+}
+
+build_ubuntu_cpu_release() {
+    set -ex
+
+    build_ccache_wrappers
+
+    make  \
+        DEV=0                         \
+        ENABLE_TESTCOVERAGE=0         \
+        USE_CPP_PACKAGE=0             \
+        USE_MKLDNN=0                  \
+        USE_BLAS=openblas             \
+        USE_SIGNAL_HANDLER=1          \
+        -j$(nproc)
+}
+
+build_ubuntu_cpu_mkldnn_release() {
+    set -ex
+
+    build_ccache_wrappers
+
+    make  \
+        DEV=0                         \
+        ENABLE_TESTCOVERAGE=0         \
+        USE_CPP_PACKAGE=0             \
+        USE_MKLDNN=1                  \
+        USE_BLAS=openblas             \
+        USE_SIGNAL_HANDLER=1          \
+        -j$(nproc)
+}
+
+build_ubuntu_gpu_release() {
+    set -ex
+    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
+    # build_ccache_wrappers
+
+    make \
+        DEV=0                                     \
+        ENABLE_TESTCOVERAGE=0                     \
+        USE_BLAS=openblas                         \
+        USE_MKLDNN=0                              \
+        USE_CUDA=1                                \
+        USE_CUDA_PATH=/usr/local/cuda             \
+        USE_CUDNN=1                               \
+        USE_CPP_PACKAGE=0                         \
+        USE_DIST_KVSTORE=1                        \
+        USE_SIGNAL_HANDLER=1                      \
+        -j$(nproc)
+}
+
+build_ubuntu_gpu_mkldnn_release() {
+    set -ex
+    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
+    # build_ccache_wrappers
+
+    make \
+        DEV=0                                     \
+        ENABLE_TESTCOVERAGE=0                     \
+        USE_BLAS=openblas                         \
+        USE_MKLDNN=1                              \
+        USE_CUDA=1                                \
+        USE_CUDA_PATH=/usr/local/cuda             \
+        USE_CUDNN=1                               \
+        USE_CPP_PACKAGE=0                         \
+        USE_DIST_KVSTORE=1                        \
+        USE_SIGNAL_HANDLER=1                      \
+        -j$(nproc)
+}
+
+# Compiles the dynamic mxnet library
+# Parameters:
+# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, cu100, cu92mkl, etc.
+build_dynamic_libmxnet() {
+    set -ex
+
+    local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"}
+
+    # relevant licenses will be placed in the licenses directory
+    gather_licenses
+
+    if [[ ${mxnet_variant} = "cpu" ]]; then
+        build_ubuntu_cpu_release
+    elif [[ ${mxnet_variant} = "mkl" ]]; then
+        build_ubuntu_cpu_mkldnn_release
+    elif [[ ${mxnet_variant} =~ cu[0-9]+$ ]]; then
+        build_ubuntu_gpu_release
+    elif [[ ${mxnet_variant} =~ cu[0-9]+mkl$ ]]; then
+        build_ubuntu_gpu_mkldnn_release
+    else
+        echo "Error: Unrecognized mxnet variant '${mxnet_variant}'"
+    fi
+}
+
 build_jetson() {
     set -ex
     pushd .