diff --git a/cd/Jenkinsfile_cd_pipeline b/cd/Jenkinsfile_cd_pipeline index d7872204fcb7..4e4e6a7af3d4 100644 --- a/cd/Jenkinsfile_cd_pipeline +++ b/cd/Jenkinsfile_cd_pipeline @@ -48,10 +48,16 @@ pipeline { script { cd_utils.error_checked_parallel([ - "Static libmxnet based Release": { + "Static libmxnet based release": { stage("Build") { cd_utils.trigger_release_job("Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS) } + }, + + "Dynamic libmxnet based release": { + stage("Build") { + cd_utils.trigger_release_job("Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS) + } } ]) diff --git a/cd/Jenkinsfile_release_job b/cd/Jenkinsfile_release_job index 92ea72fca9c4..8c561a581f86 100644 --- a/cd/Jenkinsfile_release_job +++ b/cd/Jenkinsfile_release_job @@ -37,7 +37,7 @@ pipeline { parameters { // Release parameters string(defaultValue: "Generic release job", description: "Optional Job name", name: "RELEASE_JOB_NAME") - choice(choices: ["mxnet_lib/static"], description: "Pipeline to build", name: "RELEASE_JOB_TYPE") + choice(choices: ["mxnet_lib/static", "mxnet_lib/dynamic"], description: "Pipeline to build", name: "RELEASE_JOB_TYPE") string(defaultValue: "cpu,mkl,cu80,cu80mkl,cu90,cu90mkl,cu92,cu92mkl,cu100,cu100mkl", description: "Comma separated list of variants", name: "MXNET_VARIANTS") booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD") } diff --git a/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy b/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy new file mode 100644 index 000000000000..57812d2c3690 --- /dev/null +++ b/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy @@ -0,0 +1,57 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// NOTE: ci_utils is loaded by the originating Jenkins job, e.g. jenkins/Jenkinsfile_release_job + +// libmxnet location +libmxnet = 'lib/libmxnet.so' + +// licenses +licenses = 'licenses/*' + +// libmxnet dependencies +mx_deps = '' +mx_mkldnn_deps = 'lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so' + +// library type +// either static or dynamic - depending on how it links to its dependencies +libtype = 'dynamic' + +libmxnet_pipeline = load('cd/mxnet_lib/mxnet_lib_pipeline.groovy') + +// Builds the dynamic binary for the specified mxnet variant +def build(mxnet_variant) { + node(NODE_LINUX_CPU) { + ws("workspace/mxnet_${libtype}/${mxnet_variant}/${env.BUILD_NUMBER}") { + def image = libmxnet_pipeline.get_environment(mxnet_variant) + ci_utils.init_git() + ci_utils.docker_run(image, "build_dynamic_libmxnet ${mxnet_variant}", false) + ci_utils.pack_lib("mxnet_${mxnet_variant}", libmxnet_pipeline.get_stash(mxnet_variant)) + } + } +} + +def get_pipeline(mxnet_variant) { + return libmxnet_pipeline.get_pipeline(mxnet_variant, this.&build) +} + +return this diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 67cc98c73620..de753a56be7f 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -146,6 +146,107 @@ build_wheel() { # Build commands: Every platform in docker/Dockerfile.build. should have a corresponding # function here with the same suffix: +gather_licenses() { + mkdir -p licenses + + cp tools/dependencies/LICENSE.binary.dependencies licenses/ + cp NOTICE licenses/ + cp LICENSE licenses/ + cp DISCLAIMER licenses/ +} + +build_ubuntu_cpu_release() { + set -ex + + build_ccache_wrappers + + make \ + DEV=0 \ + ENABLE_TESTCOVERAGE=0 \ + USE_CPP_PACKAGE=0 \ + USE_MKLDNN=0 \ + USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ + -j$(nproc) +} + +build_ubuntu_cpu_mkldnn_release() { + set -ex + + build_ccache_wrappers + + make \ + DEV=0 \ + ENABLE_TESTCOVERAGE=0 \ + USE_CPP_PACKAGE=0 \ + USE_MKLDNN=1 \ + USE_BLAS=openblas \ + USE_SIGNAL_HANDLER=1 \ + -j$(nproc) +} + +build_ubuntu_gpu_release() { + set -ex + # unfortunately this build has problems in 3rdparty dependencies with ccache and make + # build_ccache_wrappers + + make \ + DEV=0 \ + ENABLE_TESTCOVERAGE=0 \ + USE_BLAS=openblas \ + USE_MKLDNN=0 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=0 \ + USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ + -j$(nproc) +} + +build_ubuntu_gpu_mkldnn_release() { + set -ex + # unfortunately this build has problems in 3rdparty dependencies with ccache and make + # build_ccache_wrappers + + make \ + DEV=0 \ + ENABLE_TESTCOVERAGE=0 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=0 \ + USE_DIST_KVSTORE=1 \ + USE_SIGNAL_HANDLER=1 \ + -j$(nproc) +} + +# Compiles the dynamic mxnet library +# Parameters: +# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, cu100, cu92mkl, etc. +build_dynamic_libmxnet() { + set -ex + + local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"} + + # relevant licenses will be placed in the licenses directory + gather_licenses + + if [[ ${mxnet_variant} = "cpu" ]]; then + build_ubuntu_cpu_release + elif [[ ${mxnet_variant} = "mkl" ]]; then + build_ubuntu_cpu_mkldnn_release + elif [[ ${mxnet_variant} =~ cu[0-9]+$ ]]; then + build_ubuntu_gpu_release + elif [[ ${mxnet_variant} =~ cu[0-9]+mkl$ ]]; then + build_ubuntu_gpu_mkldnn_release + else + echo "Error: Unrecognized mxnet variant '${mxnet_variant}'" + fi +} + build_jetson() { set -ex pushd .