From 52a26945d1fe539b36b057f422f4464fb81b641f Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Thu, 2 Aug 2018 14:51:03 +0200 Subject: [PATCH 1/7] Rework Jenkinsfile --- Jenkinsfile | 184 +++++++++++++++++++++++++++++----------------------- 1 file changed, 104 insertions(+), 80 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6d21f496426e..92fddb1f093e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,10 +168,46 @@ def python3_gpu_ut_nocudnn(docker_container_name) { } } -try { +NODE_LINUX_CPU = 'mxnetlinux-cpu' +NODE_LINUX_GPU = 'mxnetlinux-gpu' +NODE_LINUX_GPU_P3 = 'mxnetlinux-gpu-p3' +NODE_WINDOWS_CPU = 'mxnetwindows-cpu' +NODE_WINDOWS_GPU = 'mxnetwindows-gpu' + +def main_wrapper(args) { + // hander: Core logic + // failure_handler: Failure handler + + try { + // Call actual handler + args['handler']() + + // set build status to success at the end + currentBuild.result = "SUCCESS" + } catch (caughtError) { + node(NODE_LINUX_CPU) { + sh "echo caught ${caughtError}" + err = caughtError + currentBuild.result = "FAILURE" + } + } finally { + node(NODE_LINUX_CPU) { + // Call failure handler + args['failure_handler']() + + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } + } +} + +main_wrapper( +handler: { stage('Sanity Check') { parallel 'Lint': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/sanity-lint') { init_git() docker_run('ubuntu_cpu', 'sanity_check', false) @@ -179,7 +215,7 @@ try { } }, 'RAT License': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/sanity-rat') { init_git() docker_run('ubuntu_rat', 'nightly_test_rat_check', false) @@ -190,7 +226,7 @@ try { stage('Build') { parallel 'CPU: CentOS 7': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -201,7 +237,7 @@ try { } }, 'CPU: CentOS 7 MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-centos7-mkldnn') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -212,7 +248,7 @@ try { } }, 'GPU: CentOS 7': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -223,7 +259,7 @@ try { } }, 'CPU: Openblas': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu-openblas') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -234,7 +270,7 @@ try { } }, 'CPU: Clang 3.9': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu-clang39') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -244,7 +280,7 @@ try { } }, 'CPU: Clang 5': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu-clang50') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -254,7 +290,7 @@ try { } }, 'CPU: Clang 3.9 MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu-mkldnn-clang39') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -265,7 +301,7 @@ try { } }, 'CPU: Clang 5 MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu-mkldnn-clang50') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -276,7 +312,7 @@ try { } }, 'CPU: MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -287,7 +323,7 @@ try { } }, 'GPU: MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -298,7 +334,7 @@ try { } }, 'GPU: MKLDNN_CUDNNOFF': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu-nocudnn') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -309,7 +345,7 @@ try { } }, 'GPU: CUDA9.1+cuDNN7': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -330,7 +366,7 @@ try { } }, 'Amalgamation MIN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/amalgamationmin') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -340,7 +376,7 @@ try { } }, 'Amalgamation': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/amalgamation') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -351,7 +387,7 @@ try { }, 'GPU: CMake MKLDNN': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cmake-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -362,7 +398,7 @@ try { } }, 'GPU: CMake': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cmake-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -373,7 +409,7 @@ try { } }, 'Build CPU windows':{ - node('mxnetwindows-cpu') { + node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { @@ -407,7 +443,7 @@ try { }, 'Build GPU windows':{ - node('mxnetwindows-cpu') { + node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { @@ -439,7 +475,7 @@ try { } }, 'Build GPU MKLDNN windows':{ - node('mxnetwindows-cpu') { + node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { @@ -480,7 +516,7 @@ try { } }, 'NVidia Jetson / ARMv8':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-jetson-armv8') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -490,7 +526,7 @@ try { } }, 'ARMv7':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-ARMv7') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -500,7 +536,7 @@ try { } }, 'ARMv6':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-ARMv6') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -510,7 +546,7 @@ try { } }, 'ARMv8':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-ARMv8') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -520,7 +556,7 @@ try { } }, 'Android / ARMv8':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/android64') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -530,7 +566,7 @@ try { } }, 'Android / ARMv7':{ - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/androidv7') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -544,7 +580,7 @@ try { stage('Tests') { parallel 'Python2: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-python2-cpu') { try { init_git() @@ -560,7 +596,7 @@ try { } }, 'Python3: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-python3-cpu') { try { init_git() @@ -575,7 +611,7 @@ try { } }, 'Python2: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-python2-gpu') { try { init_git() @@ -589,7 +625,7 @@ try { } }, 'Python3: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { init_git() @@ -603,7 +639,7 @@ try { } }, 'Python2: Quantize GPU': { - node('mxnetlinux-gpu-p3') { + node(NODE_LINUX_GPU_P3) { ws('workspace/ut-python2-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { @@ -619,7 +655,7 @@ try { } }, 'Python3: Quantize GPU': { - node('mxnetlinux-gpu-p3') { + node(NODE_LINUX_GPU_P3) { ws('workspace/ut-python3-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { @@ -635,7 +671,7 @@ try { } }, 'Python2: MKLDNN-CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-python2-mkldnn-cpu') { try { init_git() @@ -651,7 +687,7 @@ try { } }, 'Python2: MKLDNN-GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-python2-mkldnn-gpu') { try { init_git() @@ -665,7 +701,7 @@ try { } }, 'Python3: MKLDNN-CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-python3-mkldnn-cpu') { try { init_git() @@ -680,7 +716,7 @@ try { } }, 'Python3: MKLDNN-GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu') { try { init_git() @@ -694,7 +730,7 @@ try { } }, 'Python3: MKLDNN-GPU-NOCUDNN': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { try { init_git() @@ -708,7 +744,7 @@ try { } }, 'Python3: CentOS 7 CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { try { @@ -725,7 +761,7 @@ try { } }, 'Python3: CentOS 7 GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { @@ -741,7 +777,7 @@ try { } }, 'Scala: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-scala-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -753,7 +789,7 @@ try { } }, 'Clojure: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-clojure-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -765,7 +801,7 @@ try { } }, 'Perl: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-perl-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -777,7 +813,7 @@ try { } }, 'Perl: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -789,7 +825,7 @@ try { } }, 'Cpp: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -801,7 +837,7 @@ try { } }, 'Cpp: MKLDNN+GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -813,7 +849,7 @@ try { } }, 'R: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/ut-r-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -825,7 +861,7 @@ try { } }, 'R: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -838,7 +874,7 @@ try { }, 'Python 2: CPU Win':{ - node('mxnetwindows-cpu') { + node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-cpu') { try { @@ -861,7 +897,7 @@ try { } }, 'Python 3: CPU Win': { - node('mxnetwindows-cpu') { + node(NODE_WINDOWS_CPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-cpu') { try { @@ -883,7 +919,7 @@ try { } }, 'Python 2: GPU Win':{ - node('mxnetwindows-gpu') { + node(NODE_WINDOWS_GPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { @@ -907,7 +943,7 @@ try { } }, 'Python 3: GPU Win':{ - node('mxnetwindows-gpu') { + node(NODE_WINDOWS_GPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { @@ -930,7 +966,7 @@ try { } }, 'Python 3: MKLDNN-GPU Win':{ - node('mxnetwindows-gpu') { + node(NODE_WINDOWS_GPU) { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { @@ -953,7 +989,7 @@ try { } }, 'Onnx CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -965,7 +1001,7 @@ try { } }, 'Python GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -978,7 +1014,7 @@ try { }, // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 // 'Caffe GPU': { - // node('mxnetlinux-gpu') { + // node(NODE_LINUX_GPU) { // ws('workspace/it-caffe') { // timeout(time: max_time, unit: 'MINUTES') { // init_git() @@ -990,7 +1026,7 @@ try { // } // }, 'cpp-package GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/it-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -1012,7 +1048,7 @@ try { } }, 'dist-kvstore tests GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -1028,7 +1064,7 @@ try { * https://github.com/apache/incubator-mxnet/issues/11801 'dist-kvstore tests CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -1040,7 +1076,7 @@ try { } }, */ 'Scala: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/ut-scala-gpu') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -1054,7 +1090,7 @@ try { } stage('Deploy') { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { init_git() @@ -1064,24 +1100,12 @@ try { } } } +}, - // set build status to success at the end - currentBuild.result = "SUCCESS" -} catch (caughtError) { - node("mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("mxnetlinux-cpu") { - // Only send email if master or release branches failed - if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { - emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } +failure_handler: { + // Only send email if master or release branches failed + if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' } } +) From 19f25f169e84b34bb4452a7ea5e5922b1556d351 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Thu, 2 Aug 2018 14:56:26 +0200 Subject: [PATCH 2/7] Add functionality to assign node labels dynamically --- Jenkinsfile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 92fddb1f093e..7c922aa019d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,11 +168,13 @@ def python3_gpu_ut_nocudnn(docker_container_name) { } } -NODE_LINUX_CPU = 'mxnetlinux-cpu' -NODE_LINUX_GPU = 'mxnetlinux-gpu' -NODE_LINUX_GPU_P3 = 'mxnetlinux-gpu-p3' -NODE_WINDOWS_CPU = 'mxnetwindows-cpu' -NODE_WINDOWS_GPU = 'mxnetwindows-gpu' +def assign_node_labels(args) { + NODE_LINUX_CPU = args.linux_cpu + NODE_LINUX_GPU = args.linux_gpu + NODE_LINUX_GPU_P3 = args.linux_gpu_p3 + NODE_WINDOWS_CPU = args.windows_cpu + NODE_WINDOWS_GPU = args.windows_gpu +} def main_wrapper(args) { // hander: Core logic @@ -203,6 +205,8 @@ def main_wrapper(args) { } } +assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_cpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') + main_wrapper( handler: { stage('Sanity Check') { From aff116abb4e362ee75ac5d47036f2c65c7fb55f0 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Thu, 2 Aug 2018 15:04:38 +0200 Subject: [PATCH 3/7] Extract functions into util file --- Jenkinsfile | 550 ++++++++++++++---------------------- ci/Jenkinsfile_utils.groovy | 151 ++++++++++ 2 files changed, 364 insertions(+), 337 deletions(-) create mode 100644 ci/Jenkinsfile_utils.groovy diff --git a/Jenkinsfile b/Jenkinsfile index 7c922aa019d4..349ecbbfee72 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,118 +30,26 @@ mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc- mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // timeout in minutes max_time = 120 -// assign any caught errors here -err = null -// initialize source codes -def init_git() { - deleteDir() - retry(5) { - try { - // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of - // retries as this will increase the amount of requests and worsen the throttling - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - -def init_git_win() { - deleteDir() - retry(5) { - try { - // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of - // retries as this will increase the amount of requests and worsen the throttling - timeout(time: 15, unit: 'MINUTES') { - checkout scm - bat 'git submodule update --init --recursive' - bat 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - -// pack libraries for later use -def pack_lib(name, libs=mx_lib) { - sh """ -echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" - stash includes: libs, name: name -} - -// unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { - unstash name - sh """ -echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" -} - -def publish_test_coverage() { - // Fall back to our own copy of the bash helper if it failed to download the public version - sh '(curl --retry 10 -s https://codecov.io/bash | bash -s -) || (curl --retry 10 -s https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data/codecov-bash.txt | bash -s -)' -} - -def collect_test_results_unix(original_file_name, new_file_name) { - if (fileExists(original_file_name)) { - // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage - // Thus, we have to pick a name manually and rename the files so that they can be stored separately. - sh 'cp ' + original_file_name + ' ' + new_file_name - archiveArtifacts artifacts: new_file_name - } -} - -def collect_test_results_windows(original_file_name, new_file_name) { - // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage - // Thus, we have to pick a name manually and rename the files so that they can be stored separately. - if (fileExists(original_file_name)) { - bat 'xcopy ' + original_file_name + ' ' + new_file_name + '*' - archiveArtifacts artifacts: new_file_name - } -} - - -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - - sh command -} // Python unittest for CPU // Python 2 def python2_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false) } } // Python 3 def python3_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false) } } def python3_ut_mkldnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false) } } @@ -150,79 +58,47 @@ def python3_ut_mkldnn(docker_container_name) { // Python 2 def python2_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true) } } // Python 3 def python3_gpu_ut(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true) } } // Python 3 NOCUDNN def python3_gpu_ut_nocudnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { - docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_nocudnn', true) + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_nocudnn', true) } } -def assign_node_labels(args) { - NODE_LINUX_CPU = args.linux_cpu - NODE_LINUX_GPU = args.linux_gpu - NODE_LINUX_GPU_P3 = args.linux_gpu_p3 - NODE_WINDOWS_CPU = args.windows_cpu - NODE_WINDOWS_GPU = args.windows_gpu -} - -def main_wrapper(args) { - // hander: Core logic - // failure_handler: Failure handler - - try { - // Call actual handler - args['handler']() - - // set build status to success at the end - currentBuild.result = "SUCCESS" - } catch (caughtError) { - node(NODE_LINUX_CPU) { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } - } finally { - node(NODE_LINUX_CPU) { - // Call failure handler - args['failure_handler']() - - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } - } - } +node('mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_cpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') -assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_cpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') - -main_wrapper( +utils.main_wrapper( handler: { stage('Sanity Check') { parallel 'Lint': { node(NODE_LINUX_CPU) { ws('workspace/sanity-lint') { - init_git() - docker_run('ubuntu_cpu', 'sanity_check', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'sanity_check', false) } } }, 'RAT License': { node(NODE_LINUX_CPU) { ws('workspace/sanity-rat') { - init_git() - docker_run('ubuntu_rat', 'nightly_test_rat_check', false) + utils.init_git() + utils.docker_run('ubuntu_rat', 'nightly_test_rat_check', false) } } } @@ -233,9 +109,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('centos7_cpu', 'build_centos7_cpu', false) - pack_lib('centos7_cpu') + utils.init_git() + utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) + utils.pack_lib('centos7_cpu') } } } @@ -244,9 +120,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-centos7-mkldnn') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('centos7_cpu', 'build_centos7_mkldnn', false) - pack_lib('centos7_mkldnn') + utils.init_git() + utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false) + utils.pack_lib('centos7_mkldnn') } } } @@ -255,9 +131,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('centos7_gpu', 'build_centos7_gpu', false) - pack_lib('centos7_gpu') + utils.init_git() + utils.docker_run('centos7_gpu', 'build_centos7_gpu', false) + utils.pack_lib('centos7_gpu') } } } @@ -266,9 +142,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-openblas') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - pack_lib('cpu', mx_dist_lib) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) + utils.pack_lib('cpu', mx_dist_lib) } } } @@ -277,8 +153,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-clang39') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false) } } } @@ -287,8 +163,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-clang50') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false) } } } @@ -297,9 +173,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-mkldnn-clang39') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false) - pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false) + utils.pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib) } } } @@ -308,9 +184,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-mkldnn-clang50') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false) - pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false) + utils.pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib) } } } @@ -319,9 +195,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false) - pack_lib('mkldnn_cpu', mx_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false) + utils.pack_lib('mkldnn_cpu', mx_mkldnn_lib) } } } @@ -330,9 +206,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) - pack_lib('mkldnn_gpu', mx_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false) + utils.pack_lib('mkldnn_gpu', mx_mkldnn_lib) } } } @@ -341,9 +217,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-gpu-nocudnn') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) - pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false) + utils.pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) } } } @@ -352,9 +228,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) - pack_lib('gpu', mx_dist_lib) + utils.init_git() + utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) + utils.pack_lib('gpu', mx_dist_lib) stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet' @@ -373,8 +249,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/amalgamationmin') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false) } } } @@ -383,8 +259,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/amalgamation') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false) } } } @@ -394,9 +270,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cmake-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false) - pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) + utils.init_git() + utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false) + utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) } } } @@ -405,9 +281,9 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-cmake-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) - pack_lib('cmake_gpu', mx_cmake_lib) + utils.init_git() + utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false) + utils.pack_lib('cmake_gpu', mx_cmake_lib) } } } @@ -417,7 +293,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() + utils.init_git_win() bat """mkdir build_vc14_cpu call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" cd build_vc14_cpu @@ -451,7 +327,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() + utils.init_git_win() bat """mkdir build_vc14_gpu call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" cd build_vc14_gpu @@ -483,7 +359,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - init_git_win() + utils.init_git_win() bat """mkdir build_%BUILD_NAME% call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" cd build_%BUILD_NAME% @@ -523,8 +399,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-jetson-armv8') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('jetson', 'build_jetson', false) + utils.init_git() + utils.docker_run('jetson', 'build_jetson', false) } } } @@ -533,8 +409,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-ARMv7') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('armv7', 'build_armv7', false) + utils.init_git() + utils.docker_run('armv7', 'build_armv7', false) } } } @@ -543,8 +419,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-ARMv6') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('armv6', 'build_armv6', false) + utils.init_git() + utils.docker_run('armv6', 'build_armv6', false) } } } @@ -553,8 +429,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/build-ARMv8') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('armv8', 'build_armv8', false) + utils.init_git() + utils.docker_run('armv8', 'build_armv8', false) } } } @@ -563,8 +439,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/android64') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('android_armv8', 'build_android_armv8', false) + utils.init_git() + utils.docker_run('android_armv8', 'build_android_armv8', false) } } } @@ -573,8 +449,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/androidv7') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('android_armv7', 'build_android_armv7', false) + utils.init_git() + utils.docker_run('android_armv7', 'build_android_armv7', false) } } } @@ -587,14 +463,14 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-cpu') { try { - init_git() - unpack_lib('cpu') + utils.init_git() + utils.unpack_lib('cpu') python2_ut('ubuntu_cpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') - collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_cpu_train.xml') - collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_cpu_quantization.xml') + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_cpu_train.xml') + utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_cpu_quantization.xml') } } } @@ -603,13 +479,13 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-cpu') { try { - init_git() - unpack_lib('cpu') + utils.init_git() + utils.unpack_lib('cpu') python3_ut('ubuntu_cpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') - collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml') } } } @@ -618,12 +494,12 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-python2-gpu') { try { - init_git() - unpack_lib('gpu', mx_lib) + utils.init_git() + utils.unpack_lib('gpu', mx_lib) python2_gpu_ut('ubuntu_gpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml') } } } @@ -632,12 +508,12 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { - init_git() - unpack_lib('gpu', mx_lib) + utils.init_git() + utils.unpack_lib('gpu', mx_lib) python3_gpu_ut('ubuntu_gpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml') } } } @@ -647,12 +523,12 @@ handler: { ws('workspace/ut-python2-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - init_git() - unpack_lib('gpu', mx_lib) - docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu', mx_lib) + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true) + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml') + utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml') } } } @@ -663,12 +539,12 @@ handler: { ws('workspace/ut-python3-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - init_git() - unpack_lib('gpu', mx_lib) - docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu', mx_lib) + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true) + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml') + utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml') } } } @@ -678,14 +554,14 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-mkldnn-cpu') { try { - init_git() - unpack_lib('mkldnn_cpu', mx_mkldnn_lib) + utils.init_git() + utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) python2_ut('ubuntu_cpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_mkldnn_cpu_unittest.xml') - collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_mkldnn_cpu_train.xml') - collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_mkldnn_cpu_quantization.xml') + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_mkldnn_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_mkldnn_cpu_train.xml') + utils.collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_mkldnn_cpu_quantization.xml') } } } @@ -694,12 +570,12 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-python2-mkldnn-gpu') { try { - init_git() - unpack_lib('mkldnn_gpu', mx_mkldnn_lib) + utils.init_git() + utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) python2_gpu_ut('ubuntu_gpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml') } } } @@ -708,13 +584,13 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-mkldnn-cpu') { try { - init_git() - unpack_lib('mkldnn_cpu', mx_mkldnn_lib) + utils.init_git() + utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) python3_ut_mkldnn('ubuntu_cpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml') - collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml') + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml') } } } @@ -723,12 +599,12 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu') { try { - init_git() - unpack_lib('mkldnn_gpu', mx_mkldnn_lib) + utils.init_git() + utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) python3_gpu_ut('ubuntu_gpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml') } } } @@ -737,12 +613,12 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { try { - init_git() - unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + utils.init_git() + utils.unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) python3_gpu_ut_nocudnn('ubuntu_gpu') - publish_test_coverage() + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml') } } } @@ -752,13 +628,13 @@ handler: { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { try { - init_git() - unpack_lib('centos7_cpu') - docker_run('centos7_cpu', 'unittest_centos7_cpu', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('centos7_cpu') + utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false) + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml') - collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_train.xml') + utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml') + utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_train.xml') } } } @@ -769,12 +645,12 @@ handler: { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - init_git() - unpack_lib('centos7_gpu') - docker_run('centos7_gpu', 'unittest_centos7_gpu', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('centos7_gpu') + utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true) + utils.publish_test_coverage() } finally { - collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml') + utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml') } } } @@ -784,10 +660,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-scala-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu', mx_dist_lib) - docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu', mx_dist_lib) + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) + utils.publish_test_coverage() } } } @@ -796,10 +672,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-clojure-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu', mx_dist_lib) - docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu', mx_dist_lib) + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false) + utils.publish_test_coverage() } } } @@ -808,10 +684,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-perl-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu') - docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu') + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) + utils.publish_test_coverage() } } } @@ -820,10 +696,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu') - docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu') + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) + utils.publish_test_coverage() } } } @@ -832,10 +708,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cmake_gpu', mx_cmake_lib) - docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cmake_gpu', mx_cmake_lib) + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) + utils.publish_test_coverage() } } } @@ -844,10 +720,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) - docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) + utils.publish_test_coverage() } } } @@ -856,10 +732,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/ut-r-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu') - docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu') + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) + utils.publish_test_coverage() } } } @@ -868,10 +744,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu') - docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu') + utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) + utils.publish_test_coverage() } } } @@ -882,7 +758,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-cpu') { try { - init_git_win() + utils.init_git_win() unstash 'vc14_cpu' bat '''rmdir /s/q pkg_vc14_cpu 7z x -y vc14_cpu.7z''' @@ -894,7 +770,7 @@ handler: { del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" } finally { - collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') + utils.collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') } } } @@ -905,7 +781,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-cpu') { try { - init_git_win() + utils.init_git_win() unstash 'vc14_cpu' bat '''rmdir /s/q pkg_vc14_cpu 7z x -y vc14_cpu.7z''' @@ -916,7 +792,7 @@ handler: { del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" } finally { - collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') + utils.collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') } } } @@ -927,7 +803,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { - init_git_win() + utils.init_git_win() unstash 'vc14_gpu' bat '''rmdir /s/q pkg_vc14_gpu 7z x -y vc14_gpu.7z''' @@ -939,8 +815,8 @@ handler: { del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') + utils.collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') + utils.collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') } } } @@ -951,7 +827,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { - init_git_win() + utils.init_git_win() unstash 'vc14_gpu' bat '''rmdir /s/q pkg_vc14_gpu 7z x -y vc14_gpu.7z''' @@ -962,8 +838,8 @@ handler: { del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') + utils.collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') + utils.collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') } } } @@ -974,7 +850,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/ut-python-gpu') { try { - init_git_win() + utils.init_git_win() unstash 'vc14_gpu_mkldnn' bat '''rmdir /s/q pkg_vc14_gpu_mkldnn 7z x -y vc14_gpu_mkldnn.7z''' @@ -985,8 +861,8 @@ handler: { del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') + utils.collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') + utils.collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') } } } @@ -996,10 +872,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu') - docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu') + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) + utils.publish_test_coverage() } } } @@ -1008,10 +884,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu') - docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu') + utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) + utils.publish_test_coverage() } } } @@ -1021,10 +897,10 @@ handler: { // node(NODE_LINUX_GPU) { // ws('workspace/it-caffe') { // timeout(time: max_time, unit: 'MINUTES') { - // init_git() - // unpack_lib('gpu') - // docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) - // publish_test_coverage() + // utils.init_git() + // utils.unpack_lib('gpu') + // utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) + // utils.publish_test_coverage() // } // } // } @@ -1033,8 +909,8 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/it-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu') + utils.init_git() + utils.unpack_lib('gpu') unstash 'cpp_lenet' unstash 'cpp_alexnet' unstash 'cpp_googlenet' @@ -1045,8 +921,8 @@ handler: { unstash 'cpp_mlp_gpu' unstash 'cpp_test_score' unstash 'cpp_test_optimizer' - docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) - publish_test_coverage() + utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) + utils.publish_test_coverage() } } } @@ -1055,10 +931,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu') - docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu') + utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) + utils.publish_test_coverage() } } } @@ -1071,10 +947,10 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('cpu') - docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('cpu') + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) + utils.publish_test_coverage() } } } @@ -1083,10 +959,10 @@ handler: { node(NODE_LINUX_GPU) { ws('workspace/ut-scala-gpu') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - unpack_lib('gpu', mx_dist_lib) - docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true) - publish_test_coverage() + utils.init_git() + utils.unpack_lib('gpu', mx_dist_lib) + utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true) + utils.publish_test_coverage() } } } @@ -1097,8 +973,8 @@ handler: { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - init_git() - docker_run('ubuntu_cpu', 'deploy_docs', false) + utils.init_git() + utils.docker_run('ubuntu_cpu', 'deploy_docs', false) sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" } } diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy new file mode 100644 index 000000000000..9d2252ae178f --- /dev/null +++ b/ci/Jenkinsfile_utils.groovy @@ -0,0 +1,151 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// initialize source codes +def init_git() { + deleteDir() + retry(5) { + try { + // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of + // retries as this will increase the amount of requests and worsen the throttling + timeout(time: 15, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init --recursive' + sh 'git clean -d -f' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes with ${exc}" + sleep 2 + } + } +} + +def init_git_win() { + deleteDir() + retry(5) { + try { + // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of + // retries as this will increase the amount of requests and worsen the throttling + timeout(time: 15, unit: 'MINUTES') { + checkout scm + bat 'git submodule update --init --recursive' + bat 'git clean -d -f' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes with ${exc}" + sleep 2 + } + } +} + +// pack libraries for later use +def pack_lib(name, libs=mx_lib) { + sh """ +echo "Packing ${libs} into ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" + stash includes: libs, name: name +} + +// unpack libraries saved before +def unpack_lib(name, libs=mx_lib) { + unstash name + sh """ +echo "Unpacked ${libs} from ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" +} + +def publish_test_coverage() { + // Fall back to our own copy of the bash helper if it failed to download the public version + sh '(curl --retry 10 -s https://codecov.io/bash | bash -s -) || (curl --retry 10 -s https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data/codecov-bash.txt | bash -s -)' +} + +def collect_test_results_unix(original_file_name, new_file_name) { + if (fileExists(original_file_name)) { + // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage + // Thus, we have to pick a name manually and rename the files so that they can be stored separately. + sh 'cp ' + original_file_name + ' ' + new_file_name + archiveArtifacts artifacts: new_file_name + } +} + +def collect_test_results_windows(original_file_name, new_file_name) { + // Rename file to make it distinguishable. Unfortunately, it's not possible to get STAGE_NAME in a parallel stage + // Thus, we have to pick a name manually and rename the files so that they can be stored separately. + if (fileExists(original_file_name)) { + bat 'xcopy ' + original_file_name + ' ' + new_file_name + '*' + archiveArtifacts artifacts: new_file_name + } +} + + +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { + def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') + command = command.replaceAll('%PLATFORM%', platform) + command = command.replaceAll('%FUNCTION_NAME%', function_name) + command = command.replaceAll('%SHARED_MEM%', shared_mem) + + sh command +} + + + +def assign_node_labels(args) { + NODE_LINUX_CPU = args.linux_cpu + NODE_LINUX_GPU = args.linux_gpu + NODE_LINUX_GPU_P3 = args.linux_gpu_p3 + NODE_WINDOWS_CPU = args.windows_cpu + NODE_WINDOWS_GPU = args.windows_gpu +} + +// assign any caught errors here +err = null +def main_wrapper(args) { + // hander: Core logic + // failure_handler: Failure handler + + try { + // Call actual handler + args['handler']() + + // set build status to success at the end + currentBuild.result = "SUCCESS" + } catch (caughtError) { + node(NODE_LINUX_CPU) { + sh "echo caught ${caughtError}" + err = caughtError + currentBuild.result = "FAILURE" + } + } finally { + node(NODE_LINUX_CPU) { + // Call failure handler + args['failure_handler']() + + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } + } +} +return this \ No newline at end of file From 6c39cd8f924361ebdc2aec89db6ab096c2af92de Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Thu, 2 Aug 2018 15:42:40 +0200 Subject: [PATCH 4/7] Change all Jenkinsfiles to use utils --- Jenkinsfile | 4 +- ci/Jenkinsfile_docker_cache | 57 +++-------- docs/Jenkinsfile | 54 +++-------- tests/nightly/Jenkinsfile | 95 +++++-------------- tests/nightly/JenkinsfileForBinaries | 87 ++++------------- .../JenkinsfileForBLC | 74 ++++----------- .../JenkinsfileForMBCC | 79 +++------------ 7 files changed, 107 insertions(+), 343 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 349ecbbfee72..a55cb90f0628 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -980,8 +980,8 @@ handler: { } } } -}, - +} +, failure_handler: { // Only send email if master or release branches failed if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) { diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache index 550425bb932a..d38f936f14d7 100644 --- a/ci/Jenkinsfile_docker_cache +++ b/ci/Jenkinsfile_docker_cache @@ -22,34 +22,18 @@ // timeout in minutes total_timeout = 300 -git_timeout = 15 -// assign any caught errors here -err = null -// initialize source codes -def init_git() { - deleteDir() - retry(5) { - try { - // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of - // retries as this will increase the amount of requests and worsen the throttling - timeout(time: git_timeout, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -x -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } +node('restricted-mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') - -try { +utils.main_wrapper( +handler: { stage("Docker cache build & publish") { - node('restricted-mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/docker_cache') { timeout(time: total_timeout, unit: 'MINUTES') { init_git() @@ -58,24 +42,11 @@ try { } } } - - // set build status to success at the end - currentBuild.result = "SUCCESS" -} catch (caughtError) { - node("restricted-mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("restricted-mxnetlinux-cpu") { - // Only send email if master failed - if (currentBuild.result == "FAILURE") { - emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } +, +failure_handler: +{ + if (currentBuild.result == "FAILURE") { + emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' } } +) \ No newline at end of file diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile index ef0755faac7c..172aaffb5d8c 100644 --- a/docs/Jenkinsfile +++ b/docs/Jenkinsfile @@ -22,32 +22,18 @@ // timeout in minutes max_time = 60 -// assign any caught errors here -err = null -// initialize source code -def init_git() { - deleteDir() - retry(5) { - try { - // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of - // retries as this will increase the amount of requests and worsen the throttling - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } +node('restricted-mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') -try { +utils.main_wrapper( +handler: { stage('Build Docs') { - node('restricted-mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/docs') { init_git() timeout(time: max_time, unit: 'MINUTES') { @@ -58,24 +44,10 @@ try { } } } - - // set build status to success at the end - currentBuild.result = "SUCCESS" -} catch (caughtError) { - node("restricted-mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("restricted-mxnetlinux-cpu") { - // Only send email if master failed - if (currentBuild.result == "FAILURE") { - emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } +) \ No newline at end of file diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 173a33ab4881..9d8ac88675e0 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -15,62 +15,23 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - - +// //This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile -err = null mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a' -// pack libraries for later use -def pack_lib(name, libs=mx_lib) { - sh """ -echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" - stash includes: libs, name: name -} - -// unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { - unstash name - sh """ -echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" -} - -def init_git() { - deleteDir() - retry(5) { - try { - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - - sh command +node('mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') -try { +utils.main_wrapper( +handler: { stage('NightlyTests'){ parallel 'CompilationWarnings: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-compilationTest') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_compilation_warning', false) @@ -78,7 +39,7 @@ try { } }, 'InstallationGuide: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-Installation-cpu') { init_git() //Some install guide tests are currently diabled and tracked here: @@ -92,7 +53,7 @@ try { } }, 'InstallationGuide: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/nt-Installation-gpu') { init_git() //Some install guide tests are currently diabled and tracked here: @@ -106,14 +67,14 @@ try { } }, 'PipTest: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/nt-pipTest') { init_git() } } }, 'Amalgamation-atlas: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-amalgamation1') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas', false) @@ -121,7 +82,7 @@ try { } }, 'Amalgamation-atlas-min: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-amalgamation2') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MIN=1', false) @@ -129,7 +90,7 @@ try { } }, 'Amalgamation-atlas-mkl: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-amalgamation3') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_MKL=1', false) @@ -137,7 +98,7 @@ try { } }, 'Amalgamation-atlas-cuda: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-amalgamation4') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_CUDA=1', false) @@ -145,7 +106,7 @@ try { } }, 'Amalgamation-atlas-openmp: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-amalgamation5') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas DISABLE_OPENMP=0', false) @@ -153,7 +114,7 @@ try { } }, 'MXNetJS: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/nt-mxnetjs') { init_git() docker_run('ubuntu_nightly_cpu', 'nightly_test_javascript', false) @@ -161,21 +122,11 @@ try { } } } -} catch (caughtError) { - node("mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("mxnetlinux-cpu") { - // Only send email if nightly test failed - if (currentBuild.result == "FAILURE") { +, +failure_handler: { + // Only send email if nightly test failed + if (currentBuild.result == "FAILURE") { emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } } } +) \ No newline at end of file diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 0b009d28a55a..6086c9f7d87f 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -15,61 +15,23 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +// //This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile -err = null mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -// pack libraries for later use -def pack_lib(name, libs=mx_lib) { - sh """ -echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" - stash includes: libs, name: name -} - -// unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { - unstash name - sh """ -echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" +node('mxnetlinux-cpu) { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') -def init_git() { - deleteDir() - retry(5) { - try { - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - - sh command -} - -try { +utils.main_wrapper( +handler: { stage('Build') { parallel 'GPU: CUDA9.1+cuDNN7': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-gpu') { init_git() //sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7" @@ -82,7 +44,7 @@ try { stage('NightlyTests'){ parallel 'ImageClassification: GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/nt-ImageClassificationTest') { init_git() unpack_lib('gpu', mx_lib) @@ -100,7 +62,7 @@ try { } }, 'StraightDope: Python2 Single-GPU': { - node('mxnetlinux-gpu-p3') { + node(NODE_LINUX_GPU_P3) { ws('workspace/straight_dope-single_gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -109,7 +71,7 @@ try { } }, 'StraightDope: Python2 Multi-GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/straight_dope-multi_gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -118,7 +80,7 @@ try { } }, 'StraightDope: Python3 Single-GPU': { - node('mxnetlinux-gpu-p3') { + node(NODE_LINUX_GPU_P3) { ws('workspace/straight_dope-single_gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -127,7 +89,7 @@ try { } }, 'StraightDope: Python3 Multi-GPU': { - node('mxnetlinux-gpu') { + node(NODE_LINUX_GPU) { ws('workspace/straight_dope-multi_gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -136,21 +98,10 @@ try { } } } -} catch (caughtError) { - node("mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("mxnetlinux-cpu") { - // Only send email if master failed - if (currentBuild.result == "FAILURE") { - emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' } } +) diff --git a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC index 912b65b9bbc4..63c2b40b7850 100755 --- a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC +++ b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC @@ -15,45 +15,23 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - - +// //This is a Jenkinsfile for the broken link checker test. -err = null - -def init_git() { - deleteDir() - retry(5) { - try { - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } +node('mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_cpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - - sh command -} - -try { +utils.main_wrapper( +handler: { stage('BLC'){ parallel 'BrokenLinkChecker: CPU': { - node('mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/brokenLinkChecker') { - timeout(time: 40, unit: 'MINUTES') { + timeout(time: 60, unit: 'MINUTES') { try { init_git() sh 'aws s3 cp s3://mxnet-ci-prod-slave-data/url_list.txt ./tests/nightly/broken_link_checker_test/url_list.txt' @@ -67,26 +45,16 @@ try { } } } -} catch (caughtError) { - node("mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("mxnetlinux-cpu") { - // Only send email if nightly test failed - if (currentBuild.result == "FAILURE") { - emailext body: '''https://mxnet.incubator.apache.org broken link test summary: - | - |Please view the logs at ${BUILD_URL} - | - |${BUILD_LOG_EXCERPT, start="START - Broken links summary", end="END - Broken links summary"}''', - replyTo: '${EMAIL}', subject: '[BROKEN LINK CHECKER FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' - } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } +, +failure_handler: +{ + if (currentBuild.result == "FAILURE") { + emailext body: '''https://mxnet.incubator.apache.org broken link test summary: + | + |Please view the logs at ${BUILD_URL} + | + |${BUILD_LOG_EXCERPT, start="START - Broken links summary", end="END - Broken links summary"}''', + replyTo: '${EMAIL}', subject: '[BROKEN LINK CHECKER FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' } } +) diff --git a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC index 412d68d56ff3..a48e6eeaabad 100644 --- a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC +++ b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC @@ -15,61 +15,22 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - - +// //This is a Jenkinsfile for the model backwards compatibility checker. The format and some functions have been picked up from the top-level Jenkinsfile. -err = null mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -def init_git() { - deleteDir() - retry(5) { - try { - timeout(time: 15, unit: 'MINUTES') { - checkout scm - sh 'git submodule update --init --recursive' - sh 'git clean -d -f' - } - } catch (exc) { - deleteDir() - error "Failed to fetch source codes with ${exc}" - sleep 2 - } - } -} - -// pack libraries for later use -def pack_lib(name, libs=mx_lib) { - sh """ -echo "Packing ${libs} into ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" - stash includes: libs, name: name +node('restricted-mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') } +utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') -// unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { - unstash name - sh """ -echo "Unpacked ${libs} from ${name}" -echo ${libs} | sed -e 's/,/ /g' | xargs md5sum -""" -} - -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" - command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') - command = command.replaceAll('%PLATFORM%', platform) - command = command.replaceAll('%FUNCTION_NAME%', function_name) - command = command.replaceAll('%SHARED_MEM%', shared_mem) - - sh command -} - -try { +utils.main_wrapper( +handler: { stage('MBCC Train'){ - node('restricted-mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/modelBackwardsCompat') { init_git() // Train models on older versions @@ -81,7 +42,7 @@ try { } stage('MXNet Build'){ - node('restricted-mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/build-cpu') { init_git() docker_run('ubuntu_cpu','build_ubuntu_cpu', false) @@ -91,7 +52,7 @@ try { } stage('MBCC Inference'){ - node('restricted-mxnetlinux-cpu') { + node(NODE_LINUX_CPU) { ws('workspace/modelBackwardsCompat') { init_git() unpack_lib('cpu', mx_lib) @@ -100,21 +61,11 @@ try { } } } -} catch (caughtError) { - node("restricted-mxnetlinux-cpu") { - sh "echo caught ${caughtError}" - err = caughtError - currentBuild.result = "FAILURE" - } -} finally { - node("restricted-mxnetlinux-cpu") { - // Only send email if model backwards compat test failed +, +failure_handler: { +// Only send email if model backwards compat test failed if (currentBuild.result == "FAILURE") { emailext body: 'Nightly tests for model backwards compatibity on MXNet branch : ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[MODEL BACKWARDS COMPATIBILITY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' } - // Remember to rethrow so the build is marked as failing - if (err) { - throw err - } - } } +) \ No newline at end of file From 34d288d558ccfd00157e67a78380e894382e7b20 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Fri, 3 Aug 2018 13:10:10 +0200 Subject: [PATCH 5/7] Make a new commit... --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a55cb90f0628..6c653901ce88 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +// // Jenkins pipeline // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ From 33d838fdffcaf515189ebe50ec12a46547dc63c5 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Mon, 6 Aug 2018 14:49:27 +0200 Subject: [PATCH 6/7] Address review comments 1 --- ci/Jenkinsfile_docker_cache | 3 ++- ci/Jenkinsfile_utils.groovy | 7 ++++--- tests/nightly/Jenkinsfile | 3 ++- .../model_backwards_compatibility_check/JenkinsfileForMBCC | 3 ++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache index d38f936f14d7..bee769a0a321 100644 --- a/ci/Jenkinsfile_docker_cache +++ b/ci/Jenkinsfile_docker_cache @@ -49,4 +49,5 @@ failure_handler: emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}' } } -) \ No newline at end of file +) + diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 9d2252ae178f..3892ccc5173d 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -27,7 +27,7 @@ def init_git() { timeout(time: 15, unit: 'MINUTES') { checkout scm sh 'git submodule update --init --recursive' - sh 'git clean -d -f' + sh 'git clean -xdff' } } catch (exc) { deleteDir() @@ -46,7 +46,7 @@ def init_git_win() { timeout(time: 15, unit: 'MINUTES') { checkout scm bat 'git submodule update --init --recursive' - bat 'git clean -d -f' + bat 'git clean -xdff' } } catch (exc) { deleteDir() @@ -148,4 +148,5 @@ def main_wrapper(args) { } } } -return this \ No newline at end of file +return this + diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 9d8ac88675e0..9f2a4000d2a3 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -129,4 +129,5 @@ failure_handler: { emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' } } -) \ No newline at end of file +) + diff --git a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC index a48e6eeaabad..7f0494c442c5 100644 --- a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC +++ b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC @@ -68,4 +68,5 @@ failure_handler: { emailext body: 'Nightly tests for model backwards compatibity on MXNet branch : ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[MODEL BACKWARDS COMPATIBILITY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}' } } -) \ No newline at end of file +) + From fc4c2e8ab457ead582f733486462a25a7176ace4 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Wed, 8 Aug 2018 14:45:48 +0200 Subject: [PATCH 7/7] Address review comments 2 --- Jenkinsfile | 36 +++++++++---------- ci/Jenkinsfile_docker_cache | 2 +- ci/Jenkinsfile_utils.groovy | 21 +++++------ docs/Jenkinsfile | 4 +-- tests/nightly/Jenkinsfile | 2 +- tests/nightly/JenkinsfileForBinaries | 2 +- .../JenkinsfileForBLC | 2 +- .../JenkinsfileForMBCC | 2 +- 8 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 82a9907eebf4..d74f0b43c212 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,7 +86,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Sanity Check') { parallel 'Lint': { node(NODE_LINUX_CPU) { @@ -113,7 +113,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) - utils.pack_lib('centos7_cpu') + utils.pack_lib('centos7_cpu', mx_lib) } } } @@ -124,7 +124,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false) - utils.pack_lib('centos7_mkldnn') + utils.pack_lib('centos7_mkldnn', mx_lib) } } } @@ -135,7 +135,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_gpu', 'build_centos7_gpu', false) - utils.pack_lib('centos7_gpu') + utils.pack_lib('centos7_gpu', mx_lib) } } } @@ -410,7 +410,7 @@ handler: { ws('workspace/ut-python2-cpu') { try { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) python2_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -426,7 +426,7 @@ handler: { ws('workspace/ut-python3-cpu') { try { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) python3_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -589,7 +589,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { try { utils.init_git() - utils.unpack_lib('centos7_cpu') + utils.unpack_lib('centos7_cpu', mx_lib) utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false) utils.publish_test_coverage() } finally { @@ -606,7 +606,7 @@ handler: { timeout(time: max_time, unit: 'MINUTES') { try { utils.init_git() - utils.unpack_lib('centos7_gpu') + utils.unpack_lib('centos7_gpu', mx_lib) utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true) utils.publish_test_coverage() } finally { @@ -645,7 +645,7 @@ handler: { ws('workspace/ut-perl-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) utils.publish_test_coverage() } @@ -657,7 +657,7 @@ handler: { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) utils.publish_test_coverage() } @@ -693,7 +693,7 @@ handler: { ws('workspace/ut-r-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) utils.publish_test_coverage() } @@ -705,7 +705,7 @@ handler: { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) utils.publish_test_coverage() } @@ -796,7 +796,7 @@ handler: { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) utils.publish_test_coverage() } @@ -808,7 +808,7 @@ handler: { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) utils.publish_test_coverage() } @@ -821,7 +821,7 @@ handler: { // ws('workspace/it-caffe') { // timeout(time: max_time, unit: 'MINUTES') { // utils.init_git() - // utils.unpack_lib('gpu') + // utils.unpack_lib('gpu', mx_lib) // utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true) // utils.publish_test_coverage() // } @@ -833,7 +833,7 @@ handler: { ws('workspace/it-cpp-package') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) unstash 'cpp_lenet' unstash 'cpp_alexnet' unstash 'cpp_googlenet' @@ -855,7 +855,7 @@ handler: { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('gpu') + utils.unpack_lib('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) utils.publish_test_coverage() } @@ -871,7 +871,7 @@ handler: { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.unpack_lib('cpu') + utils.unpack_lib('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) utils.publish_test_coverage() } diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache index bee769a0a321..e8556c625ff4 100644 --- a/ci/Jenkinsfile_docker_cache +++ b/ci/Jenkinsfile_docker_cache @@ -31,7 +31,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage("Docker cache build & publish") { node(NODE_LINUX_CPU) { ws('workspace/docker_cache') { diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 3892ccc5173d..dfa2519bd0e5 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -57,7 +57,7 @@ def init_git_win() { } // pack libraries for later use -def pack_lib(name, libs=mx_lib) { +def pack_lib(name, libs) { sh """ echo "Packing ${libs} into ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -66,7 +66,7 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum } // unpack libraries saved before -def unpack_lib(name, libs=mx_lib) { +def unpack_lib(name, libs) { unstash name sh """ echo "Unpacked ${libs} from ${name}" @@ -118,15 +118,17 @@ def assign_node_labels(args) { NODE_WINDOWS_GPU = args.windows_gpu } -// assign any caught errors here -err = null def main_wrapper(args) { - // hander: Core logic - // failure_handler: Failure handler - + // Main Jenkinsfile pipeline wrapper handler that allows to wrap core logic into a format + // that supports proper failure handling + // args: + // - core_logic: Jenkins pipeline containing core execution logic + // - failure_handler: Failure handler + + // assign any caught errors here + err = null try { - // Call actual handler - args['handler']() + args['core_logic']() // set build status to success at the end currentBuild.result = "SUCCESS" @@ -149,4 +151,3 @@ def main_wrapper(args) { } } return this - diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile index 172aaffb5d8c..2431825c97ca 100644 --- a/docs/Jenkinsfile +++ b/docs/Jenkinsfile @@ -31,7 +31,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Build Docs') { node(NODE_LINUX_CPU) { ws('workspace/docs') { @@ -50,4 +50,4 @@ failure_handler: { emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } -) \ No newline at end of file +) diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 9f2a4000d2a3..b8debb21344f 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -28,7 +28,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('NightlyTests'){ parallel 'CompilationWarnings: CPU': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 6086c9f7d87f..d3454dc6832b 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -28,7 +28,7 @@ node('mxnetlinux-cpu) { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('Build') { parallel 'GPU: CUDA9.1+cuDNN7': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC index 185708d0875a..6e859ed6bf92 100755 --- a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC +++ b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC @@ -26,7 +26,7 @@ node('mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('BLC'){ parallel 'BrokenLinkChecker: CPU': { node(NODE_LINUX_CPU) { diff --git a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC index 7f0494c442c5..c3fe4fd4cd89 100644 --- a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC +++ b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC @@ -28,7 +28,7 @@ node('restricted-mxnetlinux-cpu') { utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( -handler: { +core_logic: { stage('MBCC Train'){ node(NODE_LINUX_CPU) { ws('workspace/modelBackwardsCompat') {