Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
Change all Jenkinsfiles to use utils
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcoabreu
marcoabreu committed Aug 2, 2018
1 parent aff116a commit 4e26ebc
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 343 deletions.
4 changes: 2 additions & 2 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -980,8 +980,8 @@ handler: {
}
}
}
},

}
,
failure_handler: {
// Only send email if master or release branches failed
if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) {
Expand Down
57 changes: 14 additions & 43 deletions ci/Jenkinsfile_docker_cache
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,34 +22,18 @@

// timeout in minutes
total_timeout = 300
git_timeout = 15
// assign any caught errors here
err = null

// initialize source codes
def init_git() {
deleteDir()
retry(5) {
try {
// Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
// retries as this will increase the amount of requests and worsen the throttling
timeout(time: git_timeout, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init --recursive'
sh 'git clean -x -d -f'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes with ${exc}"
sleep 2
}
}
node('restricted-mxnetlinux-cpu') {
// Loading the utilities requires a node context unfortunately
checkout scm
utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_cpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')


try {
utils.main_wrapper(
handler: {
stage("Docker cache build & publish") {
node('restricted-mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/docker_cache') {
timeout(time: total_timeout, unit: 'MINUTES') {
init_git()
Expand All @@ -58,24 +42,11 @@ try {
}
}
}

// set build status to success at the end
currentBuild.result = "SUCCESS"
} catch (caughtError) {
node("restricted-mxnetlinux-cpu") {
sh "echo caught ${caughtError}"
err = caughtError
currentBuild.result = "FAILURE"
}
} finally {
node("restricted-mxnetlinux-cpu") {
// Only send email if master failed
if (currentBuild.result == "FAILURE") {
emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}'
}
// Remember to rethrow so the build is marked as failing
if (err) {
throw err
}
,
failure_handler:
{
if (currentBuild.result == "FAILURE") {
emailext body: 'Generating the Docker Cache has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[DOCKER CACHE FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}'
}
}
)
54 changes: 13 additions & 41 deletions docs/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,32 +22,18 @@

// timeout in minutes
max_time = 60
// assign any caught errors here
err = null

// initialize source code
def init_git() {
deleteDir()
retry(5) {
try {
// Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
// retries as this will increase the amount of requests and worsen the throttling
timeout(time: 15, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init --recursive'
sh 'git clean -d -f'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes with ${exc}"
sleep 2
}
}
node('restricted-mxnetlinux-cpu') {
// Loading the utilities requires a node context unfortunately
checkout scm
utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_cpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')

try {
utils.main_wrapper(
handler: {
stage('Build Docs') {
node('restricted-mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/docs') {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -58,24 +44,10 @@ try {
}
}
}

// set build status to success at the end
currentBuild.result = "SUCCESS"
} catch (caughtError) {
node("restricted-mxnetlinux-cpu") {
sh "echo caught ${caughtError}"
err = caughtError
currentBuild.result = "FAILURE"
}
} finally {
node("restricted-mxnetlinux-cpu") {
// Only send email if master failed
if (currentBuild.result == "FAILURE") {
emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
}
// Remember to rethrow so the build is marked as failing
if (err) {
throw err
}
,
failure_handler: {
if (currentBuild.result == "FAILURE") {
emailext body: 'Generating the website has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[WEBSITE FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
}
}
)
95 changes: 23 additions & 72 deletions tests/nightly/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,70 +15,31 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.


//
//This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile

err = null
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a'

// pack libraries for later use
def pack_lib(name, libs=mx_lib) {
sh """
echo "Packing ${libs} into ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
"""
stash includes: libs, name: name
}

// unpack libraries saved before
def unpack_lib(name, libs=mx_lib) {
unstash name
sh """
echo "Unpacked ${libs} from ${name}"
echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
"""
}

def init_git() {
deleteDir()
retry(5) {
try {
timeout(time: 15, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init --recursive'
sh 'git clean -d -f'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes with ${exc}"
sleep 2
}
}
}

def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
command = command.replaceAll('%PLATFORM%', platform)
command = command.replaceAll('%FUNCTION_NAME%', function_name)
command = command.replaceAll('%SHARED_MEM%', shared_mem)

sh command
node('mxnetlinux-cpu') {
// Loading the utilities requires a node context unfortunately
checkout scm
utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_cpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')

try {
utils.main_wrapper(
handler: {
stage('NightlyTests'){
parallel 'CompilationWarnings: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-compilationTest') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_compilation_warning', false)
}
}
},
'InstallationGuide: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-Installation-cpu') {
init_git()
//Some install guide tests are currently diabled and tracked here:
Expand All @@ -92,7 +53,7 @@ try {
}
},
'InstallationGuide: GPU': {
node('mxnetlinux-gpu') {
node(NODE_LINUX_GPU) {
ws('workspace/nt-Installation-gpu') {
init_git()
//Some install guide tests are currently diabled and tracked here:
Expand All @@ -106,76 +67,66 @@ try {
}
},
'PipTest: GPU': {
node('mxnetlinux-gpu') {
node(NODE_LINUX_GPU) {
ws('workspace/nt-pipTest') {
init_git()
}
}
},
'Amalgamation-atlas: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-amalgamation1') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas', false)
}
}
},
'Amalgamation-atlas-min: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-amalgamation2') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MIN=1', false)
}
}
},
'Amalgamation-atlas-mkl: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-amalgamation3') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_MKL=1', false)
}
}
},
'Amalgamation-atlas-cuda: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-amalgamation4') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas MSHADOW_USE_CUDA=1', false)
}
}
},
'Amalgamation-atlas-openmp: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-amalgamation5') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_amalgamation USE_BLAS=atlas DISABLE_OPENMP=0', false)
}
}
},
'MXNetJS: CPU': {
node('mxnetlinux-cpu') {
node(NODE_LINUX_CPU) {
ws('workspace/nt-mxnetjs') {
init_git()
docker_run('ubuntu_nightly_cpu', 'nightly_test_javascript', false)
}
}
}
}
} catch (caughtError) {
node("mxnetlinux-cpu") {
sh "echo caught ${caughtError}"
err = caughtError
currentBuild.result = "FAILURE"
}
} finally {
node("mxnetlinux-cpu") {
// Only send email if nightly test failed
if (currentBuild.result == "FAILURE") {
,
failure_handler: {
// Only send email if nightly test failed
if (currentBuild.result == "FAILURE") {
emailext body: 'Nightly tests for MXNet branch ${BRANCH_NAME} failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY TEST FAILED] build ${BUILD_NUMBER}', to: '${EMAIL}'
}
// Remember to rethrow so the build is marked as failing
if (err) {
throw err
}
}
}
)
Loading

0 comments on commit 4e26ebc

Please sign in to comment.