From 167ff22ff33ad7426f5a50962868b8533cb1d9aa Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Wed, 3 Jan 2024 16:35:29 -0500 Subject: [PATCH 1/3] Initial Jenkinsfile Add a basic Jenkinsfile that will build and test Includes fixes for deploy/*.sh scripts that didn't work on the new jenkins --- Jenkinsfile | 109 +++++++++++++++++++ deploy/git_revision.sh | 4 +- deploy/platform/bootstrap/bootstrap_build.sh | 3 +- deploy/platform/platform_build.sh | 26 ++++- 4 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..01dd993ff3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,109 @@ +@Library('camunda-community') _ + +def OSList = [ + 'windows', + 'ubuntu18', 'ubuntu20', 'ubuntu22', + 'rhel7', 'rhel8', 'rhel9', + // 'alpine3.9-armhf', 'alpine3.9-x86_64', 'alpine3.9-x86', + 'debian9-64', 'debian10-64', 'debian11-64', + 'test-asan', 'test-tsan', 'test-ubsan', +] + +def AdminList = [ + 'AndreaRigoni', + 'GabrieleManduchi', + 'joshStillerman', + 'mwinkel-dev', + 'santorofer', + 'tfredian', + 'WhoBrokeTheBuild', + 'zack-vii', +] + +def schedule = ""; +if (BRANCH_NAME == "alpha") { + schedule = "0 18 * * *"; +} +if (BRANCH_NAME == "stable") { + schedule = "0 19 * * *"; +} + +pipeline { + agent any + options { + skipDefaultCheckout() + timeout(time: 1, unit: 'HOURS') + } + triggers { + cron(schedule) + issueCommentTrigger('(?i).*retest\\s+this\\s+please.*') + } + + stages { + stage('Setup') { + steps { + sh 'printenv' + echo schedule + + script { + + // is PR + if (env.CHANGE_ID) { + // This is safe because untrusted PRs will use Jenkinsfile from the target branch + if (env.GITHUB_COMMENT_AUTHOR) { + if (!AdminList.contains(env.GITHUB_COMMENT_AUTHOR)) { + currentBuild.result = 'ABORTED' + error 'This user does not have permission to trigger builds.' + } + else { + echo("Build was started by ${GITHUB_COMMENT_AUTHOR}, who wrote: \"${GITHUB_COMMENT}\", which matches the trigger pattern.") + } + } + else if (!AdminList.contains(env.CHANGE_AUTHOR)) { + currentBuild.result = 'ABORTED' + error 'This user does not have permission to trigger builds.' + } + } + } + + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } + stage('Distributions') { + steps { + dynamicMatrix([ + failFast: false, + axes: [ + OS: OSList + ], + actions: { + ws("${WORKSPACE}/${OS}") { + + stage("${OS} Clone") { + checkout scm; + } + + stage("${OS} Bootstrap") { + sh "GIT_BRANCH=${BRANCH_NAME} ./deploy/build.sh --os=bootstrap" + + if (env.OS.endsWith("armhf")) { + sh "docker run --rm --privileged multiarch/qemu-user-static:register --reset" + } + } + + stage("${OS} Test") { + sh "./deploy/build.sh --os=${OS} --test --eventport=\$((4100+\${EXECUTOR_NUMBER}))" + + // TODO: Why does this hang on windows? + if (env.OS != "windows") { + archiveArtifacts artifacts: 'tests/**/*.log,tests/**/test-suite.tap,tests/**/core' + } + } + } + } + ]) + } + } + } +} + diff --git a/deploy/git_revision.sh b/deploy/git_revision.sh index 9becd0d4d3..3d65018444 100755 --- a/deploy/git_revision.sh +++ b/deploy/git_revision.sh @@ -2,7 +2,9 @@ SRC=$(realpath "$(dirname "$0")/..") GIT="git --git-dir=${SRC}/.git --work-tree=${SRC}" GIT_TAG="$(${GIT} describe --tag)" -GIT_BRANCH="$(${GIT} rev-parse --abbrev-ref HEAD)" +if [ -z "${GIT_BRANCH}" ]; then + GIT_BRANCH="$(${GIT} rev-parse --abbrev-ref HEAD)" +fi GIT_REMOTE="$(${GIT} config branch.${GIT_BRANCH}.remote)" GIT_REMOTE_URL="$(${GIT} config remote.${GIT_REMOTE}.url)" GIT_COMMIT="$(${GIT} rev-parse HEAD)" diff --git a/deploy/platform/bootstrap/bootstrap_build.sh b/deploy/platform/bootstrap/bootstrap_build.sh index 805a1a0d36..89b6989494 100755 --- a/deploy/platform/bootstrap/bootstrap_build.sh +++ b/deploy/platform/bootstrap/bootstrap_build.sh @@ -1,12 +1,13 @@ #!/bin/bash # # runs $srcdir/bootstrap in a controlled manner -cid=/tmp/bootstrap-docker-cid +cid=$(mktemp -d)/bootstrap-docker-cid cleanup() { if [ -f $cid ] then docker rm -f $(cat $cid) rm -f $cid + rmdir $(dirname $cid) fi } trap cleanup EXIT INT diff --git a/deploy/platform/platform_build.sh b/deploy/platform/platform_build.sh index 99ab8cc642..d3fed16aee 100755 --- a/deploy/platform/platform_build.sh +++ b/deploy/platform/platform_build.sh @@ -46,7 +46,8 @@ rundocker() { VALGRIND_TOOLS="$(spacedelim $VALGRIND_TOOLS)" idx=0 if [ -z "$INTERACTIVE" ]; then - stdio="-a stdout -a stderr" + # stdio="-a stdout -a stderr" + stdio="--detach" program="${DOCKER_SRCDIR}/deploy/platform/platform_docker_build.sh" else stdio="-i" @@ -90,6 +91,22 @@ rundocker() { DOCKER_NETWORK=bridge NETWORK= fi + + function kill_docker() { + if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then + docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) || true + docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) || true + rm -f ${WORKSPACE}/${OS}_docker-cid + fi + } + + function abort() { + kill_docker + status=1 + } + + trap abort SIGINT + status=127 loop_count=0 while [ $status = 127 -a $loop_count -lt 5 ]; do @@ -131,10 +148,9 @@ rundocker() { $(volume "${KEYS}" /sign_keys) \ ${image} $program status=$? - if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then - sleep 3 - docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) - rm -f ${WORKSPACE}/${OS}_docker-cid + + if [ -z "$INTERACTIVE" ]; then + docker logs -f $(cat ${WORKSPACE}/${OS}_docker-cid) fi done if [ -z ${DOCKER_NETWORK} ]; then From 5df41b96c33abfa6c20e6a508df860ce92149a56 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Wed, 3 Jan 2024 18:16:26 -0500 Subject: [PATCH 2/3] Add build badge to README, add --rm to platform_build.sh --- README.md | 2 ++ deploy/platform/platform_build.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 1d699ee07f..2511314975 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![Build Status](https://jenkins.mdsplus.org/buildStatus/icon?job=MDSplus%2Falpha) + ## We want to know who you are The MDSplus developers want to know who you are. If you or your site is using MDSplus please fill the following survey. We promise not to share your contact information. Do not assume that other's from your institution have also filled this out, we will combine the results. diff --git a/deploy/platform/platform_build.sh b/deploy/platform/platform_build.sh index d3fed16aee..c7e9103bcb 100755 --- a/deploy/platform/platform_build.sh +++ b/deploy/platform/platform_build.sh @@ -111,7 +111,11 @@ rundocker() { loop_count=0 while [ $status = 127 -a $loop_count -lt 5 ]; do let loop_count=$loop_count+1 + + kill_docker + docker run --cap-add=SYS_PTRACE -t $stdio \ + --rm \ --cidfile=${WORKSPACE}/${OS}_docker-cid \ ${NETWORK} \ -u $(id -u):$(id -g) --privileged -h $DISTNAME -e "srcdir=${DOCKER_SRCDIR}" \ From 1a33f520050ade80e74ac07ee616f2f989f28369 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Wed, 3 Jan 2024 19:15:36 -0500 Subject: [PATCH 3/3] Remove docker networks from build.sh --- Jenkinsfile | 1 - deploy/platform/platform_build.sh | 17 ----------------- 2 files changed, 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 01dd993ff3..dbb3347009 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,6 @@ pipeline { stage('Setup') { steps { sh 'printenv' - echo schedule script { diff --git a/deploy/platform/platform_build.sh b/deploy/platform/platform_build.sh index c7e9103bcb..ff7d8a832c 100755 --- a/deploy/platform/platform_build.sh +++ b/deploy/platform/platform_build.sh @@ -78,19 +78,6 @@ rundocker() { port_forwarding="-p ${FORWARD_PORT}:${FORWARD_PORT}" echo $port_forwarding fi - if docker network >/dev/null 2>&1; then - #docker supports --network - if [ -z ${DOCKER_NETWORK} ]; then - docker network rm ${OS} || : - docker network create ${OS} - NETWORK=--network=${OS} - else - NETWORK=--network=${DOCKER_NETWORK} - fi - else - DOCKER_NETWORK=bridge - NETWORK= - fi function kill_docker() { if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then @@ -117,7 +104,6 @@ rundocker() { docker run --cap-add=SYS_PTRACE -t $stdio \ --rm \ --cidfile=${WORKSPACE}/${OS}_docker-cid \ - ${NETWORK} \ -u $(id -u):$(id -g) --privileged -h $DISTNAME -e "srcdir=${DOCKER_SRCDIR}" \ -e "ARCH=${arch}" \ -e "ARCHES=${ARCH}" \ @@ -157,9 +143,6 @@ rundocker() { docker logs -f $(cat ${WORKSPACE}/${OS}_docker-cid) fi done - if [ -z ${DOCKER_NETWORK} ]; then - docker network rm ${OS} - fi if [ ! "$status" = "0" ]; then RED cat <&2