From 242e3cf85dd4d69426567c298030374549e427fe Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Thu, 11 Jan 2024 18:06:56 -0500 Subject: [PATCH 01/18] Begin work on publishing Add Calculate Version to Jenkins Calls get_new_version.py, which replaces commit_type_check.sh Add Build Release to the normal build Add Publish, which calls build.sh --publish for each distro Add Publish Version, which will push the git tags in the future --- Jenkinsfile | 75 +++++++++++++++++++++++++++++++++++++++ deploy/get_new_version.py | 59 ++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 deploy/get_new_version.py diff --git a/Jenkinsfile b/Jenkinsfile index 6c753b3653..0d283271bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,6 +50,7 @@ pipeline { } stages { + stage('Setup') { steps { sh 'printenv' @@ -78,6 +79,32 @@ pipeline { cleanWs disableDeferredWipeout: true, deleteDirs: true } } + + stage("Calculate Version") { + // when { + // allOf { + // anyOf { + // branch 'alpha'; + // branch 'stable'; + // } + + // triggeredBy 'TimerTrigger' + // } + // } + steps { + ws("${WORKSPACE}/publish") { + checkout scm; + + env.VERSION = sh( + script: "./deploy/get-new-version.py", + returnStdout: true + ).trim() + + echo "Calculated new version to be \$VERSION" + } + } + } + stage('Distributions') { steps { script { @@ -109,6 +136,15 @@ pipeline { echo "Testing complete" } } + + stage("${OS} Build Release") { + if (env.VERSION) { + sh "./deploy/build.sh --os=${OS} --release=\$VERSION" + } + else { + sh "./deploy/build.sh --os=${OS} --release" + } + } } } }] @@ -143,6 +179,45 @@ pipeline { } } } + + // TODO: Only publish when the $VERSION is newer than the last release + stage('Publish') { + // when { + // allOf { + // anyOf { + // branch 'alpha'; + // branch 'stable'; + // } + + // triggeredBy 'TimerTrigger' + // } + // } + steps { + script { + parallel OSList.collectEntries { + OS -> [ "${OS}": { + stage("${OS}") { + ws("${WORKSPACE}/${OS}") { + + stage("${OS} Publish") { + sh "./deploy/build.sh --os=${OS} --publish=\$VERSION --publishdir=/tmp/publish" + } + } + }] + } + } + + stage("Publish Version") { + ws("${WORKSPACE}/publish") { + def tag = "${BRANCH_NAME}_release-" + env.VERSION.replaceAll(".", "-") + sh "git tag ${tag}" + + echo "Publishing tag ${tag}" + // git push --tags + } + } + } + } } post { always { diff --git a/deploy/get_new_version.py b/deploy/get_new_version.py new file mode 100644 index 0000000000..d1d334128d --- /dev/null +++ b/deploy/get_new_version.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import subprocess +import shutil + +git_executable = shutil.which('git') + +def git(command): + # print(f'git {command}') + proc = subprocess.Popen( + [ git_executable ] + command.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + + stdout, _ = proc.communicate() + return stdout.decode().strip() + +last_release = git('describe --tags --abbrev=0') +last_release_commit = git(f'rev-list -n 1 {last_release}') +commit_log = git(f'log {last_release_commit}..HEAD --no-merges --decorate=short --pretty=format:%s') + +version_bump = 'SAME' +for commit in commit_log.splitlines(): + commit = commit.lower() + + if commit.startswith('feature') or commit.startswith('revert "feature'): + version_bump = 'MINOR' + + elif commit.startswith('fix') or commit.startswith('revert "fix'): + if version_bump != 'MINOR': + version_bump = 'PATCH' + + elif commit.startswith('tests') or commit.startswith('revert "tests'): + pass + + elif commit.startswith('build') or commit.startswith('revert "build'): + pass + + elif commit.startswith('docs') or commit.startswith('revert "docs'): + pass + +version_parts = last_release.split('-') +if len(version_parts) < 4: + print('1.2.3') + exit(1) +else: + major = int(version_parts[-3]) + minor = int(version_parts[-2]) + patch = int(version_parts[-1]) + +if version_bump == 'MAJOR': + major += 1 +elif version_bump == 'MINOR': + minor += 1 +elif version_bump == 'PATCH': + patch += 1 + +print(f'{major}.{minor}.{patch}') \ No newline at end of file From 509397b8a280dab16ca69ef093258da4626a0409 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Thu, 11 Jan 2024 18:13:19 -0500 Subject: [PATCH 02/18] Groovy syntax error --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d283271bd..d5abd9ea9f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -198,9 +198,9 @@ pipeline { OS -> [ "${OS}": { stage("${OS}") { ws("${WORKSPACE}/${OS}") { - - stage("${OS} Publish") { - sh "./deploy/build.sh --os=${OS} --publish=\$VERSION --publishdir=/tmp/publish" + stage("${OS} Publish") { + sh "./deploy/build.sh --os=${OS} --publish=\$VERSION --publishdir=/tmp/publish" + } } } }] From f9bf522a5e2ba77f8628475667ca62870317b503 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 10:53:15 -0500 Subject: [PATCH 03/18] Groovy syntax error --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d5abd9ea9f..209b2f1f96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -205,15 +205,15 @@ pipeline { } }] } - } - stage("Publish Version") { - ws("${WORKSPACE}/publish") { - def tag = "${BRANCH_NAME}_release-" + env.VERSION.replaceAll(".", "-") - sh "git tag ${tag}" + stage("Publish Version") { + ws("${WORKSPACE}/publish") { + def tag = "${BRANCH_NAME}_release-" + env.VERSION.replaceAll(".", "-") + sh "git tag ${tag}" - echo "Publishing tag ${tag}" - // git push --tags + echo "Publishing tag ${tag}" + // git push --tags + } } } } From 36416b976d7ab4971cddc4455f1fbeca640f426e Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 10:54:15 -0500 Subject: [PATCH 04/18] Groovy syntax error --- Jenkinsfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 209b2f1f96..0025821d01 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,14 +93,16 @@ pipeline { // } steps { ws("${WORKSPACE}/publish") { - checkout scm; + script { + checkout scm; - env.VERSION = sh( - script: "./deploy/get-new-version.py", - returnStdout: true - ).trim() + env.VERSION = sh( + script: "./deploy/get-new-version.py", + returnStdout: true + ).trim() - echo "Calculated new version to be \$VERSION" + echo "Calculated new version to be \$VERSION" + } } } } From 4216675873073096f62300b0b7fcd86aff3204fb Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 10:58:04 -0500 Subject: [PATCH 05/18] Typo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0025821d01..e26a6641f0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,7 +97,7 @@ pipeline { checkout scm; env.VERSION = sh( - script: "./deploy/get-new-version.py", + script: "./deploy/get_new_version.py", returnStdout: true ).trim() From ec8bcfbb1933245f728a09d5aa4ee8d6a70728d1 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 11:01:55 -0500 Subject: [PATCH 06/18] chmod +x --- deploy/get_new_version.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 deploy/get_new_version.py diff --git a/deploy/get_new_version.py b/deploy/get_new_version.py old mode 100644 new mode 100755 From f5cb6b2584e41fd190086deb0388647e791b9a9b Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 11:12:45 -0500 Subject: [PATCH 07/18] Don't build release test-* --- Jenkinsfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e26a6641f0..4a02e93023 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { returnStdout: true ).trim() - echo "Calculated new version to be \$VERSION" + echo "Calculated new version to be ${VERSION}" } } } @@ -139,12 +139,14 @@ pipeline { } } - stage("${OS} Build Release") { - if (env.VERSION) { - sh "./deploy/build.sh --os=${OS} --release=\$VERSION" - } - else { - sh "./deploy/build.sh --os=${OS} --release" + if (!OS.startsWith("test-")) { + stage("${OS} Build Release") { + if (env.VERSION) { + sh "./deploy/build.sh --os=${OS} --release=\$VERSION" + } + else { + sh "./deploy/build.sh --os=${OS} --release" + } } } } From cf38b855656c683fba1ec361921f655277150dd3 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 11:40:41 -0500 Subject: [PATCH 08/18] Restructure release flow in Jenkinsfile Filter out test-* distros Move Calculate Version down to the publishing block --- Jenkinsfile | 69 ++++++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4a02e93023..341a960070 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,33 +80,6 @@ pipeline { } } - stage("Calculate Version") { - // when { - // allOf { - // anyOf { - // branch 'alpha'; - // branch 'stable'; - // } - - // triggeredBy 'TimerTrigger' - // } - // } - steps { - ws("${WORKSPACE}/publish") { - script { - checkout scm; - - env.VERSION = sh( - script: "./deploy/get_new_version.py", - returnStdout: true - ).trim() - - echo "Calculated new version to be ${VERSION}" - } - } - } - } - stage('Distributions') { steps { script { @@ -139,14 +112,9 @@ pipeline { } } - if (!OS.startsWith("test-")) { - stage("${OS} Build Release") { - if (env.VERSION) { - sh "./deploy/build.sh --os=${OS} --release=\$VERSION" - } - else { - sh "./deploy/build.sh --os=${OS} --release" - } + if (!env.VERSION && !OS.startsWith("test-")) { + stage("${OS} Test Packaging") { + sh "./deploy/build.sh --os=${OS} --release" } } } @@ -198,12 +166,33 @@ pipeline { // } steps { script { + stage("Calculate Version") { + ws("${WORKSPACE}/publish") { + script { + checkout scm; + + def new_version = sh( + script: "./deploy/get_new_version.py", + returnStdout: true + ).trim() + + echo "Calculated new version to be ${new_version}" + } + } + } + parallel OSList.collectEntries { OS -> [ "${OS}": { - stage("${OS}") { - ws("${WORKSPACE}/${OS}") { - stage("${OS} Publish") { - sh "./deploy/build.sh --os=${OS} --publish=\$VERSION --publishdir=/tmp/publish" + if (!OS.startsWith("test-")) { + stage("${OS}") { + ws("${WORKSPACE}/${OS}") { + stage("${OS} Release") { + sh "./deploy/build.sh --os=${OS} --release=${new_version}" + } + + stage("${OS} Publish") { + sh "./deploy/build.sh --os=${OS} --publish=${new_version} --publishdir=/tmp/publish" + } } } } @@ -212,7 +201,7 @@ pipeline { stage("Publish Version") { ws("${WORKSPACE}/publish") { - def tag = "${BRANCH_NAME}_release-" + env.VERSION.replaceAll(".", "-") + def tag = "${BRANCH_NAME}_release-" + new_version.replaceAll(".", "-") sh "git tag ${tag}" echo "Publishing tag ${tag}" From ff293a47326713edb1ce3b04b5f95bc8cc353276 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 13:56:19 -0500 Subject: [PATCH 09/18] Groovy weirdness --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 341a960070..355d49f716 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ pipeline { script { parallel OSList.collectEntries { OS -> [ "${OS}": { - stage("${OS}") { + stage("${OS} Build & Test") { ws("${WORKSPACE}/${OS}") { stage("${OS} Clone") { checkout scm; @@ -182,9 +182,9 @@ pipeline { } parallel OSList.collectEntries { - OS -> [ "${OS}": { - if (!OS.startsWith("test-")) { - stage("${OS}") { + if (!OS.startsWith("test-")) { + OS -> [ "${OS}": { + stage("${OS} Release & Publish") { ws("${WORKSPACE}/${OS}") { stage("${OS} Release") { sh "./deploy/build.sh --os=${OS} --release=${new_version}" @@ -195,8 +195,8 @@ pipeline { } } } - } - }] + }] + } } stage("Publish Version") { From 6c372eab1e4a323651c713d3d633726cec56138c Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 13:59:39 -0500 Subject: [PATCH 10/18] Groovy weirdness --- Jenkinsfile | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 355d49f716..3dccb6593a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -181,22 +181,20 @@ pipeline { } } - parallel OSList.collectEntries { - if (!OS.startsWith("test-")) { - OS -> [ "${OS}": { - stage("${OS} Release & Publish") { - ws("${WORKSPACE}/${OS}") { - stage("${OS} Release") { - sh "./deploy/build.sh --os=${OS} --release=${new_version}" - } + parallel OSList.findAll{ (!var.startsWith("test-")) }.collectEntries { + OS -> [ "${OS}": { + stage("${OS} Release & Publish") { + ws("${WORKSPACE}/${OS}") { + stage("${OS} Release") { + sh "./deploy/build.sh --os=${OS} --release=${new_version}" + } - stage("${OS} Publish") { - sh "./deploy/build.sh --os=${OS} --publish=${new_version} --publishdir=/tmp/publish" - } + stage("${OS} Publish") { + sh "./deploy/build.sh --os=${OS} --publish=${new_version} --publishdir=/tmp/publish" } } - }] - } + } + }] } stage("Publish Version") { From 7bb677f112dd1e2cbd89c46560ce3e868922600a Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 14:01:36 -0500 Subject: [PATCH 11/18] Groovy weirdness --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3dccb6593a..dccf3d2c56 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ pipeline { steps { script { parallel OSList.collectEntries { - OS -> [ "${OS}": { + OS -> [ "${OS} Build & Test": { stage("${OS} Build & Test") { ws("${WORKSPACE}/${OS}") { stage("${OS} Clone") { @@ -182,7 +182,7 @@ pipeline { } parallel OSList.findAll{ (!var.startsWith("test-")) }.collectEntries { - OS -> [ "${OS}": { + OS -> [ "${OS} Release & Publish": { stage("${OS} Release & Publish") { ws("${WORKSPACE}/${OS}") { stage("${OS} Release") { From 98e8dd6d3beb249b6ec46054e0187d3dafed1836 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 14:06:30 -0500 Subject: [PATCH 12/18] Only test packaging for PRs --- Jenkinsfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dccf3d2c56..bd1dc34595 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -100,19 +100,19 @@ pipeline { } stage("${OS} Test") { - try { - def network = "jenkins-${EXECUTOR_NUMBER}-${OS}" - sh "./deploy/build.sh --os=${OS} --test --dockernetwork=${network}" - } - finally { - sh "./deploy/tap-to-junit.py --junit-suite-name=${OS}" - junit skipPublishingChecks: true, testResults: 'mdsplus-junit.xml', keepLongStdio: true - - echo "Testing complete" - } + // try { + // def network = "jenkins-${EXECUTOR_NUMBER}-${OS}" + // sh "./deploy/build.sh --os=${OS} --test --dockernetwork=${network}" + // } + // finally { + // sh "./deploy/tap-to-junit.py --junit-suite-name=${OS}" + // junit skipPublishingChecks: true, testResults: 'mdsplus-junit.xml', keepLongStdio: true + + // echo "Testing complete" + // } } - if (!env.VERSION && !OS.startsWith("test-")) { + if (env.CHANGE_ID && !OS.startsWith("test-")) { stage("${OS} Test Packaging") { sh "./deploy/build.sh --os=${OS} --release" } From 36df9377d998ac710e124c9992ecc5289d6fdf5e Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 14:07:01 -0500 Subject: [PATCH 13/18] Disable slow testing temporarily --- Jenkinsfile | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd1dc34595..c9b3ff6ac5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -125,32 +125,32 @@ pipeline { } } - stage('Additional Testing') { - parallel { - stage("Test IDL") { - steps { - // The IDL tests have to be run with the same OS as the builder - ws("${WORKSPACE}/ubuntu22") { - withEnv(["MDSPLUS_DIR=${WORKSPACE}/tests/64/buildroot"]) { - sh """ - set +x - . \$MDSPLUS_DIR/setup.sh - set -x - ./idl/testing/run_tests.py - """ - } - } - } - } - - stage("Test MATLAB") { - steps { - echo "Testing MATLAB" - // TODO - } - } - } - } + // stage('Additional Testing') { + // parallel { + // stage("Test IDL") { + // steps { + // // The IDL tests have to be run with the same OS as the builder + // ws("${WORKSPACE}/ubuntu22") { + // withEnv(["MDSPLUS_DIR=${WORKSPACE}/tests/64/buildroot"]) { + // sh """ + // set +x + // . \$MDSPLUS_DIR/setup.sh + // set -x + // ./idl/testing/run_tests.py + // """ + // } + // } + // } + // } + + // stage("Test MATLAB") { + // steps { + // echo "Testing MATLAB" + // // TODO + // } + // } + // } + // } // TODO: Only publish when the $VERSION is newer than the last release stage('Publish') { From 5a9d11986c46eccb92d5e4fe8d1ef1a4fdba9a1b Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 14:09:45 -0500 Subject: [PATCH 14/18] Disable slow testing temporarily --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c9b3ff6ac5..6e5a1c2dbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,7 +112,7 @@ pipeline { // } } - if (env.CHANGE_ID && !OS.startsWith("test-")) { + if (false && env.CHANGE_ID && !OS.startsWith("test-")) { stage("${OS} Test Packaging") { sh "./deploy/build.sh --os=${OS} --release" } From c4c977893a3859a30cbbdccbf5bda56f120b2913 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 14:14:53 -0500 Subject: [PATCH 15/18] Groovy syntax error --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6e5a1c2dbe..ff869828de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -181,7 +181,7 @@ pipeline { } } - parallel OSList.findAll{ (!var.startsWith("test-")) }.collectEntries { + parallel OSList.findAll{ OS -> (!OS.startsWith("test-")) }.collectEntries { OS -> [ "${OS} Release & Publish": { stage("${OS} Release & Publish") { ws("${WORKSPACE}/${OS}") { From 1def12bbbebc8a811fb090bb32fdcb2288a1b1a0 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 15:28:33 -0500 Subject: [PATCH 16/18] Groovy syntax error --- Jenkinsfile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ff869828de..3a1f753a95 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -168,16 +168,14 @@ pipeline { script { stage("Calculate Version") { ws("${WORKSPACE}/publish") { - script { - checkout scm; + checkout scm; - def new_version = sh( - script: "./deploy/get_new_version.py", - returnStdout: true - ).trim() + def new_version = sh( + script: "./deploy/get_new_version.py", + returnStdout: true + ).trim() - echo "Calculated new version to be ${new_version}" - } + echo "Calculated new version to be ${new_version}" } } From 7bd28971dc3f3cc7608e5933dd3b697f93fe6025 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 12 Jan 2024 16:02:58 -0500 Subject: [PATCH 17/18] Groovy syntax error --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3a1f753a95..d5f2bf88ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -166,11 +166,13 @@ pipeline { // } steps { script { + def new_version = '' + stage("Calculate Version") { ws("${WORKSPACE}/publish") { checkout scm; - def new_version = sh( + new_version = sh( script: "./deploy/get_new_version.py", returnStdout: true ).trim() From 7418d7bd6fe0b867581257a59be6f0bfa9c064b3 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Wed, 17 Jan 2024 17:38:05 -0500 Subject: [PATCH 18/18] Hopefully add support for releases All debian and redhat platforms will also generate tgz files for the GitHub release One tgz for /usr/local/mdsplus, and one for all the debs or rpms Add the publishing step, which will: * Calculate the new version * Build a release with that version for each distro * Publish each distro * Create a github release --- Jenkinsfile | 127 ++++++++++-------- deploy/create_github_release.py | 96 +++++++++++++ deploy/get_new_version.py | 2 +- deploy/platform/debian/debian_docker_build.sh | 13 +- deploy/platform/platform_build.sh | 4 +- deploy/platform/redhat/redhat_docker_build.sh | 13 +- 6 files changed, 194 insertions(+), 61 deletions(-) create mode 100755 deploy/create_github_release.py diff --git a/Jenkinsfile b/Jenkinsfile index d5f2bf88ac..05f858d099 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -100,19 +100,19 @@ pipeline { } stage("${OS} Test") { - // try { - // def network = "jenkins-${EXECUTOR_NUMBER}-${OS}" - // sh "./deploy/build.sh --os=${OS} --test --dockernetwork=${network}" - // } - // finally { - // sh "./deploy/tap-to-junit.py --junit-suite-name=${OS}" - // junit skipPublishingChecks: true, testResults: 'mdsplus-junit.xml', keepLongStdio: true - - // echo "Testing complete" - // } + try { + def network = "jenkins-${EXECUTOR_NUMBER}-${OS}" + sh "./deploy/build.sh --os=${OS} --test --dockernetwork=${network}" + } + finally { + sh "./deploy/tap-to-junit.py --junit-suite-name=${OS}" + junit skipPublishingChecks: true, testResults: 'mdsplus-junit.xml', keepLongStdio: true + + echo "Testing complete" + } } - if (false && env.CHANGE_ID && !OS.startsWith("test-")) { + if (env.CHANGE_ID && !OS.startsWith("test-")) { stage("${OS} Test Packaging") { sh "./deploy/build.sh --os=${OS} --release" } @@ -125,48 +125,47 @@ pipeline { } } - // stage('Additional Testing') { - // parallel { - // stage("Test IDL") { - // steps { - // // The IDL tests have to be run with the same OS as the builder - // ws("${WORKSPACE}/ubuntu22") { - // withEnv(["MDSPLUS_DIR=${WORKSPACE}/tests/64/buildroot"]) { - // sh """ - // set +x - // . \$MDSPLUS_DIR/setup.sh - // set -x - // ./idl/testing/run_tests.py - // """ - // } - // } - // } - // } - - // stage("Test MATLAB") { - // steps { - // echo "Testing MATLAB" - // // TODO - // } - // } - // } - // } - - // TODO: Only publish when the $VERSION is newer than the last release + stage('Additional Testing') { + parallel { + stage("Test IDL") { + steps { + // The IDL tests have to be run with the same OS as the builder + ws("${WORKSPACE}/ubuntu22") { + withEnv(["MDSPLUS_DIR=${WORKSPACE}/tests/64/buildroot"]) { + sh """ + set +x + . \$MDSPLUS_DIR/setup.sh + set -x + ./idl/testing/run_tests.py + """ + } + } + } + } + + stage("Test MATLAB") { + steps { + echo "Testing MATLAB" + // TODO + } + } + } + } + stage('Publish') { - // when { - // allOf { - // anyOf { - // branch 'alpha'; - // branch 'stable'; - // } - - // triggeredBy 'TimerTrigger' - // } - // } + when { + allOf { + anyOf { + branch 'alpha'; + branch 'stable'; + } + + triggeredBy 'TimerTrigger' + } + } steps { script { - def new_version = '' + def new_version = '0.0.0' stage("Calculate Version") { ws("${WORKSPACE}/publish") { @@ -177,16 +176,26 @@ pipeline { returnStdout: true ).trim() + if (new_version == '0.0.0') { + error "Failed to calculate new version" + } + echo "Calculated new version to be ${new_version}" } } + release_file_list = [] + parallel OSList.findAll{ OS -> (!OS.startsWith("test-")) }.collectEntries { OS -> [ "${OS} Release & Publish": { stage("${OS} Release & Publish") { ws("${WORKSPACE}/${OS}") { stage("${OS} Release") { sh "./deploy/build.sh --os=${OS} --release=${new_version}" + + findFiles(glob: "tarfiles/*.tgz").each { + file -> release_file_list.add(file.path) + } } stage("${OS} Publish") { @@ -199,11 +208,21 @@ pipeline { stage("Publish Version") { ws("${WORKSPACE}/publish") { - def tag = "${BRANCH_NAME}_release-" + new_version.replaceAll(".", "-") - sh "git tag ${tag}" + def tag = "${BRANCH_NAME}_release-" + new_version.replaceAll("\\.", "-") + + echo "Creating GitHub Release and Tag for ${tag}" + withCredentials([ + usernamePassword( + credentialsId: 'MDSplusJenkins', + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + )]) { + + // TODO: Protect against spaces in filenames + def release_file_list_arg = release_file_list.join(" ") + sh "./deploy/create_github_release.py --tag ${tag} --api-token \$GITHUB_ACCESS_TOKEN ${release_file_list_arg}" + } - echo "Publishing tag ${tag}" - // git push --tags } } } diff --git a/deploy/create_github_release.py b/deploy/create_github_release.py new file mode 100755 index 0000000000..0634aa96ac --- /dev/null +++ b/deploy/create_github_release.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +import requests +import argparse +import subprocess +import shutil + +API_URL = 'https://api.github.com/repos/MDSplus/mdsplus' +UPLOAD_URL = 'https://uploads.github.com/repos/MDSplus/mdsplus' + +parser = argparse.ArgumentParser() + +parser.add_argument( + '--github-name', + default='MDSplusBuilder' +) + +parser.add_argument( + '--github-email', + default='mdsplusadmin@psfc.mit.edu' +) + +parser.add_argument( + '--tag', + required=True +) + +parser.add_argument( + '--api-token', + required=True +) + +parser.add_argument( + 'files', + nargs=argparse.REMAINDER +) + +args = parser.parse_args() + +git_executable = shutil.which('git') + +result = subprocess.run( + [ git_executable, 'rev-parse', args.tag ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT +) + +if result.returncode == 0: + print(f"The tag {args.tag} already exists, exiting") + exit(0) + +result = subprocess.run( + [ git_executable, 'rev-parse', 'HEAD'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT +) + +git_hash = result.stdout.decode().strip() + +headers = { + 'Accept': 'application/vnd.github+json', + 'Authorization': f'Bearer {args.api_token}', + 'X-GitHub-Api-Version': '2022-11-28', # TODO: Is this needed? +} + +print('Creating release') + +create_release = { + 'tag_name': args.tag, + 'target_commitish': git_hash, + # This will automatically generate the name and body of the release + 'generate_release_notes': True +} + +create_release_response = requests.post(f'{API_URL}/releases', json=create_release, headers=headers) +if create_release_response.status_code != 201: + print(create_release_response.content.decode()) + exit(1) + +release_id = create_release_response.json()['id'] + +asset_headers = headers +asset_headers['Content-Type'] = 'application/octet-stream' + +for file in args.files: + + file_name = os.path.basename(file) + data = open(file, 'rb').read() + + upload_release_asset_response = requests.post(f'{UPLOAD_URL}/releases/{release_id}/assets?name={file_name}', data=data, headers=asset_headers) + print(upload_release_asset_response.request.headers) + print(upload_release_asset_response.request.url) + if upload_release_asset_response.status_code != 201: + print(upload_release_asset_response.content.decode()) + # attempt to upload the rest of the files diff --git a/deploy/get_new_version.py b/deploy/get_new_version.py index d1d334128d..0f7d76e718 100755 --- a/deploy/get_new_version.py +++ b/deploy/get_new_version.py @@ -42,7 +42,7 @@ def git(command): version_parts = last_release.split('-') if len(version_parts) < 4: - print('1.2.3') + print('0.0.0') exit(1) else: major = int(version_parts[-3]) diff --git a/deploy/platform/debian/debian_docker_build.sh b/deploy/platform/debian/debian_docker_build.sh index 065109d021..477db4ff92 100755 --- a/deploy/platform/debian/debian_docker_build.sh +++ b/deploy/platform/debian/debian_docker_build.sh @@ -65,10 +65,11 @@ buildrelease() { # ${RELEASEDIR}/${FLAVOR}/DEBS will be cleaned in debian_build.sh RELEASEDEBS=/release/${FLAVOR}/DEBS/${ARCH} RELEASEBLD=/workspace/releasebld + TARFILES=/workspace/tarfiles BUILDROOT=${RELEASEBLD}/buildroot MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus - rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT} - mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR} + rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT} ${TARFILES} + mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES} pushd ${RELEASEBLD}/${bits} config ${config_param} ${CONFIGURE_EXTRA} if [ -z "$NOMAKE" ]; then @@ -152,6 +153,14 @@ EOF fi done popd + + pushd ${MDSPLUS_DIR} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz * + popd + + pushd ${RELEASEDEBS} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_debs.tgz *.deb + popd fi #abort fi #nomake } diff --git a/deploy/platform/platform_build.sh b/deploy/platform/platform_build.sh index cf5b158488..db332e1ec9 100755 --- a/deploy/platform/platform_build.sh +++ b/deploy/platform/platform_build.sh @@ -81,8 +81,8 @@ rundocker() { function kill_docker() { if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then - docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) || true - docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) || true + docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true + docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true rm -f ${WORKSPACE}/${OS}_docker-cid fi if [ ! -z $DOCKERNETWORK ]; then diff --git a/deploy/platform/redhat/redhat_docker_build.sh b/deploy/platform/redhat/redhat_docker_build.sh index eebb801195..df8d462887 100755 --- a/deploy/platform/redhat/redhat_docker_build.sh +++ b/deploy/platform/redhat/redhat_docker_build.sh @@ -57,9 +57,10 @@ buildrelease() { set -e RELEASEBLD=/workspace/releasebld BUILDROOT=${RELEASEBLD}/buildroot + TARFILES=/workspace/tarfiles MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus - rm -Rf ${RELEASEBLD} /release/${FLAVOR} - mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR} + rm -Rf ${RELEASEBLD} /release/${FLAVOR} ${TARFILES} + mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES} pushd ${RELEASEBLD}/64 config ${test64} ${CONFIGURE_EXTRA} if [ -z "$NOMAKE" ]; then @@ -132,6 +133,14 @@ EOF fi done checkstatus abort "Failure: Problem with contents of one or more rpms. (see above)" $badrpm + + pushd ${MDSPLUS_DIR} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz * + popd + + pushd /release/${FLAVOR}/RPMS + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_rpms.tgz */*.rpm + popd fi #nomake }