diff --git a/Jenkinsfile b/Jenkinsfile index 6c753b3653..05f858d099 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,6 +50,7 @@ pipeline { } stages { + stage('Setup') { steps { sh 'printenv' @@ -78,12 +79,13 @@ pipeline { cleanWs disableDeferredWipeout: true, deleteDirs: true } } + stage('Distributions') { steps { script { parallel OSList.collectEntries { - OS -> [ "${OS}": { - stage("${OS}") { + OS -> [ "${OS} Build & Test": { + stage("${OS} Build & Test") { ws("${WORKSPACE}/${OS}") { stage("${OS} Clone") { checkout scm; @@ -109,6 +111,12 @@ pipeline { echo "Testing complete" } } + + if (env.CHANGE_ID && !OS.startsWith("test-")) { + stage("${OS} Test Packaging") { + sh "./deploy/build.sh --os=${OS} --release" + } + } } } }] @@ -143,6 +151,83 @@ pipeline { } } } + + stage('Publish') { + when { + allOf { + anyOf { + branch 'alpha'; + branch 'stable'; + } + + triggeredBy 'TimerTrigger' + } + } + steps { + script { + def new_version = '0.0.0' + + stage("Calculate Version") { + ws("${WORKSPACE}/publish") { + checkout scm; + + new_version = sh( + script: "./deploy/get_new_version.py", + returnStdout: true + ).trim() + + if (new_version == '0.0.0') { + error "Failed to calculate new version" + } + + echo "Calculated new version to be ${new_version}" + } + } + + release_file_list = [] + + parallel OSList.findAll{ OS -> (!OS.startsWith("test-")) }.collectEntries { + OS -> [ "${OS} Release & Publish": { + stage("${OS} Release & Publish") { + ws("${WORKSPACE}/${OS}") { + stage("${OS} Release") { + sh "./deploy/build.sh --os=${OS} --release=${new_version}" + + findFiles(glob: "tarfiles/*.tgz").each { + file -> release_file_list.add(file.path) + } + } + + stage("${OS} Publish") { + sh "./deploy/build.sh --os=${OS} --publish=${new_version} --publishdir=/tmp/publish" + } + } + } + }] + } + + stage("Publish Version") { + ws("${WORKSPACE}/publish") { + def tag = "${BRANCH_NAME}_release-" + new_version.replaceAll("\\.", "-") + + echo "Creating GitHub Release and Tag for ${tag}" + withCredentials([ + usernamePassword( + credentialsId: 'MDSplusJenkins', + usernameVariable: 'GITHUB_APP', + passwordVariable: 'GITHUB_ACCESS_TOKEN' + )]) { + + // TODO: Protect against spaces in filenames + def release_file_list_arg = release_file_list.join(" ") + sh "./deploy/create_github_release.py --tag ${tag} --api-token \$GITHUB_ACCESS_TOKEN ${release_file_list_arg}" + } + + } + } + } + } + } } post { always { diff --git a/deploy/create_github_release.py b/deploy/create_github_release.py new file mode 100755 index 0000000000..0634aa96ac --- /dev/null +++ b/deploy/create_github_release.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +import requests +import argparse +import subprocess +import shutil + +API_URL = 'https://api.github.com/repos/MDSplus/mdsplus' +UPLOAD_URL = 'https://uploads.github.com/repos/MDSplus/mdsplus' + +parser = argparse.ArgumentParser() + +parser.add_argument( + '--github-name', + default='MDSplusBuilder' +) + +parser.add_argument( + '--github-email', + default='mdsplusadmin@psfc.mit.edu' +) + +parser.add_argument( + '--tag', + required=True +) + +parser.add_argument( + '--api-token', + required=True +) + +parser.add_argument( + 'files', + nargs=argparse.REMAINDER +) + +args = parser.parse_args() + +git_executable = shutil.which('git') + +result = subprocess.run( + [ git_executable, 'rev-parse', args.tag ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT +) + +if result.returncode == 0: + print(f"The tag {args.tag} already exists, exiting") + exit(0) + +result = subprocess.run( + [ git_executable, 'rev-parse', 'HEAD'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT +) + +git_hash = result.stdout.decode().strip() + +headers = { + 'Accept': 'application/vnd.github+json', + 'Authorization': f'Bearer {args.api_token}', + 'X-GitHub-Api-Version': '2022-11-28', # TODO: Is this needed? +} + +print('Creating release') + +create_release = { + 'tag_name': args.tag, + 'target_commitish': git_hash, + # This will automatically generate the name and body of the release + 'generate_release_notes': True +} + +create_release_response = requests.post(f'{API_URL}/releases', json=create_release, headers=headers) +if create_release_response.status_code != 201: + print(create_release_response.content.decode()) + exit(1) + +release_id = create_release_response.json()['id'] + +asset_headers = headers +asset_headers['Content-Type'] = 'application/octet-stream' + +for file in args.files: + + file_name = os.path.basename(file) + data = open(file, 'rb').read() + + upload_release_asset_response = requests.post(f'{UPLOAD_URL}/releases/{release_id}/assets?name={file_name}', data=data, headers=asset_headers) + print(upload_release_asset_response.request.headers) + print(upload_release_asset_response.request.url) + if upload_release_asset_response.status_code != 201: + print(upload_release_asset_response.content.decode()) + # attempt to upload the rest of the files diff --git a/deploy/get_new_version.py b/deploy/get_new_version.py new file mode 100755 index 0000000000..0f7d76e718 --- /dev/null +++ b/deploy/get_new_version.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import subprocess +import shutil + +git_executable = shutil.which('git') + +def git(command): + # print(f'git {command}') + proc = subprocess.Popen( + [ git_executable ] + command.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + + stdout, _ = proc.communicate() + return stdout.decode().strip() + +last_release = git('describe --tags --abbrev=0') +last_release_commit = git(f'rev-list -n 1 {last_release}') +commit_log = git(f'log {last_release_commit}..HEAD --no-merges --decorate=short --pretty=format:%s') + +version_bump = 'SAME' +for commit in commit_log.splitlines(): + commit = commit.lower() + + if commit.startswith('feature') or commit.startswith('revert "feature'): + version_bump = 'MINOR' + + elif commit.startswith('fix') or commit.startswith('revert "fix'): + if version_bump != 'MINOR': + version_bump = 'PATCH' + + elif commit.startswith('tests') or commit.startswith('revert "tests'): + pass + + elif commit.startswith('build') or commit.startswith('revert "build'): + pass + + elif commit.startswith('docs') or commit.startswith('revert "docs'): + pass + +version_parts = last_release.split('-') +if len(version_parts) < 4: + print('0.0.0') + exit(1) +else: + major = int(version_parts[-3]) + minor = int(version_parts[-2]) + patch = int(version_parts[-1]) + +if version_bump == 'MAJOR': + major += 1 +elif version_bump == 'MINOR': + minor += 1 +elif version_bump == 'PATCH': + patch += 1 + +print(f'{major}.{minor}.{patch}') \ No newline at end of file diff --git a/deploy/platform/debian/debian_docker_build.sh b/deploy/platform/debian/debian_docker_build.sh index 065109d021..477db4ff92 100755 --- a/deploy/platform/debian/debian_docker_build.sh +++ b/deploy/platform/debian/debian_docker_build.sh @@ -65,10 +65,11 @@ buildrelease() { # ${RELEASEDIR}/${FLAVOR}/DEBS will be cleaned in debian_build.sh RELEASEDEBS=/release/${FLAVOR}/DEBS/${ARCH} RELEASEBLD=/workspace/releasebld + TARFILES=/workspace/tarfiles BUILDROOT=${RELEASEBLD}/buildroot MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus - rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT} - mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR} + rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT} ${TARFILES} + mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES} pushd ${RELEASEBLD}/${bits} config ${config_param} ${CONFIGURE_EXTRA} if [ -z "$NOMAKE" ]; then @@ -152,6 +153,14 @@ EOF fi done popd + + pushd ${MDSPLUS_DIR} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz * + popd + + pushd ${RELEASEDEBS} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_debs.tgz *.deb + popd fi #abort fi #nomake } diff --git a/deploy/platform/platform_build.sh b/deploy/platform/platform_build.sh index cf5b158488..db332e1ec9 100755 --- a/deploy/platform/platform_build.sh +++ b/deploy/platform/platform_build.sh @@ -81,8 +81,8 @@ rundocker() { function kill_docker() { if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then - docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) || true - docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) || true + docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true + docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true rm -f ${WORKSPACE}/${OS}_docker-cid fi if [ ! -z $DOCKERNETWORK ]; then diff --git a/deploy/platform/redhat/redhat_docker_build.sh b/deploy/platform/redhat/redhat_docker_build.sh index eebb801195..df8d462887 100755 --- a/deploy/platform/redhat/redhat_docker_build.sh +++ b/deploy/platform/redhat/redhat_docker_build.sh @@ -57,9 +57,10 @@ buildrelease() { set -e RELEASEBLD=/workspace/releasebld BUILDROOT=${RELEASEBLD}/buildroot + TARFILES=/workspace/tarfiles MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus - rm -Rf ${RELEASEBLD} /release/${FLAVOR} - mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR} + rm -Rf ${RELEASEBLD} /release/${FLAVOR} ${TARFILES} + mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES} pushd ${RELEASEBLD}/64 config ${test64} ${CONFIGURE_EXTRA} if [ -z "$NOMAKE" ]; then @@ -132,6 +133,14 @@ EOF fi done checkstatus abort "Failure: Problem with contents of one or more rpms. (see above)" $badrpm + + pushd ${MDSPLUS_DIR} + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz * + popd + + pushd /release/${FLAVOR}/RPMS + tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_rpms.tgz */*.rpm + popd fi #nomake }