Begin work on publishing

Jenkinsfile

@@ -50,6 +50,7 @@ pipeline {
     }
 
     stages {
+
         stage('Setup') {
             steps {
                 sh 'printenv'
@@ -78,12 +79,13 @@ pipeline {
                 cleanWs disableDeferredWipeout: true, deleteDirs: true
             }
         }
+
         stage('Distributions') {
             steps {
                 script {
                     parallel OSList.collectEntries {
-                        OS -> [ "${OS}": {
-                            stage("${OS}") {
+                        OS -> [ "${OS} Build & Test": {
+                            stage("${OS} Build & Test") {
                                 ws("${WORKSPACE}/${OS}") {
                                     stage("${OS} Clone") {
                                         checkout scm;
@@ -109,6 +111,12 @@ pipeline {
                                             echo "Testing complete"
                                         }
                                     }
+
+                                    if (env.CHANGE_ID && !OS.startsWith("test-")) {
+                                        stage("${OS} Test Packaging") {
+                                            sh "./deploy/build.sh --os=${OS} --release"
+                                        }
+                                    }
                                 }
                             }
                         }]
@@ -143,6 +151,83 @@ pipeline {
                 }
             }
         }
+
+        stage('Publish') {
+            when {
+                allOf {
+                    anyOf {
+                        branch 'alpha';
+                        branch 'stable';
+                    }
+
+                    triggeredBy 'TimerTrigger'
+                }
+            }
+            steps {
+                script {
+                    def new_version = '0.0.0'
+
+                    stage("Calculate Version") {
+                        ws("${WORKSPACE}/publish") {
+                            checkout scm;
+
+                            new_version = sh(
+                                script: "./deploy/get_new_version.py",
+                                returnStdout: true
+                            ).trim()
+
+                            if (new_version == '0.0.0') {
+                                error "Failed to calculate new version"
+                            }
+
+                            echo "Calculated new version to be ${new_version}"
+                        }
+                    }
+
+                    release_file_list = []
+
+                    parallel OSList.findAll{ OS -> (!OS.startsWith("test-")) }.collectEntries {
+                        OS -> [ "${OS} Release & Publish": {
+                            stage("${OS} Release & Publish") {
+                                ws("${WORKSPACE}/${OS}") {
+                                    stage("${OS} Release") {
+                                        sh "./deploy/build.sh --os=${OS} --release=${new_version}"
+
+                                        findFiles(glob: "tarfiles/*.tgz").each {
+                                            file -> release_file_list.add(file.path)
+                                        }
+                                    }
+
+                                    stage("${OS} Publish") {
+                                        sh "./deploy/build.sh --os=${OS} --publish=${new_version} --publishdir=/tmp/publish"
+                                    }
+                                }
+                            }
+                        }]
+                    }
+
+                    stage("Publish Version") {
+                        ws("${WORKSPACE}/publish") {
+                            def tag = "${BRANCH_NAME}_release-" + new_version.replaceAll("\\.", "-")
+
+                            echo "Creating GitHub Release and Tag for ${tag}"
+                            withCredentials([
+                                usernamePassword(
+                                    credentialsId: 'MDSplusJenkins',
+                                    usernameVariable: 'GITHUB_APP',
+                                    passwordVariable: 'GITHUB_ACCESS_TOKEN'
+                                )]) {
+
+                                // TODO: Protect against spaces in filenames
+                                def release_file_list_arg = release_file_list.join(" ")
+                                sh "./deploy/create_github_release.py --tag ${tag} --api-token \$GITHUB_ACCESS_TOKEN ${release_file_list_arg}"
+                            }
+
+                        }
+                    }
+                }
+            }
+        }
     }
     post {
         always {

deploy/create_github_release.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+import os
+import requests
+import argparse
+import subprocess
+import shutil
+
+API_URL = 'https://api.github.com/repos/MDSplus/mdsplus'
+UPLOAD_URL = 'https://uploads.github.com/repos/MDSplus/mdsplus'
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument(
+    '--github-name',
+    default='MDSplusBuilder'
+)
+
+parser.add_argument(
+    '--github-email',
+    default='mdsplusadmin@psfc.mit.edu'
+)
+
+parser.add_argument(
+    '--tag',
+    required=True
+)
+
+parser.add_argument(
+    '--api-token',
+    required=True
+)
+
+parser.add_argument(
+    'files',
+    nargs=argparse.REMAINDER
+)
+
+args = parser.parse_args()
+
+git_executable = shutil.which('git')
+
+result = subprocess.run(
+    [ git_executable, 'rev-parse', args.tag ],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT
+)
+
+if result.returncode == 0:
+    print(f"The tag {args.tag} already exists, exiting")
+    exit(0)
+
+result = subprocess.run(
+    [ git_executable, 'rev-parse', 'HEAD'],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT
+)
+
+git_hash = result.stdout.decode().strip()
+
+headers = {
+    'Accept': 'application/vnd.github+json',
+    'Authorization': f'Bearer {args.api_token}',
+    'X-GitHub-Api-Version': '2022-11-28', # TODO: Is this needed?
+}
+
+print('Creating release')
+
+create_release = {
+    'tag_name': args.tag,
+    'target_commitish': git_hash,
+    # This will automatically generate the name and body of the release
+    'generate_release_notes': True
+}
+
+create_release_response = requests.post(f'{API_URL}/releases', json=create_release, headers=headers)
+if create_release_response.status_code != 201:
+    print(create_release_response.content.decode())
+    exit(1)
+
+release_id = create_release_response.json()['id']
+
+asset_headers = headers
+asset_headers['Content-Type'] = 'application/octet-stream'
+
+for file in args.files:
+
+    file_name = os.path.basename(file)
+    data = open(file, 'rb').read()
+
+    upload_release_asset_response = requests.post(f'{UPLOAD_URL}/releases/{release_id}/assets?name={file_name}', data=data, headers=asset_headers)
+    print(upload_release_asset_response.request.headers)
+    print(upload_release_asset_response.request.url)
+    if upload_release_asset_response.status_code != 201:
+        print(upload_release_asset_response.content.decode())
+        # attempt to upload the rest of the files

deploy/get_new_version.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import subprocess
+import shutil
+
+git_executable = shutil.which('git')
+
+def git(command):
+    # print(f'git {command}')
+    proc = subprocess.Popen(
+        [ git_executable ] + command.split(),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT
+    )
+
+    stdout, _ = proc.communicate()
+    return stdout.decode().strip()
+
+last_release = git('describe --tags --abbrev=0')
+last_release_commit = git(f'rev-list -n 1 {last_release}')
+commit_log = git(f'log {last_release_commit}..HEAD --no-merges --decorate=short --pretty=format:%s')
+
+version_bump = 'SAME'
+for commit in commit_log.splitlines():
+    commit = commit.lower()
+
+    if commit.startswith('feature') or commit.startswith('revert "feature'):
+        version_bump = 'MINOR'
+
+    elif commit.startswith('fix') or commit.startswith('revert "fix'):
+        if version_bump != 'MINOR':
+            version_bump = 'PATCH'
+
+    elif commit.startswith('tests') or commit.startswith('revert "tests'):
+        pass
+
+    elif commit.startswith('build') or commit.startswith('revert "build'):
+        pass
+
+    elif commit.startswith('docs') or commit.startswith('revert "docs'):
+        pass
+
+version_parts = last_release.split('-')
+if len(version_parts) < 4:
+    print('0.0.0')
+    exit(1)
+else:
+    major = int(version_parts[-3])
+    minor = int(version_parts[-2])
+    patch = int(version_parts[-1])
+
+if version_bump == 'MAJOR':
+    major += 1
+elif version_bump == 'MINOR':
+    minor += 1
+elif version_bump == 'PATCH':
+    patch += 1
+
+print(f'{major}.{minor}.{patch}')

deploy/platform/debian/debian_docker_build.sh

@@ -65,10 +65,11 @@ buildrelease() {
   # ${RELEASEDIR}/${FLAVOR}/DEBS will be cleaned in debian_build.sh
   RELEASEDEBS=/release/${FLAVOR}/DEBS/${ARCH}
   RELEASEBLD=/workspace/releasebld
+  TARFILES=/workspace/tarfiles
   BUILDROOT=${RELEASEBLD}/buildroot
   MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus
-  rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT}
-  mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR}
+  rm -Rf ${RELEASEBLD}/${bits} ${BUILDROOT} ${TARFILES}
+  mkdir -p ${RELEASEBLD}/${bits} ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES}
   pushd ${RELEASEBLD}/${bits}
   config ${config_param} ${CONFIGURE_EXTRA}
   if [ -z "$NOMAKE" ]; then
@@ -152,6 +153,14 @@ EOF
         fi
       done
       popd
+
+      pushd ${MDSPLUS_DIR}
+      tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz *
+      popd
+
+      pushd ${RELEASEDEBS}
+      tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_debs.tgz *.deb
+      popd
     fi #abort
   fi   #nomake
 }

deploy/platform/platform_build.sh

@@ -81,8 +81,8 @@ rundocker() {
 
     function kill_docker() {
       if [ -r ${WORKSPACE}/${OS}_docker-cid ]; then
-        docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) || true
-        docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) || true
+        docker kill $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true
+        docker rm $(cat ${WORKSPACE}/${OS}_docker-cid) 2>/dev/null || true
         rm -f ${WORKSPACE}/${OS}_docker-cid
       fi
       if [ ! -z $DOCKERNETWORK ]; then

deploy/platform/redhat/redhat_docker_build.sh

@@ -57,9 +57,10 @@ buildrelease() {
   set -e
   RELEASEBLD=/workspace/releasebld
   BUILDROOT=${RELEASEBLD}/buildroot
+  TARFILES=/workspace/tarfiles
   MDSPLUS_DIR=${BUILDROOT}/usr/local/mdsplus
-  rm -Rf ${RELEASEBLD} /release/${FLAVOR}
-  mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR}
+  rm -Rf ${RELEASEBLD} /release/${FLAVOR} ${TARFILES}
+  mkdir -p ${RELEASEBLD}/64 ${BUILDROOT} ${MDSPLUS_DIR} ${TARFILES}
   pushd ${RELEASEBLD}/64
   config ${test64} ${CONFIGURE_EXTRA}
   if [ -z "$NOMAKE" ]; then
@@ -132,6 +133,14 @@ EOF
       fi
     done
     checkstatus abort "Failure: Problem with contents of one or more rpms. (see above)" $badrpm
+
+    pushd ${MDSPLUS_DIR}
+    tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}.tgz *
+    popd
+
+    pushd /release/${FLAVOR}/RPMS
+    tar -czf $TARFILES/mdsplus_${FLAVOR}_${RELEASE_VERSION}_${OS}_${ARCH}_rpms.tgz */*.rpm
+    popd
   fi #nomake
 }