Initial commit - move scripts into new release/ dir

.gitignore

@@ -51,3 +51,6 @@ crypto/libsodium-fork/build-aux/
 
 # doc intermediates
 data/transactions/logic/*.md
+
+*.pem
+

scripts/release/Jenkinsfile

@@ -0,0 +1,104 @@
+pipeline {
+    environment {
+        AWS_ACCESS_KEY_ID = credentials("aws-access-key-id")
+        AWS_SECRET_ACCESS_KEY = credentials("aws-secret-access-key")
+    }
+
+    agent any
+
+    stages {
+        stage("create ec2 instance") {
+            steps {
+                sh script: 'scripts/release/start_ec2_instance.sh us-west-1 ami-0dd655843c87b6930 t2.2xlarge'
+            }
+        }
+
+        stage("setup ec2 instance") {
+            steps {
+                sh 'aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 .'
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) mkdir docker_test_resources'
+                sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r gnupg2.2.9_centos7_amd64.tar.bz2 ubuntu@$(cat scripts/release/tmp/instance):~/docker_test_resources/'
+                sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r scripts/release/master-controller/setup.sh ubuntu@$(cat scripts/release/tmp/instance):~/setup.sh'
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash setup.sh'
+            }
+        }
+
+        stage("build") {
+            steps {
+                /*
+                script {
+                    def PASSPHRASE = input(
+                        id: 'passphrase',
+                        message: 'Enter the GPG passphrase',
+                        parameters: [
+                            [
+                                $class: 'StringParameterDefinition',
+                                defaultValue: '',
+                                description: 'GPG Passphrase',
+                                name: 'gpg_pass'
+                            ]
+                        ])
+
+                        sh "bash scripts/release/socket.sh \$(cat scripts/release/tmp/instance) ${PASSPHRASE}"
+                }
+                */
+
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/build.sh'
+            }
+        }
+
+        stage("package") {
+            steps {
+                input "GPG remote socket"
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/package.sh'
+            }
+        }
+
+        stage("sign") {
+            steps {
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/sign.sh'
+            }
+        }
+
+        stage("upload") {
+            steps {
+                script {
+                    RSTAMP = sh(returnStdout: true, script: 'scripts/reverse_hex_timestamp')
+                    BRANCH = sh(returnStdout: true, script: 'bash scripts/compute_branch.sh')
+                    CHANNEL = sh(returnStdout: true, script: 'bash scripts/compute_branch_channel.sh ${BRANCH}')
+                    FULLVERSION = sh(returnStdout: true, script: 'bash scripts/compute_build_number.sh -f')
+
+                    // Bash scripts return vars with a trailing [:space:]
+                    BRANCH = BRANCH.replaceAll("\\s","")
+                    CHANNEL = CHANNEL.replaceAll("\\s","")
+                    FULLVERSION = FULLVERSION.replaceAll("\\s","")
+
+                    sh "rm -rf node_pkg/* && mkdir -p node_pkg/${RSTAMP}"
+                    sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@\$(cat scripts/release/tmp/instance):~/node_pkg/* node_pkg/${RSTAMP}/"
+                    sh "aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read node_pkg/${RSTAMP} s3://algorand-dev-deb-repo/releases/${CHANNEL}/${RSTAMP}_${FULLVERSION}/"
+                    // Create the buildlog file.
+                    sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/upload.sh'
+                    // sh "aws s3 cp --quiet ubuntu@\$(cat scripts/release/tmp/instance)/build_status_${CHANNEL}_${FULLVERSION}.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/"
+                    sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@\$(cat scripts/release/tmp/instance):~/build_status_${CHANNEL}_*.asc.gz node_pkg/"
+                    sh "aws s3 cp --quiet node_pkg/build_status_${CHANNEL}_*.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/"
+                }
+            }
+        }
+
+        /*
+        stage("tag") {
+            steps {
+                sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/tag.sh'
+            }
+        }
+        */
+
+        stage("delete ec2 instance") {
+            steps {
+                input "Halt!"
+                sh script: "scripts/release/shutdown_ec2_instance.sh us-west-1"
+            }
+        }
+    }
+}
+

scripts/release/README.md

@@ -0,0 +1,61 @@
+## Jenkins Release Build
+
+The `Jenkinsfile` uses the pipeline module to define its build stages.  Currently, they are:
+
+1. create ec2 instance
+1. setup ec2 instance
+1. build
+1. package
+1. sign
+1. upload
+1. tag (TODO)
+1. delete ec2 instance
+
+The only thing that is not automated at this point is pre-setting the `gpg-agent` with the passphrase of the private key.  At the beginning of the `package` stage, Jenkins will pause and wait for the initiator of the build to do this and set up an SSH connection that will forward a Unix socket from the remote ec2 instance to the client, which in this case is most likely your laptop.
+
+## Workflow
+
+Take a look at the Jenkins build configuration.  This will set the build in motion by downloading the project from GitHub.
+
+## Setting up the Forwarded Connection
+
+To complete this step, you will need to do the following:
+
+1. Download the `BuilderInstanceKey.pem` certificate from the appropriate Jenkins workspace and `chmod 400` on it or GPG will complain.  A subsequent step will assume that you moved this to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory.
+1. Get the instance name from AWS, i.e., https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#Instances:sort=instanceState
+1. Change to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory and execute `./socket.sh`, passing it the ec2 instance name that you just got from AWS:
+
+        ./socket ec2-13-57-188-227.us-west-1.compute.amazonaws.com
+
+1. At the prompt, input the GPG passphrase (**Don't do this in a public space!!**).
+1. You should now be logged into the remote machine!
+1. As a sanity, it is a good idea to sign some text as a test to make sure that the connection was set up properly.  Enter the following pipeline:
+
+        echo foo | gpg -u rpm@algorand.com --clearsign
+
+    If there are any errors or if you are prompted for the passphrase, log out and run the above command again.
+
+1. Go back to Jenkins, hover over the build step that is currently paused, and click "Proceed".
+
+This is all of the manual work that needs to be done.
+
+**Note** that I'd currently like to fully automate this, but the only way to do that is to install the GPG keys on the Jenkins production maching.
+
+> You may be wondering why it's necessary to automate the GPG bits.  Well, this is to circumvent the need to somehow get the private key onto the remote machine, which we definitely don't want to do.  See this explanation.
+
+## Build Artifacts
+
+The result of running this job will be to put the build artifacts and their detached signatures in the AWS `algorand-dev-deb-repo` bucket.  The location will depend on the type of artifact, of course.
+
+In addition, the build logs will be placed into the AWS `algorand-devops-misc` S3 bucket under `buildlog`.
+
+## Notes
+
+All of the `aws ...` commands are now executed by Jenkins and are defined in the `Jenkinsfile`.  The reason for this is simple:  Jenkins already has the AWS auth credentials, and we don't want or need to be sending them anywhere else in the cloud.
+
+## TODO
+
+Create the git tag.
+Upload the deb package via `aptly`.
+Add ability to specify branch and/or channel.
+

scripts/release/helper/build_release_centos_docker.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# shellcheck disable=2012
+#
+# build centos rpm from inside docker
+#
+# mount src from outside
+# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src
+#
+# mount golang install from outside
+# --mount type=bind,src=/usr/local/go,dst=/usr/local/go
+#
+# output copied to /root/subhome/node_pkg
+# --mount type=bind,src=${HOME},dst=/root/subhome
+
+set -ex
+
+export HOME=/root
+export GOPATH=${HOME}/go
+export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
+
+# Anchor our repo root reference location
+#REPO_DIR="$( cd "$(dirname "$0")" ; pwd -P )"/..
+REPO_DIR=/root/go/src/github.com/algorand/go-algorand
+
+${REPO_DIR}/scripts/configure_dev-deps.sh
+
+cd ${REPO_DIR}
+
+# definitely rebuild libsodium which could link to external C libraries
+if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then
+    (cd ${REPO_DIR}/crypto/libsodium-fork && make distclean)
+fi
+rm -rf ${REPO_DIR}/crypto/lib
+make crypto/lib/libsodium.a
+
+make build
+
+#export NO_BUILD=1
+
+RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
+trap 'rm -rf ${RPMTMP}' 0
+"${REPO_DIR}/scripts/release/helper/build_rpm.sh" "${RPMTMP}"
+cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg
+
+(cd ${HOME} && tar jxf /root/stuff/gnupg*.tar.bz2)
+export PATH="${HOME}/gnupg2/bin:${PATH}"
+export LD_LIBRARY_PATH=${HOME}/gnupg2/lib
+
+umask 0077
+mkdir -p "${HOME}/.gnupg"
+umask 0022
+touch "${HOME}/.gnupg/gpg.conf"
+if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then
+    echo ""
+else
+    echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf"
+fi
+rm -f ${HOME}/.gnupg/S.gpg-agent
+(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent)
+
+gpg --import /root/stuff/key.pub
+gpg --import /root/stuff/rpm.pub
+gpg --import ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand
+rpmkeys --import /root/stuff/rpm.pub
+echo "wat" | gpg -u rpm@algorand.com --clearsign
+
+cat <<EOF>"${HOME}/.rpmmacros"
+%_gpg_name ALGORAND RPM <rpm@algorand.com>
+%__gpg ${HOME}/gnupg2/bin/gpg
+%__gpg_check_password_cmd true
+EOF
+
+cat <<EOF>"${HOME}/rpmsign.py"
+import rpm
+import sys
+rpm.addSign(sys.argv[1], '')
+EOF
+
+NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm | head -1)
+python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}"
+
+cp -p "${NEWEST_RPM}" /root/dummyrepo
+createrepo --database /root/dummyrepo
+rm -f /root/dummyrepo/repodata/repomd.xml.asc
+gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml
+
+OLDRPM=$(ls -t /root/stuff/*.rpm | head -1)
+if [ -f "${OLDRPM}" ]; then
+    yum install -y "${OLDRPM}"
+    algod -v
+    if algod -v | grep -q "${FULLVERSION}"
+    then
+        echo "already installed current version. wat?"
+        false
+    fi
+
+    mkdir -p /root/testnode
+    cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode
+
+    goal node start -d /root/testnode
+    goal node wait -d /root/testnode -w 120
+    goal node stop -d /root/testnode
+fi
+
+yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo"
+
+yum install -y algorand
+algod -v
+# check that the installed version is now the current version
+algod -v | grep -q "${FULLVERSION}.${CHANNEL}"
+
+if [ ! -d /root/testnode ]; then
+    mkdir -p /root/testnode
+    cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode
+fi
+
+goal node start -d /root/testnode
+goal node wait -d /root/testnode -w 120
+goal node stop -d /root/testnode
+
+echo CENTOS_DOCKER_TEST_OK
+

scripts/release/helper/build_release_local.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+#
+# This is a file of commands to copy and paste to run release.sh on an AWS EC2 instance.
+# Should work on Ubuntu 16.04 ro 18.04
+#
+# Externally settable env vars:
+# S3_PREFIX_BUILDLOG= where upload build log (no trailing /)
+
+echo "this is a file of commands to copy and paste to run release.sh on an AWS EC2 instance"
+exit 1
+
+# use AWS console to create a new t3.large with the latest official Ubuntu 18.04
+
+# ec2 public address here:
+TARGET=
+
+cd ${GOPATH}/src/github.com/algorand/go-algorand
+
+git fetch
+git checkout rel/stable
+git merge origin/rel/stable
+scp -p ${GOPATH}/src/github.com/algorand/go-algorand/scripts/release/setup.sh ubuntu@${TARGET}:~/
+
+# upload the latest public key
+GTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
+gpg --export --armor -o "${GTMPDIR}/key.gpg" dev@algorand.com
+scp -p "${GTMPDIR}/key.gpg" "ubuntu@${TARGET}:~/key.gpg"
+rm -rf ${GTMPDIR}
+
+ssh -A ubuntu@${TARGET} bash setup.sh
+
+# setup GPG key forwarding https://wiki.gnupg.org/AgentForwarding
+umask 0077
+touch ${HOME}/.gnupg/gpg-agent.conf
+if grep -q extra-socket ${HOME}/.gnupg/gpg-agent.conf; then
+    echo "already have extra-socket"
+else
+    cat <<EOF>>${HOME}/.gnupg/gpg-agent.conf
+extra-socket ${HOME}/.gnupg/S.gpg-agent.extra
+default-cache-ttl 3600
+EOF
+fi
+umask 0002
+
+# this will require your key password, and export a private key file protected by the same password
+
+# warm up your local gpg-agent
+gpg -u dev@algorand.com --clearsign
+type some stuff
+^D
+
+gpg -u rpm@algorand.com --clearsign
+
+
+# TODO: use simpler expression when we can rely on gpg 2.2 on ubuntu >= 18.04
+#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgconf --list-dir agent-socket)
+#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} "gpgconf --list-dirs|grep agent-socket|awk -F: '{ print \$2 }'")
+REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgbin/remote_gpg_socket)
+LOCAL_GPG_SOCKET=$(gpgconf --list-dir agent-extra-socket)
+ssh -A -R "${REMOTE_GPG_SOCKET}:${LOCAL_GPG_SOCKET}" ubuntu@${TARGET}
+
+# check gpg agent connection
+gpg -u dev@algorand.com --clearsign
+blah blah
+^D
+
+
+# set AWS credentials so we can upload to S3 and connect to EFS
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+
+# where we store persistent scratch space for aptly
+export AWS_EFS_MOUNT=
+
+# release.sh needs to be run in a terminal with a human watching
+# to be prompted for GPG key password at a couple points.
+# It can still steal the outer terminal from within piping the output to tee. Nifty, huh?
+BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp")
+(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/build.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}"
+(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/sign.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}"
+(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/upload.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}"
+if [ -f "${HOME}/rstamp" ]; then
+    . "${HOME}/rstamp"
+fi
+if [ -z "${RSTAMP}" ]; then
+    RSTAMP=$(${HOME}/go/src/github.com/algorand/go-algorand/scripts/reverse_hex_timestamp)
+fi
+if [ -z "${RSTAMP}" ]; then
+    echo "could not figure out RSTAMP, script must have failed early"
+    exit 1
+fi
+gzip "${HOME}/buildlog_${BUILDTIMESTAMP}"
+if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then
+    aws s3 cp "${HOME}/buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz"
+fi