diff --git a/.gitignore b/.gitignore index 71617a4836..44bec0113d 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,6 @@ crypto/libsodium-fork/build-aux/ # doc intermediates data/transactions/logic/*.md + +*.pem + diff --git a/scripts/release/Jenkinsfile b/scripts/release/Jenkinsfile new file mode 100644 index 0000000000..96c453ae4c --- /dev/null +++ b/scripts/release/Jenkinsfile @@ -0,0 +1,104 @@ +pipeline { + environment { + AWS_ACCESS_KEY_ID = credentials("aws-access-key-id") + AWS_SECRET_ACCESS_KEY = credentials("aws-secret-access-key") + } + + agent any + + stages { + stage("create ec2 instance") { + steps { + sh script: 'scripts/release/start_ec2_instance.sh us-west-1 ami-0dd655843c87b6930 t2.2xlarge' + } + } + + stage("setup ec2 instance") { + steps { + sh 'aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 .' + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) mkdir docker_test_resources' + sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r gnupg2.2.9_centos7_amd64.tar.bz2 ubuntu@$(cat scripts/release/tmp/instance):~/docker_test_resources/' + sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r scripts/release/master-controller/setup.sh ubuntu@$(cat scripts/release/tmp/instance):~/setup.sh' + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash setup.sh' + } + } + + stage("build") { + steps { + /* + script { + def PASSPHRASE = input( + id: 'passphrase', + message: 'Enter the GPG passphrase', + parameters: [ + [ + $class: 'StringParameterDefinition', + defaultValue: '', + description: 'GPG Passphrase', + name: 'gpg_pass' + ] + ]) + + sh "bash scripts/release/socket.sh \$(cat scripts/release/tmp/instance) ${PASSPHRASE}" + } + */ + + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/build.sh' + } + } + + stage("package") { + steps { + input "GPG remote socket" + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/package.sh' + } + } + + stage("sign") { + steps { + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/master-controller/sign.sh' + } + } + + stage("upload") { + steps { + script { + RSTAMP = sh(returnStdout: true, script: 'scripts/reverse_hex_timestamp') + BRANCH = sh(returnStdout: true, script: 'bash scripts/compute_branch.sh') + CHANNEL = sh(returnStdout: true, script: 'bash scripts/compute_branch_channel.sh ${BRANCH}') + FULLVERSION = sh(returnStdout: true, script: 'bash scripts/compute_build_number.sh -f') + + // Bash scripts return vars with a trailing [:space:] + BRANCH = BRANCH.replaceAll("\\s","") + CHANNEL = CHANNEL.replaceAll("\\s","") + FULLVERSION = FULLVERSION.replaceAll("\\s","") + + sh "rm -rf node_pkg/* && mkdir -p node_pkg/${RSTAMP}" + sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@\$(cat scripts/release/tmp/instance):~/node_pkg/* node_pkg/${RSTAMP}/" + sh "aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read node_pkg/${RSTAMP} s3://algorand-dev-deb-repo/releases/${CHANNEL}/${RSTAMP}_${FULLVERSION}/" + // Create the buildlog file. + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/upload.sh' + // sh "aws s3 cp --quiet ubuntu@\$(cat scripts/release/tmp/instance)/build_status_${CHANNEL}_${FULLVERSION}.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/" + sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@\$(cat scripts/release/tmp/instance):~/build_status_${CHANNEL}_*.asc.gz node_pkg/" + sh "aws s3 cp --quiet node_pkg/build_status_${CHANNEL}_*.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/" + } + } + } + + /* + stage("tag") { + steps { + sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/tag.sh' + } + } + */ + + stage("delete ec2 instance") { + steps { + input "Halt!" + sh script: "scripts/release/shutdown_ec2_instance.sh us-west-1" + } + } + } +} + diff --git a/scripts/release/README.md b/scripts/release/README.md new file mode 100644 index 0000000000..d892ed13a8 --- /dev/null +++ b/scripts/release/README.md @@ -0,0 +1,61 @@ +## Jenkins Release Build + +The `Jenkinsfile` uses the pipeline module to define its build stages. Currently, they are: + +1. create ec2 instance +1. setup ec2 instance +1. build +1. package +1. sign +1. upload +1. tag (TODO) +1. delete ec2 instance + +The only thing that is not automated at this point is pre-setting the `gpg-agent` with the passphrase of the private key. At the beginning of the `package` stage, Jenkins will pause and wait for the initiator of the build to do this and set up an SSH connection that will forward a Unix socket from the remote ec2 instance to the client, which in this case is most likely your laptop. + +## Workflow + +Take a look at the Jenkins build configuration. This will set the build in motion by downloading the project from GitHub. + +## Setting up the Forwarded Connection + +To complete this step, you will need to do the following: + +1. Download the `BuilderInstanceKey.pem` certificate from the appropriate Jenkins workspace and `chmod 400` on it or GPG will complain. A subsequent step will assume that you moved this to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory. +1. Get the instance name from AWS, i.e., https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#Instances:sort=instanceState +1. Change to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory and execute `./socket.sh`, passing it the ec2 instance name that you just got from AWS: + + ./socket ec2-13-57-188-227.us-west-1.compute.amazonaws.com + +1. At the prompt, input the GPG passphrase (**Don't do this in a public space!!**). +1. You should now be logged into the remote machine! +1. As a sanity, it is a good idea to sign some text as a test to make sure that the connection was set up properly. Enter the following pipeline: + + echo foo | gpg -u rpm@algorand.com --clearsign + + If there are any errors or if you are prompted for the passphrase, log out and run the above command again. + +1. Go back to Jenkins, hover over the build step that is currently paused, and click "Proceed". + +This is all of the manual work that needs to be done. + +**Note** that I'd currently like to fully automate this, but the only way to do that is to install the GPG keys on the Jenkins production maching. + +> You may be wondering why it's necessary to automate the GPG bits. Well, this is to circumvent the need to somehow get the private key onto the remote machine, which we definitely don't want to do. See this explanation. + +## Build Artifacts + +The result of running this job will be to put the build artifacts and their detached signatures in the AWS `algorand-dev-deb-repo` bucket. The location will depend on the type of artifact, of course. + +In addition, the build logs will be placed into the AWS `algorand-devops-misc` S3 bucket under `buildlog`. + +## Notes + +All of the `aws ...` commands are now executed by Jenkins and are defined in the `Jenkinsfile`. The reason for this is simple: Jenkins already has the AWS auth credentials, and we don't want or need to be sending them anywhere else in the cloud. + +## TODO + +Create the git tag. +Upload the deb package via `aptly`. +Add ability to specify branch and/or channel. + diff --git a/scripts/release/helper/build_release_centos_docker.sh b/scripts/release/helper/build_release_centos_docker.sh new file mode 100755 index 0000000000..7d9a61a2f9 --- /dev/null +++ b/scripts/release/helper/build_release_centos_docker.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# shellcheck disable=2012 +# +# build centos rpm from inside docker +# +# mount src from outside +# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src +# +# mount golang install from outside +# --mount type=bind,src=/usr/local/go,dst=/usr/local/go +# +# output copied to /root/subhome/node_pkg +# --mount type=bind,src=${HOME},dst=/root/subhome + +set -ex + +export HOME=/root +export GOPATH=${HOME}/go +export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} + +# Anchor our repo root reference location +#REPO_DIR="$( cd "$(dirname "$0")" ; pwd -P )"/.. +REPO_DIR=/root/go/src/github.com/algorand/go-algorand + +${REPO_DIR}/scripts/configure_dev-deps.sh + +cd ${REPO_DIR} + +# definitely rebuild libsodium which could link to external C libraries +if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then + (cd ${REPO_DIR}/crypto/libsodium-fork && make distclean) +fi +rm -rf ${REPO_DIR}/crypto/lib +make crypto/lib/libsodium.a + +make build + +#export NO_BUILD=1 + +RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") +trap 'rm -rf ${RPMTMP}' 0 +"${REPO_DIR}/scripts/release/helper/build_rpm.sh" "${RPMTMP}" +cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg + +(cd ${HOME} && tar jxf /root/stuff/gnupg*.tar.bz2) +export PATH="${HOME}/gnupg2/bin:${PATH}" +export LD_LIBRARY_PATH=${HOME}/gnupg2/lib + +umask 0077 +mkdir -p "${HOME}/.gnupg" +umask 0022 +touch "${HOME}/.gnupg/gpg.conf" +if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then + echo "" +else + echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" +fi +rm -f ${HOME}/.gnupg/S.gpg-agent +(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent) + +gpg --import /root/stuff/key.pub +gpg --import /root/stuff/rpm.pub +gpg --import ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand +rpmkeys --import /root/stuff/rpm.pub +echo "wat" | gpg -u rpm@algorand.com --clearsign + +cat <"${HOME}/.rpmmacros" +%_gpg_name ALGORAND RPM +%__gpg ${HOME}/gnupg2/bin/gpg +%__gpg_check_password_cmd true +EOF + +cat <"${HOME}/rpmsign.py" +import rpm +import sys +rpm.addSign(sys.argv[1], '') +EOF + +NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm | head -1) +python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" + +cp -p "${NEWEST_RPM}" /root/dummyrepo +createrepo --database /root/dummyrepo +rm -f /root/dummyrepo/repodata/repomd.xml.asc +gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml + +OLDRPM=$(ls -t /root/stuff/*.rpm | head -1) +if [ -f "${OLDRPM}" ]; then + yum install -y "${OLDRPM}" + algod -v + if algod -v | grep -q "${FULLVERSION}" + then + echo "already installed current version. wat?" + false + fi + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode +fi + +yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo" + +yum install -y algorand +algod -v +# check that the installed version is now the current version +algod -v | grep -q "${FULLVERSION}.${CHANNEL}" + +if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +fi + +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 120 +goal node stop -d /root/testnode + +echo CENTOS_DOCKER_TEST_OK + diff --git a/scripts/release/helper/build_release_local.sh b/scripts/release/helper/build_release_local.sh new file mode 100755 index 0000000000..6fa089fe5e --- /dev/null +++ b/scripts/release/helper/build_release_local.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# This is a file of commands to copy and paste to run release.sh on an AWS EC2 instance. +# Should work on Ubuntu 16.04 ro 18.04 +# +# Externally settable env vars: +# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) + +echo "this is a file of commands to copy and paste to run release.sh on an AWS EC2 instance" +exit 1 + +# use AWS console to create a new t3.large with the latest official Ubuntu 18.04 + +# ec2 public address here: +TARGET= + +cd ${GOPATH}/src/github.com/algorand/go-algorand + +git fetch +git checkout rel/stable +git merge origin/rel/stable +scp -p ${GOPATH}/src/github.com/algorand/go-algorand/scripts/release/setup.sh ubuntu@${TARGET}:~/ + +# upload the latest public key +GTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") +gpg --export --armor -o "${GTMPDIR}/key.gpg" dev@algorand.com +scp -p "${GTMPDIR}/key.gpg" "ubuntu@${TARGET}:~/key.gpg" +rm -rf ${GTMPDIR} + +ssh -A ubuntu@${TARGET} bash setup.sh + +# setup GPG key forwarding https://wiki.gnupg.org/AgentForwarding +umask 0077 +touch ${HOME}/.gnupg/gpg-agent.conf +if grep -q extra-socket ${HOME}/.gnupg/gpg-agent.conf; then + echo "already have extra-socket" +else + cat <>${HOME}/.gnupg/gpg-agent.conf +extra-socket ${HOME}/.gnupg/S.gpg-agent.extra +default-cache-ttl 3600 +EOF +fi +umask 0002 + +# this will require your key password, and export a private key file protected by the same password + +# warm up your local gpg-agent +gpg -u dev@algorand.com --clearsign +type some stuff +^D + +gpg -u rpm@algorand.com --clearsign + + +# TODO: use simpler expression when we can rely on gpg 2.2 on ubuntu >= 18.04 +#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgconf --list-dir agent-socket) +#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} "gpgconf --list-dirs|grep agent-socket|awk -F: '{ print \$2 }'") +REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgbin/remote_gpg_socket) +LOCAL_GPG_SOCKET=$(gpgconf --list-dir agent-extra-socket) +ssh -A -R "${REMOTE_GPG_SOCKET}:${LOCAL_GPG_SOCKET}" ubuntu@${TARGET} + +# check gpg agent connection +gpg -u dev@algorand.com --clearsign +blah blah +^D + + +# set AWS credentials so we can upload to S3 and connect to EFS +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= + +# where we store persistent scratch space for aptly +export AWS_EFS_MOUNT= + +# release.sh needs to be run in a terminal with a human watching +# to be prompted for GPG key password at a couple points. +# It can still steal the outer terminal from within piping the output to tee. Nifty, huh? +BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/build.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/sign.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/release/upload.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" +if [ -f "${HOME}/rstamp" ]; then + . "${HOME}/rstamp" +fi +if [ -z "${RSTAMP}" ]; then + RSTAMP=$(${HOME}/go/src/github.com/algorand/go-algorand/scripts/reverse_hex_timestamp) +fi +if [ -z "${RSTAMP}" ]; then + echo "could not figure out RSTAMP, script must have failed early" + exit 1 +fi +gzip "${HOME}/buildlog_${BUILDTIMESTAMP}" +if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then + aws s3 cp "${HOME}/buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz" +fi diff --git a/scripts/release/helper/build_release_run_ubuntu_docker_build_test.sh b/scripts/release/helper/build_release_run_ubuntu_docker_build_test.sh new file mode 100755 index 0000000000..39c9bdb9e3 --- /dev/null +++ b/scripts/release/helper/build_release_run_ubuntu_docker_build_test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# This script exists to give a trap atexit context for killing the httpd so that we're not waiting on that + +set -ex + +(cd ${HOME}/dummyaptly/public && python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/httpd.py --pid ${HOME}/phttpd.pid) & +trap ${GOPATH}/src/github.com/algorand/go-algorand/scripts/kill_httpd.sh 0 + +# Ubuntu 16 binaries are deprecated. Should still work to build from source for it. +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/root/stuff --mount type=bind,src=${HOME}/go,dst=/root/go --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:16.04 bash ${GOPATH}/src/github.com/algorand/go-algorand/scripts/release/helper/build_release_ubuntu_test_docker.sh" + +export DC_IP + +sg docker "${GOPATH}/src/github.com/algorand/go-algorand/scripts/debian/start_docker_debian_test.sh ${HOME}/docker_test_resources" + diff --git a/scripts/release/helper/build_release_ubuntu_test_docker.sh b/scripts/release/helper/build_release_ubuntu_test_docker.sh new file mode 100755 index 0000000000..28774f7379 --- /dev/null +++ b/scripts/release/helper/build_release_ubuntu_test_docker.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# test ubuntu install from inside docker image +# +# expects docker run with: +# --env-file ${HOME}/build_env_docker +# --mount type=bind,src=${HOME}/centos,dst=/root/stuff +# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src +# --mount type=bind,src=/usr/local/go,dst=/usr/local/go + +set -e +set -x + +export GOPATH=${HOME}/go +export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} + +apt-get update +apt-get install -y gnupg2 curl software-properties-common python3 + +if [ "${TEST_UPGRADE}" == "no" ]; then + echo "upgrade test skipped" +else + apt install -y /root/stuff/*.deb + algod -v + if algod -v | grep -q ${FULLVERSION}; then + echo "already installed current version. wat?" + false + fi + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode +fi + +#apt-key adv --fetch-keys https://releases.algorand.com/key.pub +apt-key add /root/stuff/key.pub +add-apt-repository "deb http://${DC_IP}:8111/ stable main" +apt-get update +apt-get install -y algorand +algod -v +# check that the installed version is now the current version +algod -v | grep -q ${FULLVERSION}.${CHANNEL} + +if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +fi + +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 120 +goal node stop -d /root/testnode + +echo UBUNTU_DOCKER_TEST_OK diff --git a/scripts/release/helper/build_rpm.sh b/scripts/release/helper/build_rpm.sh new file mode 100755 index 0000000000..1825ad5231 --- /dev/null +++ b/scripts/release/helper/build_rpm.sh @@ -0,0 +1,49 @@ +#!/bin/bash -e + +# build_rpm.sh - Build a .deb package for one platform. +# +# Syntax: build_rpm.sh +# +# Examples: scripts/build_rpm.sh /tmp + +if [ ! "$#" -eq 1 ]; then + echo "Syntax: build_rpm.sh " + exit 1 +fi + +set -x + +OUTDIR="$1" + +GOPATH=$(go env GOPATH) +export GOPATH + +cd "$(dirname "$0")"/.. +#export REPO_DIR=$(pwd -P) +export REPO_DIR=$HOME/go/src/github.com/algorand/go-algorand + +echo "Building RPM package" + +#if [ -z "${NO_BUILD}" ]; then + env GOOS="${OS}" GOARCH="${ARCH}" "$REPO_DIR/scripts/build_prod.sh" +#else + #echo "already built" + #true +#fi + +VER=$("$REPO_DIR/scripts/compute_build_number.sh" -f) + +if [ "${DEFAULTNETWORK}" = "" ]; then + DEFAULTNETWORK=$("$REPO_DIR/scripts/compute_branch_network.sh") + export DEFAULTNETWORK +fi + +DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "${DEFAULTNETWORK}") +export DEFAULT_RELEASE_NETWORK + +TEMPDIR=$(mktemp -d) +trap 'rm -rf $TEMPDIR' 0 +< "$REPO_DIR/installer/rpm/algorand.spec" sed -e s,@VER@,"${VER}", > "${TEMPDIR}/algorand.spec" + +rpmbuild --define "_rpmdir ${OUTDIR}" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "${TEMPDIR}/algorand.spec" + diff --git a/scripts/release/helper/centos-build.Dockerfile b/scripts/release/helper/centos-build.Dockerfile new file mode 100644 index 0000000000..6cea7767e1 --- /dev/null +++ b/scripts/release/helper/centos-build.Dockerfile @@ -0,0 +1,6 @@ +FROM centos:7 +WORKDIR /root +RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck + +ENTRYPOINT ["/bin/bash"] diff --git a/scripts/release/helper/release_deb.sh b/scripts/release/helper/release_deb.sh new file mode 100755 index 0000000000..912c832320 --- /dev/null +++ b/scripts/release/helper/release_deb.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Usage: release_deb.sh *.deb +# +# To run on an ephemeral instance, mount AWS EFS somewhere and use it: +# APTLY_DIR=/large/persistent/filesystem ./release_deb.sh *.deb + + +set -e +set -x + +if [ -z "${APTLY_DIR}" ]; then + APTLY_DIR=${HOME}/.aptly +fi + +if [ -z "${APTLY_S3_NAME}" ]; then + APTLY_S3_NAME=algorand-releases +fi + +cat <${HOME}/.aptly.conf +{ + "rootDir": "${APTLY_DIR}", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": {}, + "S3PublishEndpoints": { + "algorand-releases": { + "region":"us-east-1", + "bucket":"algorand-releases", + "acl":"public-read", + "prefix":"deb" + } + }, + "SwiftPublishEndpoints": {} +} +EOF + +FIRSTTIME= +if aptly repo create -distribution=stable -component=main algorand; then + FIRSTTIME=1 +fi +aptly repo add algorand "$@" +SNAPSHOT=algorand-$(date +%Y%m%d_%H%M%S) +aptly snapshot create ${SNAPSHOT} from repo algorand +if [ ! -z "${FIRSTTIME}" ]; then + echo "first publish" + aptly publish snapshot -origin=Algorand -label=Algorand ${SNAPSHOT} "s3:${APTLY_S3_NAME}:" +else + echo "publish snapshot ${SNAPSHOT}" + aptly publish switch stable "s3:${APTLY_S3_NAME}:" ${SNAPSHOT} +fi diff --git a/scripts/release/master-controller/build.sh b/scripts/release/master-controller/build.sh new file mode 100755 index 0000000000..a5c61c334f --- /dev/null +++ b/scripts/release/master-controller/build.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# This script needs to be run in a terminal with a human watching to +# be prompted for GPG key password at a couple points. +# +# Externally settable env vars: +# S3_PREFIX= where to upload build artifacts (no trailing /) +# RSTAMP= `scripts/reverse_hex_timestamp` +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= + +date "+build_release start %Y%m%d_%H%M%S" + +set -ex + +# Anchor our repo root reference location +# /home/ubuntu/.. +# /home/ubuntu/go/src/github.com/algorand/go-algorand/ +#REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/.. +REPO_ROOT=/home/ubuntu/go/src/github.com/algorand/go-algorand/ +#if [ -f ${REPO_ROOT}/crypto/libsodium-fork/Makefile ]; then +# (cd ${REPO_ROOT}/crypto/libsodium-fork && make distclean) +#fi +#rm -rf ${REPO_ROOT}/crypto/lib + +cd ${REPO_ROOT} +export RELEASE_GENESIS_PROCESS=true +PLATFORM=$(./scripts/osarchtype.sh) +PLATFORM_SPLIT=(${PLATFORM//\// }) +OS=${PLATFORM_SPLIT[0]} +ARCH=${PLATFORM_SPLIT[1]} +BRANCH=$(./scripts/compute_branch.sh) +export BRANCH +CHANNEL=$(./scripts/compute_branch_channel.sh "${BRANCH}") +export CHANNEL +DEFAULTNETWORK=$(./scripts/compute_branch_network.sh) +export DEFAULTNETWORK +export PKG_ROOT=${HOME}/node_pkg +export VARIATIONS="base" +# tell underlying 'build' scripts we already built +export NO_BUILD=true + +RSTAMP=$(scripts/reverse_hex_timestamp) +echo RSTAMP="${RSTAMP}" > "${HOME}/rstamp" + +# What's my default IP address? +# get the datacenter IP address for this EC2 host. +# this might equivalently be gotten from `netstat -rn` and `ifconfig -a` +DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4) +if [ -z "${DC_IP}" ]; then + echo "ERROR: need DC_IP to be set to your local (but not localhost) IP" + exit 1 +fi + +# Update version file for this build +if [ ! -z "${BUILD_NUMBER}" ]; then + echo "using externally set BUILD_NUMBER=${BUILD_NUMBER} without incrementing" +else + if [ -e buildnumber.dat ]; then + BUILD_NUMBER=$(cat ./buildnumber.dat) + BUILD_NUMBER=$(( BUILD_NUMBER + 1 )) + else + BUILD_NUMBER=0 + fi + echo ${BUILD_NUMBER} > ./buildnumber.dat + git add -A + git commit -m "Build ${BUILD_NUMBER}" +fi +FULLVERSION=$(./scripts/compute_build_number.sh -f) +export FULLVERSION + +# a bash user might `source build_env` to manually continue a broken build +cat <"${HOME}"/build_env +export RELEASE_GENESIS_PROCESS=${RELEASE_GENESIS_PROCESS} +PLATFORM=${PLATFORM} +OS=${OS} +ARCH=${ARCH} +export BRANCH=${BRANCH} +export CHANNEL=${CHANNEL} +export DEFAULTNETWORK=${DEFAULTNETWORK} +export PKG_ROOT=${PKG_ROOT} +export VARIATIONS=${VARIATIONS} +RSTAMP=${RSTAMP} +BUILD_NUMBER=${BUILD_NUMBER} +export FULLVERSION=${FULLVERSION} +DC_IP=${DC_IP} +EOF +# strip leading 'export ' for docker --env-file +sed 's/^export //g' < "${HOME}"/build_env > "${HOME}"/build_env_docker + +export GOPATH=${HOME}/go +export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} + +# Build! +scripts/configure_dev.sh +make crypto/lib/libsodium.a +make build + +export BUILD_DEB=1 +scripts/build_packages.sh "${PLATFORM}" + +# build docker release package +cd ${REPO_ROOT}/docker/release +sg docker "./build_algod_docker.sh ${HOME}/node_pkg/node_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.tar.gz" +cd ${REPO_ROOT}/scripts + +# NEXT: package.sh + diff --git a/scripts/release/master-controller/package.sh b/scripts/release/master-controller/package.sh new file mode 100755 index 0000000000..0e38b98cdb --- /dev/null +++ b/scripts/release/master-controller/package.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# shellcheck disable=2164,2166 + +#set -e + +REPO_ROOT=/home/ubuntu/go/src/github.com/algorand/go-algorand/ + +# copy previous installers into ~/docker_test_resources +cd "${HOME}/docker_test_resources" +if [ "${TEST_UPGRADE}" == "no" -o -z "${S3_PREFIX}" ]; then + echo "upgrade test disabled" +else + python3 "${REPO_ROOT}/scripts/get_current_installers.py" "${S3_PREFIX}/${CHANNEL}" +fi + +echo "TEST_UPGRADE=${TEST_UPGRADE}" >> "${HOME}/build_env_docker" + +cat <"${HOME}"/dummyaptly.conf +{ + "rootDir": "${HOME}/dummyaptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": {}, + "S3PublishEndpoints": {}, + "SwiftPublishEndpoints": {} +} +EOF +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf repo create -distribution=stable -component=main algodummy +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf repo add algodummy "${HOME}"/node_pkg/*.deb +SNAPSHOT=algodummy-$(date +%Y%m%d_%H%M%S) +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf snapshot create "${SNAPSHOT}" from repo algodummy +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf publish snapshot -origin=Algorand -label=Algorand "${SNAPSHOT}" + +#/home/ubuntu/release/helper/build_release_run_ubuntu_docker_build_test.sh + +date "+build_release done building ubuntu %Y%m%d_%H%M%S" + +# Run RPM build in Centos7 Docker container +sg docker "docker build -t algocentosbuild - < ${REPO_ROOT}/scripts/release/helper/centos-build.Dockerfile" + +# cleanup our libsodium build +if [ -f "${REPO_ROOT}/crypto/libsodium-fork/Makefile" ]; then + (cd "${REPO_ROOT}/crypto/libsodium-fork" && make distclean) +fi +rm -rf "${REPO_ROOT}/crypto/lib" + +# do the RPM build, sign and validate it + +cat <"${HOME}"/dummyrepo/algodummy.repo +[algodummy] +name=Algorand +baseurl=http://${DC_IP}:8111/ +enabled=1 +gpgcheck=1 +gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub +EOF +#(cd "${HOME}/dummyrepo" && python3 "${REPO_ROOT}/scripts/httpd.py" --pid "${HOME}"/phttpd.pid) & +## https://github.com/koalaman/shellcheck/wiki/SC2064 +#trap '${REPO_ROOT}/scripts/kill_httpd.sh' 0 + +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=/run/user/1000/gnupg/S.gpg-agent,dst=/S.gpg-agent --mount type=bind,src=${HOME}/dummyrepo,dst=/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/root/stuff --mount type=bind,src=${HOME}/go,dst=/root/go --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/release/helper/build_release_centos_docker.sh" + +date "+build_release done building centos %Y%m%d_%H%M%S" + +# NEXT: sign.sh + diff --git a/scripts/release/master-controller/setup.sh b/scripts/release/master-controller/setup.sh new file mode 100755 index 0000000000..f9e26a59b4 --- /dev/null +++ b/scripts/release/master-controller/setup.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# +# Externally settable env vars: +# GIT_REPO_PATH= something to `git clone` from +# GIT_CHECKOUT_LABEL= something to `git checkout` and build from (branch or tag or hash) + +if [ -z "${BUILDTIMESTAMP}" ]; then + date "+%Y%m%d_%H%M%S" > "${HOME}/buildtimestamp" + BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") + export BUILDTIMESTAMP + echo run "${0}" with output to "${HOME}/buildlog_${BUILDTIMESTAMP}" + (bash "${0}" 2>&1) | tee "${HOME}/buildlog_${BUILDTIMESTAMP}" + exit 0 +fi + +date "+setup start %Y%m%d_%H%M%S" + +set -ex + +if [ -z "${GIT_REPO_PATH}" ]; then + GIT_REPO_PATH=https://github.com/btoll/go-algorand +fi + +if [ -z "${GIT_CHECKOUT_LABEL}" ]; then + GIT_CHECKOUT_LABEL="rel/stable" +fi + +export DEBIAN_FRONTEND=noninteractive + +sudo apt-get update -q +sudo apt-get upgrade -q -y + +# Some of these dirs aren't used until later scripts. +#umask 0077 +#mkdir -p ~/.gnupg +#mkdir -p "${HOME}"/{.gnupg,go,gpgbin,docker_test_resources,dummyaptly,dummyrepo,prodrepo} +mkdir -p "${HOME}"/{.gnupg,go,gpgbin,dummyaptly,dummyrepo,prodrepo} + +export GOPATH=${HOME}/go +export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} + +cat <"${HOME}/gpgbin/remote_gpg_socket" +export GOPATH=\${HOME}/go +export PATH=\${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} +gpgconf --list-dirs|grep agent-socket|awk -F: '{ print \$2 }' +EOF + +chmod +x "${HOME}/gpgbin/remote_gpg_socket" + +sudo apt-get update +sudo apt-get install -y build-essential automake autoconf awscli docker.io git gpg nfs-common python3 rpm sqlite3 python3-boto3 g++ libtool rng-tools + +sudo rngd -r /dev/urandom + +# please keep packages sorted + +# This real name and email must precisely match GPG key +git config --global user.name "Algorand developers" +git config --global user.email dev@algorand.com + +# configure GnuPG to rely on forwarded remote gpg-agent +umask 0077 +touch "${HOME}/.gnupg/gpg.conf" +if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then + echo "" +else + echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" +fi + +if [ -f "${HOME}/key.gpg" ]; then + gpg --import "${HOME}/key.gpg" +fi +# we had a tight umask for gpg setup, but need wider for git clone below +umask 0002 + +# allow ssh to clobber unix domain sockets for gpg-agent forwarding +if grep -q ^StreamLocalBindUnlink /etc/ssh/sshd_config; then + echo already have StreamLocalBindUnlink sshd config +else + sudo bash -c "echo 'StreamLocalBindUnlink yes' >> /etc/ssh/sshd_config" + sudo systemctl restart sshd +fi + +sudo usermod -a -G docker ubuntu +sg docker "docker pull centos:7" +sg docker "docker pull ubuntu:18.04" +sg docker "docker pull ubuntu:16.04" + +# Check out +mkdir -p "${GOPATH}/src/github.com/algorand" +cd "${GOPATH}/src/github.com/algorand" && git clone --single-branch --branch build_release "${GIT_REPO_PATH}" go-algorand #cd go-algorand +#git checkout "${GIT_CHECKOUT_LABEL}" +# TODO: if we are checking out a release tag, `git tag --verify` it + +gpg --import "${GOPATH}/src/github.com/algorand/go-algorand/installer/rpm/RPM-GPG-KEY-Algorand" + +# Install latest Go +cd "$HOME" +# TODO: make a config file in root of repo with single source of truth for Go major-minor version +python3 "${GOPATH}/src/github.com/algorand/go-algorand/scripts/get_latest_go.py" --version-prefix=1.12 +# $HOME will be interpreted by the outer shell to create the string passed to sudo bash +sudo bash -c "cd /usr/local && tar zxf ${HOME}/go*.tar.gz" + +cat<> "${HOME}/.bashrc" +export EDITOR=vi +EOF + +cat<> "${HOME}/.profile" +export GOPATH=\${HOME}/go +export PATH=\${HOME}/gpgbin:\${GOPATH}/bin:/usr/local/go/bin:\${PATH} +EOF + +# Install aptly for building debian repo +mkdir -p "$GOPATH/src/github.com/aptly-dev" +cd "$GOPATH/src/github.com/aptly-dev" +git clone https://github.com/aptly-dev/aptly +cd aptly && git fetch + +# As of 2019-06-06 release tag v1.3.0 is 2018-May, GnuPG 2 support was added in October but they haven't tagged a new release yet. Hash below seems to work so far. +# 2019-07-06 v1.4.0 +git checkout v1.4.0 +make install + +gpgconf --launch gpg-agent + +date "+setup finish %Y%m%d_%H%M%S" + diff --git a/scripts/release/master-controller/sign.sh b/scripts/release/master-controller/sign.sh new file mode 100755 index 0000000000..0a6fb4bf74 --- /dev/null +++ b/scripts/release/master-controller/sign.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# shellcheck disable=1090,2129,2035 +. "${HOME}/build_env" +#set -ex + +# Anchor our repo root reference location +#REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/.. +REPO_ROOT=/home/ubuntu/go/src/github.com/algorand/go-algorand/ + +#cd "${REPO_ROOT}" + +# Tag Source +#TAG=${BRANCH}-${FULLVERSION} +#echo "TAG=${TAG}" >> "${HOME}/build_env" +# creating a signed tag is now a manual process upstream of this build +# git tag -s -u "${SIGNING_KEY_ADDR}" ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" + +# TODO +#git archive --prefix="algorand-${FULLVERSION}/" "${TAG}" | gzip > "${PKG_ROOT}/algorand_${CHANNEL}_source_${FULLVERSION}.tar.gz" + +# create *.sig gpg signatures +cd "${PKG_ROOT}" +for i in *.tar.gz *.deb *.rpm +do + gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${i}" +done +HASHFILE=hashes_${CHANNEL}_${OS}_${ARCH}_${FULLVERSION} +rm -f "${HASHFILE}" +touch "${HASHFILE}" + +# For an explanation of the "-- *.tar.gz" below +# see https://github.com/koalaman/shellcheck/wiki/SC2035 +md5sum *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 256 *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 512 *.tar.gz *.deb *.rpm >> "${HASHFILE}" + +gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${HASHFILE}" +gpg -u "${SIGNING_KEY_ADDR}" --clearsign "${HASHFILE}" + +cp -p "${REPO_ROOT}/installer/rpm/algorand.repo" "${HOME}/prodrepo/algorand.repo" + +gpg --export -a dev@algorand.com > "${HOME}/docker_test_resources/key.pub" +gpg --export -a rpm@algorand.com > "${HOME}/docker_test_resources/rpm.pub" + +#GPG_AGENT_SOCKET=$("${HOME}/gpgbin/remote_gpg_socket") +# +#sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${GPG_AGENT_SOCKET},dst=/S.gpg-agent --mount type=bind,src=${HOME}/prodrepo,dst=${HOME}/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/root/stuff --mount type=bind,src=${HOME}/go,dst=/root/go --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/sign_centos_docker.sh" + +date "+build_release done signing %Y%m%d_%H%M%S" + +# NEXT: upload.sh + diff --git a/scripts/release/shutdown_ec2_instance.sh b/scripts/release/shutdown_ec2_instance.sh new file mode 100755 index 0000000000..c2443ab01d --- /dev/null +++ b/scripts/release/shutdown_ec2_instance.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# shellcheck disable=2164 + +# shutdown_ec2_instance.sh - Invokes the build host +# +# Syntax: shutdown_ec2_instance.sh +# +# Usage: Should only be used by a build host server. +# +# Exit Code: returns 0 if instance stop successfully, non-zero otherwise +# +# Examples: scripts/buildhost/shutdown_ec2_instance.sh +# +# + +AWS_REGION="$1" +GREEN_FG=$(echo -en "\e[32m") +YELLOW_FG=$(echo -en "\e[33m") +END_FG_COLOR=$(echo -en "\e[39m") +REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )" + +#if [ "$AWS_REGION" = "" ] +#then +# echo "Missing AWS_REGION argument" +# exit 1 +#fi + +pushd "$REPO_ROOT"/tmp > /dev/null +SGID=$(cat sgid) +INSTANCE_ID=$(cat instance-id) +#INSTANCE_NAME=$(cat instance) +KEY_NAME=$(cat key-name) +popd > /dev/null + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for instance to terminate." +end=$((SECONDS+1200)) +PRIOR_INSTANCE_STATE= +while [ $SECONDS -lt $end ] +do + aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" --region "$AWS_REGION" > "$REPO_ROOT"/tmp/instance.json + INSTANCE_CODE=$(< "$REPO_ROOT"/tmp/instance.json jq '.TerminatingInstances[].CurrentState.Code') + INSTANCE_STATE=$(< "$REPO_ROOT"/tmp/instance.json jq '.TerminatingInstances[].CurrentState.Name') + + if [ "$INSTANCE_CODE" = "48" ] + then + echo "$GREEN_FG[$0]$END_FG_COLOR: Instance terminated." + break + fi + + if [ "$INSTANCE_STATE" != "$PRIOR_INSTANCE_STATE" ] + then + echo "$YELLOW_FG[$0]$END_FG_COLOR: Instance is in state $INSTANCE_STATE..." + PRIOR_INSTANCE_STATE="$INSTANCE_STATE" +# else +# cat "$REPO_ROOT"/tmp/instance.json + fi + + sleep 5s +# aws ec2 describe-instance-status --instance-id "$INSTANCE_ID" --region "$AWS_REGION" --include-all-instances > "$REPO_ROOT"/tmp/instance.json +# INSTANCE_CODE=$(< "$REPO_ROOT"/tmp/instance.json jq '.InstanceStatuses[].InstanceState.Code') +# INSTANCE_STATE=$(< "$REPO_ROOT"/tmp/instance.json jq '.InstanceStatuses[].InstanceState.Name') +# +# if [ "$INSTANCE_CODE" = "48" ] +# then +# echo "$GREEN_FG[$0]$END_FG_COLOR: Instance terminated." +# break +# fi +# +# if [ "$INSTANCE_STATE" != "$PRIOR_INSTANCE_STATE" ] +# then +# echo "$YELLOW_FG[$0]$END_FG_COLOR: Instance is in state $INSTANCE_STATE..." +# PRIOR_INSTANCE_STATE="$INSTANCE_STATE" +## else +## cat "$REPO_ROOT"/tmp/instance.json +# fi +# sleep 10s +done + +if [ "$KEY_NAME" != "" ] +then + aws ec2 delete-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" +fi + +if [ "$SGID" != "" ] +then + aws ec2 delete-security-group --group-id "$SGID" --region "$AWS_REGION" +fi + +rm -rf BuilderInstanceKey.pem "$REPO_ROOT"/tmp + diff --git a/scripts/release/socket.sh b/scripts/release/socket.sh new file mode 100755 index 0000000000..b7ec933f3c --- /dev/null +++ b/scripts/release/socket.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +#set -e + +# TODO: ssh-keyscan? +# -o StrictHostKeyChecking=no suppresses the (yes/no) new key ssh question. +# This lessens the security, but it may be acceptable in this case. + +INSTANCE=$1 +gpgp=$(find /usr/lib/gnupg{2,,1} -type f -name gpg-preset-passphrase 2> /dev/null) +KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | grep Keygrip | head -1 | awk '{ print $3 }') + +echo "enter dev@ password" +$gpgp --verbose --preset "$KEYGRIP" +echo signing test | gpg -u dev@algorand.com --clearsign + +REMOTE_GPG_SOCKET=$(ssh -o StrictHostKeyChecking=no -i BuilderInstanceKey.pem ubuntu@"$INSTANCE" gpgbin/remote_gpg_socket) +LOCAL_GPG_SOCKET=$(gpgconf --list-dirs | grep agent-socket | awk -F: '{ print $2 }') + +gpg --export -a dev@algorand.com > /tmp/dev.pub +gpg --export -a rpm@algorand.com > /tmp/rpm.pub + +# TODO: Maybe scp the public keys into the $HOME/docker... dir on the remote server? +scp -o StrictHostKeyChecking=no -i BuilderInstanceKey.pem -p /tmp/{dev,rpm}.pub ubuntu@"$INSTANCE":~/ +ssh -o StrictHostKeyChecking=no -i BuilderInstanceKey.pem ubuntu@"$INSTANCE" << EOF + gpg --import dev.pub + gpg --import rpm.pub + echo "SIGNING_KEY_ADDR=dev@algorand.com" >> build_env +EOF +ssh -o StrictHostKeyChecking=no -i BuilderInstanceKey.pem -AR "$REMOTE_GPG_SOCKET:$LOCAL_GPG_SOCKET" ubuntu@"$INSTANCE" + diff --git a/scripts/release/start_ec2_instance.sh b/scripts/release/start_ec2_instance.sh new file mode 100755 index 0000000000..c6afe5a496 --- /dev/null +++ b/scripts/release/start_ec2_instance.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# shellcheck disable=2164 + +# start_ec2_instance.sh - Invokes the build host +# +# Syntax: start_ec2_instance.sh +# +# Usage: Should only be used by a build host server. +# +# Exit Code: returns 0 if instance started successfully, non-zero otherwise +# +# Examples: scripts/buildhost/start_ec2_instance.sh +# +# Upon successfull execution, the following files would be created: +# sgid - contain the security group identifier +# BuilderInstanceKey.pem - contains the certificate required to log on to the server +# instance - contains the address of the created instance +# + +AWS_REGION="$1" +# Ubuntu Server 18.04 LTS +AWS_AMI="$2" +AWS_INSTANCE_TYPE="$3" +INSTANCE_NUMBER=$RANDOM +KEY_NAME="BuilderInstanceKey_$INSTANCE_NUMBER" +KEY_NAME_FILE="BuilderInstanceKey.pem" +SECURITY_GROUP_NAME="BuilderMachineSSH_$INSTANCE_NUMBER" +CIDR="0.0.0.0/0" +RED_FG=$(echo -en "\e[31m") +GREEN_FG=$(echo -en "\e[32m") +YELLOW_FG=$(echo -en "\e[33m") +END_FG_COLOR=$(echo -en "\e[39m") +REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )" + +cleanup () { + rm -rf "$REPO_ROOT"/tmp +} + +delete_local_key () { + rm -f "$KEY_NAME_FILE" +} + +delete_key_pair () { + if ! aws ec2 delete-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" + then + exit 1 + echo "$RED_FG[$0]$END_FG_COLOR: Key pair was not deleted!" + fi +} + +delete_security_group () { + if ! aws ec2 delete-security-group --group-id "$SGID" --region "$AWS_REGION" + then + exit 1 + echo "$RED_FG[$0]$END_FG_COLOR: Security group was not deleted!" + fi +} + +manage_instance_info () { + pushd "$REPO_ROOT"/tmp > /dev/null + rm instance*.json + echo "$SGID" > sgid + echo "$INSTANCE_NAME" > instance + echo "$INSTANCE_ID" > instance-id + echo "$KEY_NAME" > key-name + popd > /dev/null + echo "$GREEN_FG[$0]$END_FG_COLOR: Created $REPO_ROOT/tmp/ dir containing instance information." +} + +if ! SGID=$(aws ec2 create-security-group --group-name "$SECURITY_GROUP_NAME" --description "Security Group for ephemeral build machine to allow port 22" --region "$AWS_REGION" | jq -r '.GroupId') +then + exit 1 +fi + +for port in {22,5022} +do + if ! aws ec2 authorize-security-group-ingress --group-name "$SECURITY_GROUP_NAME" --protocol tcp --port $port --cidr "$CIDR" --region "$AWS_REGION" + then + delete_security_group + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem opening port $port!" + exit 1 + fi +done + +delete_local_key +if ! aws ec2 create-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" | jq -r '.KeyMaterial' > "$KEY_NAME_FILE" +then + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem creating the key pair!" + delete_security_group + delete_local_key + exit 1 +else + chmod 400 "$KEY_NAME_FILE" +fi + +mkdir -p "$REPO_ROOT/tmp" + +if ! aws ec2 run-instances --image-id "$AWS_AMI" --key-name "$KEY_NAME" --security-groups "$SECURITY_GROUP_NAME" --instance-type "$AWS_INSTANCE_TYPE" --tag-specifications "ResourceType=instance,Tags=[{Key=\"Name\",Value=\"Buildhost_Ephemeral_${INSTANCE_NUMBER}\"}, {Key=\"For\",Value=\"Buildhost_Ephemeral\"}]" --block-device-mappings '{ "DeviceName": "/dev/sda1", "Ebs": { "VolumeSize": 40 } }' --count 1 --region "$AWS_REGION" > "$REPO_ROOT"/tmp/instance.json +then + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem launching the instance! Deleting the security group and the key pair!" + delete_key_pair + delete_security_group + delete_local_key + cleanup + exit 1 +fi + +INSTANCE_ID=$(< "$REPO_ROOT"/tmp/instance.json jq -r '.Instances[].InstanceId') + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for instance to start." +end=$((SECONDS+90)) +PRIOR_INSTANCE_STATE= +while [ $SECONDS -lt $end ] +do + aws ec2 describe-instance-status --instance-id "$INSTANCE_ID" --region "$AWS_REGION" --include-all-instances > "$REPO_ROOT"/tmp/instance2.json + INSTANCE_CODE=$(< "$REPO_ROOT"/tmp/instance2.json jq '.InstanceStatuses[].InstanceState.Code') + INSTANCE_STATE=$(< "$REPO_ROOT"/tmp/instance2.json jq '.InstanceStatuses[].InstanceState.Name') + + if [ "$INSTANCE_CODE" == "16" ] + then + echo "$GREEN_FG[$0]$END_FG_COLOR: Instance started." + break + fi + + if [ "$INSTANCE_STATE" != "$PRIOR_INSTANCE_STATE" ] + then + echo "$YELLOW_FG[$0]$END_FG_COLOR: Instance is in state $INSTANCE_STATE..." + PRIOR_INSTANCE_STATE="$INSTANCE_STATE" +# else +# cat "$REPO_ROOT"/tmp/instance2.json + fi + + sleep 1s +done + +aws ec2 describe-instances --region "$AWS_REGION" --instance-id "$INSTANCE_ID" > "$REPO_ROOT"/tmp/instance2.json +INSTANCE_NAME=$(< "$REPO_ROOT"/tmp/instance2.json jq -r '.Reservations[].Instances[].PublicDnsName') +echo "$GREEN_FG[$0]$END_FG_COLOR: Instance name = $INSTANCE_NAME" + +manage_instance_info + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for SSH connection" +end=$((SECONDS+90)) +while [ $SECONDS -lt $end ] +do + if ssh -i "$KEY_NAME_FILE" -o "StrictHostKeyChecking no" "ubuntu@$INSTANCE_NAME" "uname" + then + echo "$GREEN_FG[$0]$END_FG_COLOR: SSH connection ready" + exit 0 + fi + + sleep 1s +done + +echo "$RED_FG[$0]$END_FG_COLOR: Unable to establish SSH connection" +delete_local_key +cleanup +exit 1 + diff --git a/scripts/release/tag.sh b/scripts/release/tag.sh new file mode 100755 index 0000000000..40dffb0c35 --- /dev/null +++ b/scripts/release/tag.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +TAG=foo-bar +SIGNING_KEY_ADDR=dev@algorand.com + +# # Set the TIMESTAMP to use for the genesis.json file - set here so all packages use the same number +# TIMESTAMP= +# if [ -e ./genesistimestamp.dat ]; then +# TIMESTAMP=$(cat ./genesistimestamp.dat) +# else +# TIMESTAMP=$(date +%s) +# fi +# export TIMESTAMP=${TIMESTAMP} + +cd "${HOME}"/go/src/github.com/algorand/go-algorand || exit +git tag -d "${TAG}" +git checkout HEAD +git tag -s -u "${SIGNING_KEY_ADDR}" "${TAG}" -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" +git tag --verify "${TAG}" +git push -n --tags +git push --force --tags + diff --git a/scripts/release/upload.sh b/scripts/release/upload.sh new file mode 100755 index 0000000000..533061bf48 --- /dev/null +++ b/scripts/release/upload.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# shellcheck disable=1090,2129 +# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) +# AWS_EFS_MOUNT= NFS to mount for `aptly` persistent state and scratch storage + +. "${HOME}/build_env" +set -ex + +#AWS_EFS_MOUNT=fs-31159fd2.efs.us-east-1.amazonaws.com + +# persistent storage of repo manager scratch space is on EFS +if [ ! -z "${AWS_EFS_MOUNT}" ]; then + if mount | grep -q /data + then + echo /data already mounted + else + sudo mkdir -p /data + sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport "${AWS_EFS_MOUNT}":/ /data + # make environment for release_deb.sh + sudo mkdir -p /data/_aptly + sudo chown -R "${USER}" /data/_aptly + export APTLY_DIR=/data/_aptly + fi +fi + +cd "${PKG_ROOT}" + +# copy .rpm file to intermediate yum repo scratch space, actual publish manually later +if [ ! -d /data/yumrepo ]; then + sudo mkdir -p /data/yumrepo + sudo chown "${USER}" /data/yumrepo +fi + +# For an explanation of the "./*.rpm" below +# see https://github.com/koalaman/shellcheck/wiki/SC2035 +cp -p -n ./*.rpm ./*.rpm.sig /data/yumrepo + +cd "${HOME}" +STATUSFILE=build_status_${CHANNEL}_${FULLVERSION} + +echo "ami-id:" > "${STATUSFILE}" +curl --silent http://169.254.169.254/latest/meta-data/ami-id >> "${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + + +go version: +EOF + +/usr/local/go/bin/go version >>"${STATUSFILE}" + + +############################################################ + +cat <>"${STATUSFILE}" + +go env: +EOF + +/usr/local/go/bin/go env >>"${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + +build_env: +EOF + +cat <"${HOME}"/build_env >> "${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + +dpkg-l: +EOF + +############################################################ + +dpkg -l >> "${STATUSFILE}" +gpg --clearsign "${STATUSFILE}" +gzip "${STATUSFILE}".asc + +date "+build_release done uploading %Y%m%d_%H%M%S" + +# NEXT: release_deb.sh +