diff --git a/.gitignore b/.gitignore index 71617a4836..44bec0113d 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,6 @@ crypto/libsodium-fork/build-aux/ # doc intermediates data/transactions/logic/*.md + +*.pem + diff --git a/scripts/build_release.sh b/scripts/build_release.sh deleted file mode 100755 index 2a5beaa3e5..0000000000 --- a/scripts/build_release.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env bash -# -# This script needs to be run in a terminal with a human watching to -# be prompted for GPG key password at a couple points. -# -# Externally settable env vars: -# S3_PREFIX= where to upload build artifacts (no trailing /) -# RSTAMP= `scripts/reverse_hex_timestamp` -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= - -date "+build_release start %Y%m%d_%H%M%S" - -set -e -set -x - -# Anchor our repo root reference location -REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/.. - -# a previous docker centos build can leave junk owned by root. chown and clean -sudo chown -R ${USER} ${GOPATH} -if [ -f ${REPO_ROOT}/crypto/libsodium-fork/Makefile ]; then - (cd ${REPO_ROOT}/crypto/libsodium-fork && make distclean) -fi -rm -rf ${REPO_ROOT}/crypto/lib - - -cd ${REPO_ROOT} -export RELEASE_GENESIS_PROCESS=true -PLATFORM=$(./scripts/osarchtype.sh) -PLATFORM_SPLIT=(${PLATFORM//\// }) -OS=${PLATFORM_SPLIT[0]} -ARCH=${PLATFORM_SPLIT[1]} -export BRANCH=$(./scripts/compute_branch.sh) -export CHANNEL=$(./scripts/compute_branch_channel.sh ${BRANCH}) -export DEFAULTNETWORK=$(./scripts/compute_branch_network.sh) -export PKG_ROOT=${HOME}/node_pkg -export VARIATIONS="base" -# tell underlying 'build' scripts we already built -export NO_BUILD=true -if [ -z "${RSTAMP}" ]; then - RSTAMP=$(scripts/reverse_hex_timestamp) - echo RSTAMP=${RSTAMP} > "${HOME}/rstamp" -fi -# What's my default IP address? -# get the datacenter IP address for this EC2 host. -# this might equivalently be gotten from `netstat -rn` and `ifconfig -a` -if [ -z "${DC_IP}" ]; then - DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4) -fi -if [ -z "${DC_IP}" ]; then - echo "ERROR: need DC_IP to be set to your local (but not localhost) IP" - exit 1 -fi - -# Update version file for this build -if [ ! -z "${BUILD_NUMBER}" ]; then - echo "using externally set BUILD_NUMBER=${BUILD_NUMBER} without incrementing" -else - if [ -e buildnumber.dat ]; then - BUILD_NUMBER=$(cat ./buildnumber.dat) - BUILD_NUMBER=$((${BUILD_NUMBER} + 1)) - else - BUILD_NUMBER=0 - fi - echo ${BUILD_NUMBER} > ./buildnumber.dat - git add -A - git commit -m "Build ${BUILD_NUMBER}" -fi -export FULLVERSION=$(./scripts/compute_build_number.sh -f) - -# a bash user might `source build_env` to manually continue a broken build -cat <${HOME}/build_env -export RELEASE_GENESIS_PROCESS=${RELEASE_GENESIS_PROCESS} -PLATFORM=${PLATFORM} -OS=${OS} -ARCH=${ARCH} -export BRANCH=${BRANCH} -export CHANNEL=${CHANNEL} -export DEFAULTNETWORK=${DEFAULTNETWORK} -export PKG_ROOT=${PKG_ROOT} -export VARIATIONS=${VARIATIONS} -RSTAMP=${RSTAMP} -BUILD_NUMBER=${BUILD_NUMBER} -export FULLVERSION=${FULLVERSION} -DC_IP=${DC_IP} -EOF -# strip leading 'export ' for docker --env-file -sed 's/^export //g' < ${HOME}/build_env > ${HOME}/build_env_docker - -# Build! -scripts/configure_dev.sh - -make crypto/lib/libsodium.a - -make build - -export BUILD_DEB=1 -scripts/build_packages.sh "${PLATFORM}" - -# build docker release package -cd ${REPO_ROOT}/docker/release -sg docker "./build_algod_docker.sh ${HOME}/node_pkg/node_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.tar.gz" -cd ${REPO_ROOT}/scripts - -# Test .deb installer - -. get_centos_gpg.sh - -export GNUPGHOME=${HOME}/tkey -gpgconf --kill gpg-agent -rm -rf ${GNUPGHOME} -mkdir -p ${GNUPGHOME} -chmod 700 ${GNUPGHOME} -cat >${HOME}/tkey/keygenscript<${HOME}/tkey/rpmkeygenscript<${GNUPGHOME}/gpg-agent.conf -extra-socket ${GNUPGHOME}/S.gpg-agent.extra -# inable unattended daemon mode -allow-preset-passphrase -# cache password 30 days -default-cache-ttl 2592000 -max-cache-ttl 2592000 -EOF -gpg --gen-key --batch ${HOME}/tkey/keygenscript -gpg --gen-key --batch ${HOME}/tkey/rpmkeygenscript -gpg --export -a dev@algorand.com > "${HOME}/docker_test_resources/key.pub" -gpg --export -a rpm@algorand.com > "${HOME}/docker_test_resources/rpm.pub" - -gpgconf --kill gpg-agent -gpgconf --launch gpg-agent - -gpgp=$(ls /usr/lib/gnupg{2,,1}/gpg-preset-passphrase|head -1) -KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com|grep Keygrip|head -1|awk '{ print $3 }') -echo foogorand|${gpgp} --verbose --preset ${KEYGRIP} -KEYGRIP=$(gpg -K --with-keygrip --textmode rpm@algorand.com|grep Keygrip|head -1|awk '{ print $3 }') -echo foogorand|${gpgp} --verbose --preset ${KEYGRIP} - -# copy previous installers into ~/docker_test_resources -cd "${HOME}/docker_test_resources" -if [ "${TEST_UPGRADE}" == "no" -o -z "${S3_PREFIX}" ]; then - echo "upgrade test disabled" -else - python3 ${REPO_ROOT}/scripts/get_current_installers.py "${S3_PREFIX}/${CHANNEL}" -fi - -echo "TEST_UPGRADE=${TEST_UPGRADE}" >> ${HOME}/build_env_docker - -rm -rf ${HOME}/dummyaptly -mkdir -p ${HOME}/dummyaptly -cat <${HOME}/dummyaptly.conf -{ - "rootDir": "${HOME}/dummyaptly", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "architectures": [], - "dependencyFollowSuggests": false, - "dependencyFollowRecommends": false, - "dependencyFollowAllVariants": false, - "dependencyFollowSource": false, - "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "skipLegacyPool": true, - "ppaDistributorID": "ubuntu", - "ppaCodename": "", - "skipContentsPublishing": false, - "FileSystemPublishEndpoints": {}, - "S3PublishEndpoints": {}, - "SwiftPublishEndpoints": {} -} -EOF -aptly -config=${HOME}/dummyaptly.conf repo create -distribution=stable -component=main algodummy -aptly -config=${HOME}/dummyaptly.conf repo add algodummy ${HOME}/node_pkg/*.deb -SNAPSHOT=algodummy-$(date +%Y%m%d_%H%M%S) -aptly -config=${HOME}/dummyaptly.conf snapshot create ${SNAPSHOT} from repo algodummy -aptly -config=${HOME}/dummyaptly.conf publish snapshot -origin=Algorand -label=Algorand ${SNAPSHOT} - -${REPO_ROOT}/scripts/build_release_run_ubuntu_docker_build_test.sh - -date "+build_release done building ubuntu %Y%m%d_%H%M%S" - -# Run RPM bulid in Centos7 Docker container -sg docker "docker build -t algocentosbuild - < ${REPO_ROOT}/scripts/centos-build.Dockerfile" - -# cleanup our libsodium build -if [ -f ${REPO_ROOT}/crypto/libsodium-fork/Makefile ]; then - (cd ${REPO_ROOT}/crypto/libsodium-fork && make distclean) -fi -rm -rf ${REPO_ROOT}/crypto/lib - -# do the RPM build, sign and validate it - -sudo rm -rf ${HOME}/dummyrepo -mkdir -p ${HOME}/dummyrepo - -cat <${HOME}/dummyrepo/algodummy.repo -[algodummy] -name=Algorand -baseurl=http://${DC_IP}:8111/ -enabled=1 -gpgcheck=1 -gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub -EOF -(cd ${HOME}/dummyrepo && python3 ${REPO_ROOT}/scripts/httpd.py --pid ${HOME}/phttpd.pid) & -trap ${REPO_ROOT}/scripts/kill_httpd.sh 0 - -sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${GNUPGHOME}/S.gpg-agent.extra,dst=/S.gpg-agent --mount type=bind,src=${HOME}/dummyrepo,dst=/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/build_release_centos_docker.sh" - -date "+build_release done building centos %Y%m%d_%H%M%S" - -# NEXT: build_release_sign.sh diff --git a/scripts/build_release_centos_docker.sh b/scripts/build_release_centos_docker.sh deleted file mode 100755 index cf9c9ef7fc..0000000000 --- a/scripts/build_release_centos_docker.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# build centos rpm from inside docker -# -# mount src from outside -# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src -# -# mount golang install from outside -# --mount type=bind,src=/usr/local/go,dst=/usr/local/go -# -# output copied to /root/subhome/node_pkg -# --mount type=bind,src=${HOME},dst=/root/subhome - -set -e -set -x - -export HOME=/root -mkdir -p ${HOME}/go -mkdir -p ${HOME}/go/bin -export GOPATH=${HOME}/go -export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} - -# Anchor our repo root reference location -REPO_DIR="$( cd "$(dirname "$0")" ; pwd -P )"/.. - -${REPO_DIR}/scripts/configure_dev-deps.sh - -cd ${REPO_DIR} - -# definitely rebuild libsodium which could link to external C libraries -if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then - (cd ${REPO_DIR}/crypto/libsodium-fork && make distclean) -fi -rm -rf ${REPO_DIR}/crypto/lib -make crypto/lib/libsodium.a - -make build - -export NO_BUILD=1 - -RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") -trap "rm -rf ${RPMTMP}" 0 -scripts/build_rpm.sh ${RPMTMP} -cp -p ${RPMTMP}/*/*.rpm /root/subhome/node_pkg - -(cd ${HOME} && tar jxf /stuff/gnupg*.tar.bz2) -export PATH="${HOME}/gnupg2/bin:${PATH}" -export LD_LIBRARY_PATH=${HOME}/gnupg2/lib - -umask 0077 -mkdir -p ~/.gnupg -umask 0022 - -touch "${HOME}/.gnupg/gpg.conf" -if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then - echo "" -else - echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" -fi -rm -f ${HOME}/.gnupg/S.gpg-agent -(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent) - -gpg --import /stuff/key.pub -gpg --import /stuff/rpm.pub -gpg --import ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand -rpmkeys --import /stuff/rpm.pub -echo "wat"|gpg -u rpm@algorand.com --clearsign - -cat <"${HOME}/.rpmmacros" -%_gpg_name Algorand RPM -%__gpg ${HOME}/gnupg2/bin/gpg -%__gpg_check_password_cmd true -EOF - -cat <"${HOME}/rpmsign.py" -import rpm -import sys -rpm.addSign(sys.argv[1], '') -EOF - -NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm|head -1) -python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" - -cp -p "${NEWEST_RPM}" /dummyrepo -createrepo --database /dummyrepo -rm -f /dummyrepo/repodata/repomd.xml.asc -gpg -u rpm@algorand.com --detach-sign --armor /dummyrepo/repodata/repomd.xml - -OLDRPM=$(ls -t /stuff/*.rpm|head -1) -if [ -f "${OLDRPM}" ]; then - yum install -y "${OLDRPM}" - algod -v - if algod -v | grep -q ${FULLVERSION}; then - echo "already installed current version. wat?" - false - fi - - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode - - goal node start -d /root/testnode - goal node wait -d /root/testnode -w 120 - goal node stop -d /root/testnode -fi - - -yum-config-manager --add-repo http://${DC_IP}:8111/algodummy.repo - -yum install -y algorand -algod -v -# check that the installed version is now the current version -algod -v | grep -q ${FULLVERSION}.${CHANNEL} - -if [ ! -d /root/testnode ]; then - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode -fi - -goal node start -d /root/testnode -goal node wait -d /root/testnode -w 120 -goal node stop -d /root/testnode - - -echo CENTOS_DOCKER_TEST_OK diff --git a/scripts/build_release_local.sh b/scripts/build_release_local.sh deleted file mode 100755 index 9374c5f0e4..0000000000 --- a/scripts/build_release_local.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# -# This is a file of commands to copy and paste to run build_release.sh on an AWS EC2 instance. -# Should work on Ubuntu 16.04 ro 18.04 -# -# Externally settable env vars: -# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) - -echo "this is a file of commands to copy and paste to run build_release.sh on an AWS EC2 instance" -exit 1 - -# use AWS console to create a new t3.large with the latest official Ubuntu 18.04 - -# ec2 public address here: -TARGET= - -cd ${GOPATH}/src/github.com/algorand/go-algorand - -git fetch -git checkout rel/stable -git merge origin/rel/stable -scp -p ${GOPATH}/src/github.com/algorand/go-algorand/scripts/build_release_setup.sh ubuntu@${TARGET}:~/ - -# upload the latest public key -GTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") -gpg --export --armor -o "${GTMPDIR}/key.gpg" dev@algorand.com -scp -p "${GTMPDIR}/key.gpg" "ubuntu@${TARGET}:~/key.gpg" -rm -rf ${GTMPDIR} - -ssh -A ubuntu@${TARGET} bash build_release_setup.sh - -# setup GPG key forwarding https://wiki.gnupg.org/AgentForwarding -umask 0077 -mkdir -p ${HOME}/.gnupg -touch ${HOME}/.gnupg/gpg-agent.conf -if grep -q extra-socket ${HOME}/.gnupg/gpg-agent.conf; then - echo "already have extra-socket" -else - cat <>${HOME}/.gnupg/gpg-agent.conf -extra-socket ${HOME}/.gnupg/S.gpg-agent.extra -default-cache-ttl 3600 -EOF -fi -umask 0002 - -# this will require your key password, and export a private key file protected by the same password - -# warm up your local gpg-agent -gpg -u dev@algorand.com --clearsign -type some stuff -^D - -gpg -u rpm@algorand.com --clearsign - - -# TODO: use simpler expression when we can rely on gpg 2.2 on ubuntu >= 18.04 -#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgconf --list-dir agent-socket) -#REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} "gpgconf --list-dirs|grep agent-socket|awk -F: '{ print \$2 }'") -REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgbin/remote_gpg_socket) -LOCAL_GPG_SOCKET=$(gpgconf --list-dir agent-extra-socket) -ssh -A -R "${REMOTE_GPG_SOCKET}:${LOCAL_GPG_SOCKET}" ubuntu@${TARGET} - -# check gpg agent connection -gpg -u dev@algorand.com --clearsign -blah blah -^D - - -# set AWS credentials so we can upload to S3 and connect to EFS -export AWS_ACCESS_KEY_ID= -export AWS_SECRET_ACCESS_KEY= - -# where we store persistent scratch space for aptly -export AWS_EFS_MOUNT= - -# build_release.sh needs to be run in a terminal with a human watching -# to be prompted for GPG key password at a couple points. -# It can still steal the outer terminal from within piping the output to tee. Nifty, huh? -BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") -(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" -(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release_sign.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" -(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release_upload.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" -if [ -f "${HOME}/rstamp" ]; then - . "${HOME}/rstamp" -fi -if [ -z "${RSTAMP}" ]; then - RSTAMP=$(${HOME}/go/src/github.com/algorand/go-algorand/scripts/reverse_hex_timestamp) -fi -if [ -z "${RSTAMP}" ]; then - echo "could not figure out RSTAMP, script must have failed early" - exit 1 -fi -gzip "${HOME}/buildlog_${BUILDTIMESTAMP}" -if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then - aws s3 cp "${HOME}/buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz" -fi diff --git a/scripts/build_release_run_ubuntu_docker_build_test.sh b/scripts/build_release_run_ubuntu_docker_build_test.sh deleted file mode 100755 index eb92e4f2e2..0000000000 --- a/scripts/build_release_run_ubuntu_docker_build_test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# -# This script exists to give a trap atexit context for killing the httpd so that we're not waiting on that - -set -e -set -x - -(cd ${HOME}/dummyaptly/public && python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/httpd.py --pid ${HOME}/phttpd.pid) & -trap ${GOPATH}/src/github.com/algorand/go-algorand/scripts/kill_httpd.sh 0 - -# Ubuntu 16 binaries are deprecated. Should still work to build from source for it. -#sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:16.04 bash /root/go/src/github.com/algorand/go-algorand/scripts/build_release_ubuntu_test_docker.sh" -sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:18.04 bash /root/go/src/github.com/algorand/go-algorand/scripts/build_release_ubuntu_test_docker.sh" - -export DC_IP - -sg docker "${GOPATH}/src/github.com/algorand/go-algorand/scripts/debian/start_docker_debian_test.sh ${HOME}/docker_test_resources" diff --git a/scripts/build_release_setup.sh b/scripts/build_release_setup.sh deleted file mode 100755 index d1183ad5f2..0000000000 --- a/scripts/build_release_setup.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash -# -# Externally settable env vars: -# GIT_REPO_PATH= something to `git clone` from -# GIT_CHECKOUT_LABEL= something to `git checkout` and build from (branch or tag or hash) - -if [ -z "${BUILDTIMESTAMP}" ]; then - date "+%Y%m%d_%H%M%S" > "${HOME}/buildtimestamp" - BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") - export BUILDTIMESTAMP - echo run "${0}" with output to ${HOME}/buildlog_${BUILDTIMESTAMP} - (bash "${0}" 2>&1) | tee ${HOME}/buildlog_${BUILDTIMESTAMP} - exit 0 -fi - -date "+setup start %Y%m%d_%H%M%S" - -set -e -set -x - -if [ -z "${GIT_REPO_PATH}" ]; then - GIT_REPO_PATH=git@github.com:algorand/go-algorand.git -fi - -if [ -z "${GIT_CHECKOUT_LABEL}" ]; then - GIT_CHECKOUT_LABEL="rel/stable" -fi - -export DEBIAN_FRONTEND=noninteractive - -sudo apt-get update -q -sudo apt-get upgrade -q -y - -if [ -f /etc/lsb-release ]; then - . /etc/lsb-release -fi - -mkdir -p ${HOME}/go -mkdir -p ${HOME}/gpgbin - -cat <${HOME}/gpgbin/remote_gpg_socket -export GOPATH=\${HOME}/go -export PATH=\${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} -gpgconf --list-dirs|grep agent-socket|awk -F: '{ print \$2 }' -EOF -chmod +x ${HOME}/gpgbin/remote_gpg_socket - -if [ "${DISTRIB_ID}" = "Ubuntu" ]; then - if [ "${DISTRIB_RELEASE}" = "16.04" ]; then - echo "WARNING: Ubuntu 16.04 is DEPRECATED" - sudo apt-get install -y autoconf awscli docker.io g++ fakeroot git gnupg2 gpgv2 make nfs-common python3 rpm sqlite3 python3-boto3 rng-tools - cat <${HOME}/gpgbin/gpg -#!/usr/bin/env bash -exec /usr/bin/gpg2 "\$@" -EOF - cat <${HOME}/gpgbin/gpgv -#!/usr/bin/env bash -exec /usr/bin/gpgv2 "\$@" -EOF - chmod +x ${HOME}/gpgbin/* - elif [ "${DISTRIB_RELEASE}" = "18.04" ]; then - sudo apt-get install -y autoconf awscli docker.io git gpg nfs-common python3 rpm sqlite3 python3-boto3 make g++ libtool rng-tools - else - echo "don't know how to build on Ubuntu ${DISTRIB_RELEASE}" - exit 1 - fi -else - echo "don't know how to build non Ubuntu, /etc/lsb-release[DISTRIB_ID]=${DISTRIB_ID}" - exit 1 -fi - -sudo rngd -r /dev/urandom - -export GOPATH=${HOME}/go -export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} - -# please keep packages sorted - -# This real name and email must precisely match GPG key -git config --global user.name "Algorand developers" -git config --global user.email dev@algorand.com - -# configure GnuPG to rely on forwarded remote gpg-agent -umask 0077 -mkdir -p "${HOME}/.gnupg" -touch "${HOME}/.gnupg/gpg.conf" -if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then - echo "" -else - echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" -fi - -if [ -f "${HOME}/key.gpg" ]; then - gpg --import "${HOME}/key.gpg" -fi -# we had a tight umask for gpg setup, but need wider for git clone below -umask 0002 - -# allow ssh to clobber unix domain sockets for gpg-agent forwarding -if grep -q ^StreamLocalBindUnlink /etc/ssh/sshd_config; then - echo already have StreamLocalBindUnlink sshd config -else - sudo bash -c "echo 'StreamLocalBindUnlink yes' >> /etc/ssh/sshd_config" - sudo systemctl restart sshd -fi - -sudo usermod -a -G docker ubuntu -sg docker "docker pull centos:7" -sg docker "docker pull ubuntu:18.04" -sg docker "docker pull ubuntu:16.04" - -touch ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts -if grep -q github.com ~/.ssh/known_hosts; then - echo already have github in known hosts -else -cat<> ~/.ssh/known_hosts -github.com,192.30.253.113 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== -EOF -fi - -# Check out -mkdir -p ${GOPATH}/src/github.com/algorand -if [ ! -d "${GOPATH}/src/github.com/algorand/go-algorand/.git" ]; then - (cd ${GOPATH}/src/github.com/algorand && git clone "${GIT_REPO_PATH}" go-algorand) -fi -cd ${GOPATH}/src/github.com/algorand/go-algorand -git checkout "${GIT_CHECKOUT_LABEL}" -# TODO: if we are checking out a release tag, `git tag --verify` it - -gpg --import ${GOPATH}/src/github.com/algorand/go-algorand/installer/rpm/RPM-GPG-KEY-Algorand - -# Install latest Go -cd $HOME -# TODO: make a config file in root of repo with single source of truth for Go major-minor version -if [ ! -e /usr/local/go/bin/go ]; then - python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/get_latest_go.py --version-prefix=1.12 - # $HOME will be interpreted by the outer shell to create the string passed to sudo bash - sudo bash -c "cd /usr/local && tar zxf ${HOME}/go*.tar.gz" -fi - -cat<> "${HOME}/.bashrc" -export EDITOR=vi -EOF - -cat<> "${HOME}/.profile" -export GOPATH=\${HOME}/go -export PATH=\${HOME}/gpgbin:\${GOPATH}/bin:/usr/local/go/bin:\${PATH} -EOF - -# Install aptly for building debian repo -mkdir -p $GOPATH/src/github.com/aptly-dev -if [ ! -d $GOPATH/src/github.com/aptly-dev/aptly ]; then - git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly -fi -(cd $GOPATH/src/github.com/aptly-dev/aptly && git fetch) -# As of 2019-06-06 release tag v1.3.0 is 2018-May, GnuPG 2 support was added in October but they haven't tagged a new release yet. Hash below seems to work so far. -# 2019-07-06 v1.4.0 -(cd $GOPATH/src/github.com/aptly-dev/aptly && git checkout v1.4.0) -(cd $GOPATH/src/github.com/aptly-dev/aptly && make install) - - -date "+setup finish %Y%m%d_%H%M%S" diff --git a/scripts/build_release_sign.sh b/scripts/build_release_sign.sh deleted file mode 100755 index e7b887ca9c..0000000000 --- a/scripts/build_release_sign.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -. ${HOME}/build_env -set -e -set -x - -# Anchor our repo root reference location -REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/.. - -cd ${REPO_ROOT} - -# Tag Source -TAG=${BRANCH}-${FULLVERSION} -echo "TAG=${TAG}" >> ${HOME}/build_env -# creating a signed tag is now a manual process upstream of this build -# git tag -s -u "${SIGNING_KEY_ADDR}" ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" - -git archive --prefix=algorand-${FULLVERSION}/ "${TAG}" | gzip > ${PKG_ROOT}/algorand_${CHANNEL}_source_${FULLVERSION}.tar.gz - -# create *.sig gpg signatures -cd ${PKG_ROOT} -for i in *.tar.gz *.deb *.rpm; do - gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${i}" -done -HASHFILE=hashes_${CHANNEL}_${OS}_${ARCH}_${FULLVERSION} -rm -f "${HASHFILE}" -touch "${HASHFILE}" -md5sum *.tar.gz *.deb *.rpm >> "${HASHFILE}" -shasum -a 256 *.tar.gz *.deb *.rpm >> "${HASHFILE}" -shasum -a 512 *.tar.gz *.deb *.rpm >> "${HASHFILE}" -gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${HASHFILE}" -gpg -u "${SIGNING_KEY_ADDR}" --clearsign "${HASHFILE}" - -sudo rm -rf ${HOME}/prodrepo -mkdir -p ${HOME}/prodrepo -cp -p ${REPO_ROOT}/installer/rpm/algorand.repo ${HOME}/prodrepo/algorand.repo - -. ${REPO_ROOT}/scripts/get_centos_gpg.sh -gpg --export -a dev@algorand.com > "${HOME}/docker_test_resources/key.pub" -gpg --export -a rpm@algorand.com > "${HOME}/docker_test_resources/rpm.pub" - -GPG_AGENT_SOCKET=$(${HOME}/gpgbin/remote_gpg_socket) - -sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${GPG_AGENT_SOCKET},dst=/S.gpg-agent --mount type=bind,src=${HOME}/prodrepo,dst=/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/sign_centos_docker.sh" - -date "+build_release done signing %Y%m%d_%H%M%S" - -# NEXT: build_release_upload.sh diff --git a/scripts/build_release_upload.sh b/scripts/build_release_upload.sh deleted file mode 100755 index f1b4fc1d50..0000000000 --- a/scripts/build_release_upload.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# S3_PREFIX_BUILDLOG= where upload build log (no trailing /) -# AWS_EFS_MOUNT= NFS to mount for `aptly` persistent state and scratch storage - -. ${HOME}/build_env -set -e -set -x - -# persistent storage of repo manager scratch space is on EFS -if [ ! -z "${AWS_EFS_MOUNT}" ]; then - if mount|grep -q /data; then - echo /data already mounted - else - sudo mkdir -p /data - sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport "${AWS_EFS_MOUNT}":/ /data - # make environment for release_deb.sh - sudo mkdir -p /data/_aptly - sudo chown -R ${USER} /data/_aptly - export APTLY_DIR=/data/_aptly - fi -fi - -cd ${PKG_ROOT} - -if [ ! -z "${S3_PREFIX}" ]; then - aws s3 sync --quiet --exclude dev\* --exclude master\* --exclude nightly\* --exclude stable\* --acl public-read ./ ${S3_PREFIX}/${CHANNEL}/${RSTAMP}_${FULLVERSION}/ -fi - -# copy .rpm file to intermediate yum repo scratch space, actual publish manually later -if [ ! -d /data/yumrepo ]; then - sudo mkdir -p /data/yumrepo - sudo chown ${USER} /data/yumrepo -fi -cp -p -n *.rpm *.rpm.sig /data/yumrepo - -cd ${HOME} -STATUSFILE=build_status_${CHANNEL}_${FULLVERSION} -echo "ami-id:" > "${STATUSFILE}" -curl --silent http://169.254.169.254/latest/meta-data/ami-id >> "${STATUSFILE}" -cat <>"${STATUSFILE}" - - -go version: -EOF -go version >>"${STATUSFILE}" -cat <>"${STATUSFILE}" - -go env: -EOF -go env >>"${STATUSFILE}" -cat <>"${STATUSFILE}" - -build_env: -EOF -cat <${HOME}/build_env>>"${STATUSFILE}" -cat <>"${STATUSFILE}" - -dpkg-l: -EOF -dpkg -l >>"${STATUSFILE}" -gpg --clearsign "${STATUSFILE}" -gzip "${STATUSFILE}.asc" -if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then - aws s3 cp --quiet "${STATUSFILE}.asc.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/${STATUSFILE}.asc.gz" -fi - -date "+build_release done uploading %Y%m%d_%H%M%S" - -# NEXT: release_deb.sh diff --git a/scripts/build_rpm.sh b/scripts/build_rpm.sh deleted file mode 100755 index 1e897029c6..0000000000 --- a/scripts/build_rpm.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -e - -# build_rpm.sh - Build a .deb package for one platform. -# -# Syntax: build_rpm.sh -# -# Examples: scripts/build_rpm.sh /tmp - -if [ ! "$#" -eq 1 ]; then - echo "Syntax: build_rpm.sh " - exit 1 -fi - -set -x - -OUTDIR="$1" - -export GOPATH=$(go env GOPATH) - -cd "$(dirname "$0")"/.. -export REPO_DIR=$(pwd -P) - -echo "Building RPM package" - -if [ -z "${NO_BUILD}" ]; then - env GOOS=${OS} GOARCH=${ARCH} scripts/build_prod.sh -else - echo "already built" - true -fi - -VER=$(./scripts/compute_build_number.sh -f) - -if [ "${DEFAULTNETWORK}" = "" ]; then - export DEFAULTNETWORK=$(./scripts/compute_branch_network.sh) -fi -export DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "${DEFAULTNETWORK}") - -TEMPDIR=$(mktemp -d) -trap "rm -rf $TEMPDIR" 0 -cat installer/rpm/algorand.spec \ - | sed -e s,@VER@,${VER}, \ - > ${TEMPDIR}/algorand.spec - -rpmbuild --define "_rpmdir ${OUTDIR}" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $(pwd)/COPYING" -bb ${TEMPDIR}/algorand.spec diff --git a/scripts/debian/start_docker_debian_test.sh b/scripts/debian/start_docker_debian_test.sh deleted file mode 100755 index 17bdcbce69..0000000000 --- a/scripts/debian/start_docker_debian_test.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -x -set -v -echo "Starting start_docker_debian_test.sh" -TEST_NAME="DebianTest" -echo "start docker test: " $TEST_NAME - -KEY_DIR=$1 -echo "KEY_DIR: $KEY_DIR" -echo "DC_IP: $DC_IP" -STATUS=0 - -# run the docker container -docker \ - run --rm --env-file ${HOME}/build_env_docker \ - --mount type=bind,src=${GOPATH}/src/github.com/algorand/go-algorand/scripts/debian/,dst=/workdir \ - --mount type=bind,src=${GOPATH}/src/github.com/algorand/go-algorand/test/e2e-go/cli/goal/expect,dst=/expectdir \ - --mount type=bind,src=${GOPATH}/src/github.com/algorand/go-algorand/test/testdata,dst=/testdata \ - --mount type=bind,src=${KEY_DIR},dst=/stuff \ - debian:stable \ - bash /workdir/deb_test.sh - -STATUS=$? - -echo "start_docker_debian_test completed with status: " $STATUS - -exit $STATUS diff --git a/scripts/release/README.md b/scripts/release/README.md new file mode 100644 index 0000000000..cb38e2b72e --- /dev/null +++ b/scripts/release/README.md @@ -0,0 +1,84 @@ +## Jenkins Release Build + +The `Jenkinsfile` uses the pipeline module to define its build stages. Currently, they are: + +1. create ec2 instance +1. setup ec2 instance +1. build and package +1. test +1. sign +1. upload +1. delete ec2 instance + +The only thing that is not automated is pre-setting the `gpg-agent` with the passphrase of the private key. Build execution pauses at the beginning of the `sign` stage to allow for this manual process. See below for details. + +The build job is parameterized with sensible defaults except for the Git hash, which is blank and can vary for each job. + +## Workflow + +Take a look at the Jenkins build configuration. This will set the build in motion by downloading the project from GitHub. + +## Setting up the Forwarded Connection + +To complete this step, you will need to do the following: + +1. Download the `ReleaseBuildInstanceKey.pem` certificate from the appropriate Jenkins workspace and `chmod 400` on it or GPG will complain. Move this to the `$GOPATH/src/github/algorand/go-algorand/scripts/release/controller` directory. +1. Get the instance name from AWS, i.e., `https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#Instances:sort=instanceState` or from the Jenkins workspace (`scripts/release/tmp/instance`). +1. Change to the `$GOPATH/src/github/algorand/go-algorand/scripts/release/controller` directory and execute `./socket.sh`, passing it the ec2 instance name: + + ./socket ec2-13-57-188-227.us-west-1.compute.amazonaws.com + +1. At the prompt, input the GPG passphrase (**Don't do this in a public space!!**). +1. You should now be logged into the remote machine! +1. As a sanity, it is a good idea to sign some text as a test to make sure that the connection was set up properly. Enter the following pipeline: + + echo foo | gpg -u rpm@algorand.com --clearsign + + Or, simply list the secret keys: + + gpg --list-secret-keys + + If nothing is listed, then logout and re-establish the connection. + + If there are any errors or if you are prompted for the passphrase, log out and run the above command again. + + Stay logged in! + +1. Go back to Jenkins, hover over the build step that is currently paused, and click "Proceed". + +This is all of the manual work that needs to be done. + +> You may be wondering why it's necessary to automate the GPG bits. Well, this is to circumvent the need to somehow get the private key onto the remote machine, which we definitely don't want to do. See [this explanation]. + +## Build Artifacts + +The result of running this job will be to put the build artifacts and their detached signatures in the AWS `algorand-dev-deb-repo` bucket. The location will depend on the type of artifact, of course. + +In addition, the build logs will be placed into the AWS `algorand-devops-misc` S3 bucket under `buildlog`. + +## Notes + +- All of the `aws ...` commands are now kicked off by Jenkins by shelling out to a script in the `stages` directory that is named after the relevant build stage. These scripts in `stages` simply call the appropriate script in the `controller` directory. + +- An ec2 instance is created and deleted by the `*_ec2_instance.sh` scripts in `release/`. Any pertinent information, such as the instance name and security group ID, are stored in the sub-directory `release/tmp`. This information is used by the shutdown script and then removed on a successful shutdown. + +## Troublshooting + +If testing on a server, you will get bad creds errors if your system's clock is off by even a couple minutes. Examples like the following will alert you to the problem: + +``` +An error occurred (AuthFailure) when calling the CreateSecurityGroup operation: AWS was not able to validate the provided access credentials +``` + +If you're on a debian-based system, this **should** work: + + # https://github.com/mitchellh/vagrant-aws/issues/372#issuecomment-87429450 + $ sudo apt-get install ntpdate + $ sudo ntpdate ntp.ubuntu.com + +You may also try reconfiguring your `tzdata` package: + + $ sudo dpkg-reconfigure tzdata + +[this explanation]: https://stackoverflow.com/questions/30058030/how-to-use-gpg-signing-key-on-a-remote-server + diff --git a/scripts/release/controller/Jenkinsfile b/scripts/release/controller/Jenkinsfile new file mode 100755 index 0000000000..d1556b390f --- /dev/null +++ b/scripts/release/controller/Jenkinsfile @@ -0,0 +1,83 @@ +pipeline { + parameters { + string defaultValue: '', description: 'Branch name or tag name.', name: 'hash', trim: true + string defaultValue: 'stable', description: 'Staged channel which should be released.', name: 'channel', trim: true + string defaultValue: 's3://algorand-dev-deb-repo/releases', description: 's3://bucket/prefix', name: 'bucket_location', trim: true + + // AWS + string defaultValue: 'us-west-1', description: 'AWS Region', name: 'region', trim: true + string defaultValue: 'ami-0dd655843c87b6930', description: 'Amazon Machine Image (default: Ubuntu Server 18.04 LTS, 8 vCPUs, 32 GB RAM', name: 'ami', trim: true + string defaultValue: 't2.2xlarge', description: 'Instance Type', name: 'type', trim: true + } + + environment { + AWS_ACCESS_KEY_ID = credentials("aws-access-key-id") + AWS_SECRET_ACCESS_KEY = credentials("aws-secret-access-key") + } + + agent any + + stages { + stage("create ec2 instance") { + steps { + sh script: "scripts/release/controller/stages/create.sh ${params.region} ${params.ami} ${params.type}" + } + } + + stage("setup ec2 instance") { + steps { + script { + if (params.channel == null || params.channel == "") { + error("Missing required parameter [channel].") + } + } + + sh script: "scripts/release/controller/stages/setup.sh ${params.hash} ${params.channel}" + } + } + + stage("build and package") { + steps { + sh script: "scripts/release/controller/stages/build.sh ${params.hash} ${params.channel}" + } + } + + stage("test") { + steps { + sh script: "scripts/release/controller/stages/test.sh ${params.bucket_location} ${params.channel}" + } + } + + stage("sign") { + steps { + input "Forward gpg-agent" + sh script: "scripts/release/controller/stages/sign.sh" + } + } + + stage("upload") { + steps { + script { + def rstamp = sh( + script: 'scripts/reverse_hex_timestamp', + returnStdout: true + ).trim() + + def fullversion = sh( + script: 'bash scripts/compute_build_number.sh -f', + returnStdout: true + ).trim() + + sh script: "scripts/release/controller/stages/upload.sh ${params.channel} ${params.bucket_location} ${rstamp} ${fullversion}" + } + } + } + + stage("delete ec2 instance") { + steps { + sh script: "scripts/release/controller/stages/delete.sh ${params.region}" + } + } + } +} + diff --git a/scripts/release/controller/build.sh b/scripts/release/controller/build.sh new file mode 100755 index 0000000000..f2da093369 --- /dev/null +++ b/scripts/release/controller/build.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +echo +date "+build_release begin BUILD stage %Y%m%d_%H%M%S" +echo + +set -ex + +export GOPATH=${HOME}/go +export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} + +# Anchor our repo root reference location +REPO_ROOT="${HOME}"/go/src/github.com/algorand/go-algorand/ + +cd "${REPO_ROOT}" +export RELEASE_GENESIS_PROCESS=true +export HASH="$1" +export CHANNEL="$2" +PLATFORM=$("${REPO_ROOT}"/scripts/osarchtype.sh) +PLATFORM_SPLIT=(${PLATFORM//\// }) +OS=${PLATFORM_SPLIT[0]} +ARCH=${PLATFORM_SPLIT[1]} +DEFAULTNETWORK=$(PATH=${PATH} "${REPO_ROOT}"/scripts/compute_branch_network.sh) +export DEFAULTNETWORK +export PKG_ROOT=${HOME}/node_pkg +export VARIATIONS="base" +# tell underlying 'build' scripts we already built + +# What's my default IP address? +# get the datacenter IP address for this EC2 host. +# this might equivalently be gotten from `netstat -rn` and `ifconfig -a` +DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4) +if [ -z "${DC_IP}" ]; then + echo "ERROR: need DC_IP to be set to your local (but not localhost) IP" + exit 1 +fi + +# Update version file for this build +if [ ! -z "${BUILD_NUMBER}" ]; then + echo "using externally set BUILD_NUMBER=${BUILD_NUMBER} without incrementing" +else + if [ -e buildnumber.dat ]; then + BUILD_NUMBER=$(cat ./buildnumber.dat) + BUILD_NUMBER=$(( BUILD_NUMBER + 1 )) + else + BUILD_NUMBER=0 + fi + echo ${BUILD_NUMBER} > ./buildnumber.dat + git add -A + git commit -m "Build ${BUILD_NUMBER}" +fi +FULLVERSION=$(PATH=${PATH} "${REPO_ROOT}"/scripts/compute_build_number.sh -f) +export FULLVERSION + +# a bash user might `source build_env` to manually continue a broken build +cat <>"${HOME}"/build_env +export RELEASE_GENESIS_PROCESS=${RELEASE_GENESIS_PROCESS} +PLATFORM=${PLATFORM} +OS=${OS} +ARCH=${ARCH} +export HASH=${HASH} +export CHANNEL=${CHANNEL} +export DEFAULTNETWORK=${DEFAULTNETWORK} +export PKG_ROOT=${PKG_ROOT} +export VARIATIONS=${VARIATIONS} +BUILD_NUMBER=${BUILD_NUMBER} +export FULLVERSION=${FULLVERSION} +DC_IP=${DC_IP} +REPO_ROOT=${REPO_ROOT} +EOF + +# strip leading 'export ' for docker --env-file +sed 's/^export //g' < "${HOME}"/build_env > "${HOME}"/build_env_docker + +# Build! +scripts/configure_dev.sh +make crypto/lib/libsodium.a +make build + +export BUILD_DEB=1 +export NO_BUILD=true + +"${REPO_ROOT}"/scripts/build_packages.sh "${PLATFORM}" + +# build docker release package +cd "${REPO_ROOT}"/docker/release +sg docker "./build_algod_docker.sh ${HOME}/node_pkg/node_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.tar.gz" +cd "${REPO_ROOT}"/scripts + +echo +date "+build_release end BUILD stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/controller/setup.sh b/scripts/release/controller/setup.sh new file mode 100755 index 0000000000..55c968ed6f --- /dev/null +++ b/scripts/release/controller/setup.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +if [ -z "${BUILDTIMESTAMP}" ]; then + date "+%Y%m%d_%H%M%S" > "${HOME}/buildtimestamp" + BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") + export BUILDTIMESTAMP + echo run "${0}" with output to "${HOME}/buildlog_${BUILDTIMESTAMP}" + (bash "${0}" "${1}" "${2}" 2>&1) | tee "${HOME}/buildlog_${BUILDTIMESTAMP}" + exit 0 +fi + +echo +date "+build_release begin SETUP stage %Y%m%d_%H%M%S" +echo + +set -ex + +GIT_REPO_PATH=https://github.com/algorand/go-algorand +HASH=${1:-"rel/stable"} +export HASH +CHANNEL=${1:-"stable"} +export CHANNEL +export DEBIAN_FRONTEND=noninteractive + +sudo apt-get update -q +sudo apt-get upgrade -q -y +sudo apt-get install -y build-essential automake autoconf awscli docker.io git gpg nfs-common python3 rpm sqlite3 python3-boto3 g++ libtool rng-tools +sudo rngd -r /dev/urandom + +#umask 0077 +mkdir -p "${HOME}"/{.gnupg,go,gpgbin,dummyaptly,dummyrepo,prodrepo,tkey} + +# Check out +mkdir -p "${HOME}/go/src/github.com/algorand" +cd "${HOME}/go/src/github.com/algorand" && git clone --single-branch --branch "${HASH}" "${GIT_REPO_PATH}" go-algorand +# TODO: if we are checking out a release tag, `git tag --verify` it + +# Install latest Go +# TODO: make a config file in root of repo with single source of truth for Go major-minor version +cd "${HOME}" +python3 "${HOME}/go/src/github.com/algorand/go-algorand/scripts/get_latest_go.py" --version-prefix=1.12 +# $HOME will be interpreted by the outer shell to create the string passed to sudo bash +sudo bash -c "cd /usr/local && tar zxf ${HOME}/go*.tar.gz" + +GOPATH=$(/usr/local/go/bin/go env GOPATH) +export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} +export GOPATH + +cat <"${HOME}/gpgbin/remote_gpg_socket" +export GOPATH=\${HOME}/go +export PATH=\${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} +gpgconf --list-dirs | grep agent-socket | awk -F: '{ print \$2 }' +EOF + +chmod +x "${HOME}/gpgbin/remote_gpg_socket" + +# This real name and email must precisely match GPG key +git config --global user.name "Algorand developers" +git config --global user.email dev@algorand.com + +# configure GnuPG to rely on forwarded remote gpg-agent +umask 0077 +touch "${HOME}/.gnupg/gpg.conf" +if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then + echo "" +else + echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" +fi + +if [ -f "${HOME}/key.gpg" ]; then + gpg --import "${HOME}/key.gpg" +fi +# we had a tight umask for gpg setup, but need wider for git clone below +umask 0002 + +# allow ssh to clobber unix domain sockets for gpg-agent forwarding +if grep -q ^StreamLocalBindUnlink /etc/ssh/sshd_config; then + echo already have StreamLocalBindUnlink sshd config +else + sudo bash -c "echo 'StreamLocalBindUnlink yes' >> /etc/ssh/sshd_config" + sudo systemctl restart sshd +fi + +sudo usermod -a -G docker ubuntu +sg docker "docker pull centos:7" +sg docker "docker pull ubuntu:18.04" +sg docker "docker pull ubuntu:16.04" + +cat<> "${HOME}/.bashrc" +export EDITOR=vi +EOF + +cat<> "${HOME}/.profile" +export GOPATH=\${HOME}/go +export PATH=\${HOME}/gpgbin:\${GOPATH}/bin:/usr/local/go/bin:\${PATH} +EOF + +# Install aptly for building debian repo +mkdir -p "$GOPATH/src/github.com/aptly-dev" +cd "$GOPATH/src/github.com/aptly-dev" +git clone https://github.com/aptly-dev/aptly +cd aptly && git fetch + +# As of 2019-06-06 release tag v1.3.0 is 2018-May, GnuPG 2 support was added in October but they haven't tagged a new release yet. Hash below seems to work so far. +# 2019-07-06 v1.4.0 +git checkout v1.4.0 +make install + +gpgconf --launch gpg-agent + +echo +date "+build_release end SETUP stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/controller/sign.sh b/scripts/release/controller/sign.sh new file mode 100755 index 0000000000..671a1737da --- /dev/null +++ b/scripts/release/controller/sign.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# shellcheck disable=1090,2129,2035 + +echo +date "+build_release begin SIGN stage %Y%m%d_%H%M%S" +echo + +. "${HOME}/build_env" +set -ex + +# Anchor our repo root reference location +REPO_ROOT="${HOME}"/go/src/github.com/algorand/go-algorand/ + +git archive --prefix="algorand-${FULLVERSION}/" "${TAG}" | gzip > "${PKG_ROOT}/algorand_${CHANNEL}_source_${FULLVERSION}.tar.gz" + +# create *.sig gpg signatures +cd "${PKG_ROOT}" +for i in *.tar.gz *.deb *.rpm +do + gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${i}" +done +HASHFILE=hashes_${CHANNEL}_${OS}_${ARCH}_${FULLVERSION} +rm -f "${HASHFILE}" +touch "${HASHFILE}" + +# For an explanation of the "-- *.tar.gz" below +# see https://github.com/koalaman/shellcheck/wiki/SC2035 +md5sum *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 256 *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 512 *.tar.gz *.deb *.rpm >> "${HASHFILE}" + +if [ -z "${SIGNING_KEY_ADDR}" ] +then + echo "no signing key addr" + SIGNING_KEY_ADDR=dev@algorand.com +fi + +gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${HASHFILE}" +gpg -u "${SIGNING_KEY_ADDR}" --clearsign "${HASHFILE}" + +cp -p "${REPO_ROOT}/installer/rpm/algorand.repo" "${HOME}/prodrepo/algorand.repo" + +echo +date "+build_release end SIGN stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/controller/socket.sh b/scripts/release/controller/socket.sh new file mode 100755 index 0000000000..e2fb446880 --- /dev/null +++ b/scripts/release/controller/socket.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# TODO: ssh-keyscan? +# -o StrictHostKeyChecking=no suppresses the (yes/no) new key ssh question. +# This lessens the security, but it may be acceptable in this case. + +if [ -z "$1" ] +then + echo Missing \`instance\` variable. + exit 1 +fi + +INSTANCE=$1 +gpgp=$(find /usr/lib/gnupg{2,,1} -type f -name gpg-preset-passphrase 2> /dev/null) +KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | grep Keygrip | head -1 | awk '{ print $3 }') + +echo "enter dev@ password" +$gpgp --verbose --preset "$KEYGRIP" + +REMOTE_GPG_SOCKET=$(ssh -o StrictHostKeyChecking=no -i ReleaseBuildInstanceKey.pem ubuntu@"$INSTANCE" gpgbin/remote_gpg_socket) +LOCAL_GPG_SOCKET=$(gpgconf --list-dirs | grep agent-socket | awk -F: '{ print $2 }') + +gpg --export -a dev@algorand.com > /tmp/dev.pub +gpg --export -a rpm@algorand.com > /tmp/rpm.pub + +scp -o StrictHostKeyChecking=no -i ReleaseBuildInstanceKey.pem -p /tmp/{dev,rpm}.pub ubuntu@"$INSTANCE":~/docker_test_resources/ +ssh -o StrictHostKeyChecking=no -i ReleaseBuildInstanceKey.pem ubuntu@"$INSTANCE" << EOF + gpg --import docker_test_resources/dev.pub + gpg --import docker_test_resources/rpm.pub + echo "SIGNING_KEY_ADDR=dev@algorand.com" >> build_env +EOF + +ssh -o StrictHostKeyChecking=no -i ReleaseBuildInstanceKey.pem -A -R "$REMOTE_GPG_SOCKET:$LOCAL_GPG_SOCKET" ubuntu@"$INSTANCE" + diff --git a/scripts/release/controller/stages/build.sh b/scripts/release/controller/stages/build.sh new file mode 100755 index 0000000000..01f16f02c3 --- /dev/null +++ b/scripts/release/controller/stages/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# shellcheck disable=2029 + +# Path(s) are relative to the root of the Jenkins workspace. + +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$(cat scripts/release/tmp/instance)" bash go/src/github.com/algorand/go-algorand/scripts/release/controller/build.sh "$1" "$2" + diff --git a/scripts/release/controller/stages/create.sh b/scripts/release/controller/stages/create.sh new file mode 100755 index 0000000000..9fef221a1c --- /dev/null +++ b/scripts/release/controller/stages/create.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Path(s) are relative to the root of the Jenkins workspace. + +scripts/release/start_ec2_instance.sh "$1" "$2" "$3" + diff --git a/scripts/release/controller/stages/delete.sh b/scripts/release/controller/stages/delete.sh new file mode 100755 index 0000000000..06900ae569 --- /dev/null +++ b/scripts/release/controller/stages/delete.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Path(s) are relative to the root of the Jenkins workspace. + +scripts/release/shutdown_ec2_instance.sh "$1" + diff --git a/scripts/release/controller/stages/setup.sh b/scripts/release/controller/stages/setup.sh new file mode 100755 index 0000000000..c862b39e14 --- /dev/null +++ b/scripts/release/controller/stages/setup.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# shellcheck disable=2029 + +# Path(s) are relative to the root of the Jenkins workspace. + +INSTANCE=$(cat scripts/release/tmp/instance) + +aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 . +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" mkdir docker_test_resources +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r gnupg2.2.9_centos7_amd64.tar.bz2 ubuntu@"$INSTANCE":~/docker_test_resources/ +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r scripts/release/controller/setup.sh ubuntu@"$INSTANCE":~/setup.sh +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash setup.sh "$1" "$2" + diff --git a/scripts/release/controller/stages/sign.sh b/scripts/release/controller/stages/sign.sh new file mode 100755 index 0000000000..31cf22dbe8 --- /dev/null +++ b/scripts/release/controller/stages/sign.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Path(s) are relative to the root of the Jenkins workspace. + +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$(cat scripts/release/tmp/instance)" bash go/src/github.com/algorand/go-algorand/scripts/release/controller/sign.sh + diff --git a/scripts/release/controller/stages/test.sh b/scripts/release/controller/stages/test.sh new file mode 100755 index 0000000000..6ee8d80837 --- /dev/null +++ b/scripts/release/controller/stages/test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Path(s) are relative to the root of the Jenkins workspace. + +INSTANCE=$(cat scripts/release/tmp/instance) + +rm -rf ./*.deb ./*.rpm +python3 scripts/get_current_installers.py "$1/$2" + +# Copy previous installers into ~/docker_test_resources. +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no ./*.deb ubuntu@"$INSTANCE":~/docker_test_resources/ +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash go/src/github.com/algorand/go-algorand/scripts/release/controller/test.sh + diff --git a/scripts/release/controller/stages/upload.sh b/scripts/release/controller/stages/upload.sh new file mode 100755 index 0000000000..f4b16df855 --- /dev/null +++ b/scripts/release/controller/stages/upload.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Path(s) are relative to the root of the Jenkins workspace. + +CHANNEL="$1" +BUCKET_LOCATION="$2" +RSTAMP="$3" +FULLVERSION="$4" +INSTANCE=$(cat scripts/release/tmp/instance) + +rm -rf node_pkg/* && mkdir -p node_pkg/"$RSTAMP" +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@"$INSTANCE":~/node_pkg/* node_pkg/"$RSTAMP"/ +aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read node_pkg/"$RSTAMP" "$BUCKET_LOCATION"/"$CHANNEL"/"$RSTAMP"_"$FULLVERSION"/ + +# Create the buildlog file. +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash go/src/github.com/algorand/go-algorand/scripts/release/controller/upload.sh +# sh "aws s3 cp --quiet ubuntu@\$(cat scripts/release/tmp/instance)/build_status_$CHANNEL_${FULLVERSION}.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/" +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@"$INSTANCE":~/build_status_"$CHANNEL"_*.asc.gz node_pkg/ +aws s3 cp --quiet node_pkg/build_status_"$CHANNEL"_*.asc.gz s3://algorand-devops-misc/buildlog/"$RSTAMP"/ + diff --git a/scripts/release/controller/tag.sh b/scripts/release/controller/tag.sh new file mode 100755 index 0000000000..ab00c74035 --- /dev/null +++ b/scripts/release/controller/tag.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# TODO: Ensure params are sent! +TAG=$1 +BRANCH=${2:-rel/stable} + +#pushd "${HOME}"/go/src/github.com/algorand/go-algorand || exit +pushd "${HOME}"/projects/go-algorand || exit +git checkout "${BRANCH}" + +# TODO +# There should be a discussion about what we actually want in the git tag text. +# For now, just use the Unix timestamp. +git tag -s -u dev@algorand.com "${TAG}" -m "Genesis Timestamp: $(date +%s)" +git tag --verify "${TAG}" + +git push -n --tags +git push --force --tags +popd + diff --git a/scripts/release/controller/test.sh b/scripts/release/controller/test.sh new file mode 100755 index 0000000000..644a9597ec --- /dev/null +++ b/scripts/release/controller/test.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# shellcheck disable=2012 + +echo +date "+build_release begin TEST stage %Y%m%d_%H%M%S" +echo + +REPO_ROOT=/home/ubuntu/go/src/github.com/algorand/go-algorand/ + +export GNUPGHOME=${HOME}/tkey +gpgconf --kill gpg-agent +chmod 700 "${GNUPGHOME}" + +cat > "${GNUPGHOME}"/keygenscript< "${GNUPGHOME}"/rpmkeygenscript< "${GNUPGHOME}"/gpg-agent.conf +extra-socket "${GNUPGHOME}"/S.gpg-agent.extra +# Enable unattended daemon mode. +allow-preset-passphrase +# Cache password 30 days. +default-cache-ttl 2592000 +max-cache-ttl 2592000 +EOF + +gpg --gen-key --batch "${GNUPGHOME}"/keygenscript +gpg --gen-key --batch "${GNUPGHOME}"/rpmkeygenscript +gpg --export -a dev@algorand.com > "${HOME}/docker_test_resources/key.pub" +gpg --export -a rpm@algorand.com > "${HOME}/docker_test_resources/rpm.pub" + +gpgconf --kill gpg-agent +gpgconf --launch gpg-agent + +gpgp=$(ls /usr/lib/gnupg{2,,1}/gpg-preset-passphrase | head -1) +for name in {dev,rpm} +do + KEYGRIP=$(gpg -K --with-keygrip --textmode "$name"@algorand.com | grep Keygrip | head -1 | awk '{ print $3 }') + echo foogorand | "${gpgp}" --verbose --preset "${KEYGRIP}" +done + +cat <"${HOME}"/dummyaptly.conf +{ + "rootDir": "${HOME}/dummyaptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": {}, + "S3PublishEndpoints": {}, + "SwiftPublishEndpoints": {} +} +EOF + +# Creates ~/dummyaptly/db +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf repo create -distribution=stable -component=main algodummy +# Creates ~/dummyaptly/pool +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf repo add algodummy "${HOME}"/node_pkg/*.deb +SNAPSHOT=algodummy-$(date +%Y%m%d_%H%M%S) +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf snapshot create "${SNAPSHOT}" from repo algodummy +# Creates ~/dummyaptly/public +"$HOME"/go/bin/aptly -config="${HOME}"/dummyaptly.conf publish snapshot -origin=Algorand -label=Algorand "${SNAPSHOT}" + +"${REPO_ROOT}"/scripts/release/helper/run_ubuntu_build_test.sh + +date "+build_release done building ubuntu %Y%m%d_%H%M%S" + +# Run RPM build in Centos7 Docker container +sg docker "docker build -t algocentosbuild - < ${REPO_ROOT}/scripts/release/helper/centos-build.Dockerfile" + +cat <"${HOME}"/dummyrepo/algodummy.repo +[algodummy] +name=Algorand +baseurl=http://${DC_IP}:8111/ +enabled=1 +gpgcheck=1 +gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub +EOF + +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=/run/user/1000/gnupg/S.gpg-agent,dst=/S.gpg-agent --mount type=bind,src=${HOME}/dummyrepo,dst=/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/root/stuff --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/helper/build_release_centos_docker.sh" + +echo +date "+build_release end TEST stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/controller/upload.sh b/scripts/release/controller/upload.sh new file mode 100755 index 0000000000..149a7c0bde --- /dev/null +++ b/scripts/release/controller/upload.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# shellcheck disable=1090,2129 +# AWS_EFS_MOUNT= NFS to mount for `aptly` persistent state and scratch storage + +echo +date "+build_release begin UPLOAD stage %Y%m%d_%H%M%S" +echo + +. "${HOME}/build_env" +set -ex + +#AWS_EFS_MOUNT=fs-31159fd2.efs.us-east-1.amazonaws.com + +# persistent storage of repo manager scratch space is on EFS +if [ ! -z "${AWS_EFS_MOUNT}" ]; then + if mount | grep -q /data + then + echo /data already mounted + else + sudo mkdir -p /data + sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport "${AWS_EFS_MOUNT}":/ /data + # make environment for release_deb.sh + sudo mkdir -p /data/_aptly + sudo chown -R "${USER}" /data/_aptly + export APTLY_DIR=/data/_aptly + fi +fi + +cd "${PKG_ROOT}" + +# copy .rpm file to intermediate yum repo scratch space, actual publish manually later +if [ ! -d /data/yumrepo ]; then + sudo mkdir -p /data/yumrepo + sudo chown "${USER}" /data/yumrepo +fi + +# For an explanation of the "./*.rpm" below +# see https://github.com/koalaman/shellcheck/wiki/SC2035 +cp -p -n ./*.rpm ./*.rpm.sig /data/yumrepo + +cd "${HOME}" +STATUSFILE=build_status_${CHANNEL}_${FULLVERSION} + +echo "ami-id:" > "${STATUSFILE}" +curl --silent http://169.254.169.254/latest/meta-data/ami-id >> "${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + + +go version: +EOF + +/usr/local/go/bin/go version >>"${STATUSFILE}" + + +############################################################ + +cat <>"${STATUSFILE}" + +go env: +EOF + +/usr/local/go/bin/go env >>"${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + +build_env: +EOF + +cat <"${HOME}"/build_env >> "${STATUSFILE}" + +############################################################ + +cat <>"${STATUSFILE}" + +dpkg-l: +EOF + +############################################################ + +dpkg -l >> "${STATUSFILE}" +gpg --clearsign "${STATUSFILE}" +gzip "${STATUSFILE}".asc + +"${REPO_ROOT}"/scripts/release/helper/release_deb.sh + +echo +date "+build_release end UPLOAD stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/helper/build_release_centos_docker.sh b/scripts/release/helper/build_release_centos_docker.sh new file mode 100755 index 0000000000..c92626b33a --- /dev/null +++ b/scripts/release/helper/build_release_centos_docker.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# shellcheck disable=1090,2012 +# +# build centos rpm from inside docker +# +# mount src from outside +# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src +# +# mount golang install from outside +# --mount type=bind,src=/usr/local/go,dst=/usr/local/go +# +# output copied to /root/subhome/node_pkg +# --mount type=bind,src=${HOME},dst=/root/subhome + +set -ex + +export HOME=/root + +. "${HOME}"/subhome/build_env + +GIT_REPO_PATH=https://github.com/algorand/go-algorand +mkdir -p "${HOME}/go/src/github.com/algorand" +cd "${HOME}/go/src/github.com/algorand" && git clone --single-branch --branch "${HASH}" "${GIT_REPO_PATH}" go-algorand + +# Get golang 1.12 and build its own copy of go-algorand. +cd "${HOME}" +python3 "${HOME}/go/src/github.com/algorand/go-algorand/scripts/get_latest_go.py" --version-prefix=1.12 +bash -c "cd /usr/local && tar zxf ${HOME}/go*.tar.gz" + +GOPATH=$(/usr/local/go/bin/go env GOPATH) +export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} +export GOPATH + +REPO_DIR=/root/go/src/github.com/algorand/go-algorand + +# Build! +"${REPO_DIR}"/scripts/configure_dev-deps.sh +make crypto/lib/libsodium.a -C "${REPO_DIR}" +make build -C "${REPO_DIR}" + +cd "${REPO_DIR}" + +# definitely rebuild libsodium which could link to external C libraries +#if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then +# make distclean --directory ${REPO_DIR}/crypto/libsodium-fork +#fi +#rm -rf ${REPO_DIR}/crypto/lib +#make crypto/lib/libsodium.a +# +#make build + +RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") +trap 'rm -rf ${RPMTMP}' 0 +"${REPO_DIR}/scripts/release/helper/build_rpm.sh" "${RPMTMP}" +cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg + +(cd ${HOME} && tar jxf /root/stuff/gnupg*.tar.bz2) +export PATH="${HOME}/gnupg2/bin:${PATH}" +export LD_LIBRARY_PATH=${HOME}/gnupg2/lib + +umask 0077 +mkdir -p "${HOME}/.gnupg" +umask 0022 +touch "${HOME}/.gnupg/gpg.conf" +if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then + echo "" +else + echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" +fi +rm -f ${HOME}/.gnupg/S.gpg-agent +(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent) + +gpg --import /root/stuff/key.pub +gpg --import /root/stuff/rpm.pub +#gpg --import ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand +rpmkeys --import /root/stuff/rpm.pub +echo "wat" | gpg -u rpm@algorand.com --clearsign + +cat <"${HOME}/.rpmmacros" +%_gpg_name Algorand RPM +%__gpg ${HOME}/gnupg2/bin/gpg +%__gpg_check_password_cmd true +EOF + +cat <"${HOME}/rpmsign.py" +import rpm +import sys +rpm.addSign(sys.argv[1], '') +EOF + +NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm | head -1) +python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" + +cp -p "${NEWEST_RPM}" /root/dummyrepo +createrepo --database /root/dummyrepo +rm -f /root/dummyrepo/repodata/repomd.xml.asc +gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml + +OLDRPM=$(ls -t /root/stuff/*.rpm | head -1) +if [ -f "${OLDRPM}" ]; then + yum install -y "${OLDRPM}" + algod -v + if algod -v | grep -q "${FULLVERSION}" + then + echo "already installed current version. wat?" + false + fi + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode +fi + +yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo" + +yum install -y algorand +algod -v +# check that the installed version is now the current version +algod -v | grep -q "${FULLVERSION}.${CHANNEL}" + +if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +fi + +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 120 +goal node stop -d /root/testnode + +echo CENTOS_DOCKER_TEST_OK + diff --git a/scripts/release/helper/build_rpm.sh b/scripts/release/helper/build_rpm.sh new file mode 100755 index 0000000000..822291d7ea --- /dev/null +++ b/scripts/release/helper/build_rpm.sh @@ -0,0 +1,45 @@ +#!/bin/bash -e + +# build_rpm.sh - Build a .deb package for one platform. +# +# Syntax: build_rpm.sh +# +# Examples: scripts/build_rpm.sh /tmp + +if [ ! "$#" -eq 1 ]; then + echo "Syntax: build_rpm.sh " + exit 1 +fi + +set -x + +OUTDIR="$1" + +GOPATH=$(go env GOPATH) +export GOPATH + +cd "$(dirname "$0")"/.. +#export REPO_DIR=$(pwd -P) +export REPO_DIR=$HOME/go/src/github.com/algorand/go-algorand + +echo "Building RPM package" + +#env GOOS="${OS}" GOARCH="${ARCH}" "$REPO_DIR/scripts/build_prod.sh" +env GOOS="${OS}" GOARCH="${ARCH}" make build --directory "${REPO_DIR}" + +VER=$("$REPO_DIR/scripts/compute_build_number.sh" -f) + +if [ "${DEFAULTNETWORK}" = "" ]; then + DEFAULTNETWORK=$("$REPO_DIR/scripts/compute_branch_network.sh") + export DEFAULTNETWORK +fi + +DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "${DEFAULTNETWORK}") +export DEFAULT_RELEASE_NETWORK + +TEMPDIR=$(mktemp -d) +trap 'rm -rf $TEMPDIR' 0 +< "$REPO_DIR/installer/rpm/algorand.spec" sed -e s,@VER@,"${VER}", > "${TEMPDIR}/algorand.spec" + +rpmbuild --define "_rpmdir ${OUTDIR}" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "${TEMPDIR}/algorand.spec" + diff --git a/scripts/centos-build.Dockerfile b/scripts/release/helper/centos-build.Dockerfile similarity index 79% rename from scripts/centos-build.Dockerfile rename to scripts/release/helper/centos-build.Dockerfile index 2ea453c261..6cea7767e1 100644 --- a/scripts/centos-build.Dockerfile +++ b/scripts/release/helper/centos-build.Dockerfile @@ -1,6 +1,6 @@ FROM centos:7 WORKDIR /root RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 shellcheck +RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck ENTRYPOINT ["/bin/bash"] diff --git a/scripts/debian/deb_test.sh b/scripts/release/helper/deb_test.sh similarity index 84% rename from scripts/debian/deb_test.sh rename to scripts/release/helper/deb_test.sh index 77fb151efc..edef42b3b7 100755 --- a/scripts/debian/deb_test.sh +++ b/scripts/release/helper/deb_test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -x -set -v + +#set -xv echo "deb_test starting within docker container" @@ -18,12 +18,11 @@ apt-get install -y expect algod -v echo "starting test of algod with expect script testDebian.exp" -OUTPUT=$( - expect -d /workdir/testDebian.exp /var/lib/algorand /testdata -) +OUTPUT=$(expect -d /workdir/testDebian.exp /var/lib/algorand /testdata) STATUS=$? -echo $OUTPUT +echo "$OUTPUT" echo "deb_test completed with status: " $STATUS exit $STATUS + diff --git a/scripts/release/helper/docker_debian_test.sh b/scripts/release/helper/docker_debian_test.sh new file mode 100755 index 0000000000..e2f6c66d30 --- /dev/null +++ b/scripts/release/helper/docker_debian_test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -xv +echo "Starting start_docker_debian_test.sh" +GOPATH=${HOME}/go +TEST_NAME="DebianTest" +echo "start docker test: " $TEST_NAME + +KEY_DIR=$1 +echo "KEY_DIR: $KEY_DIR" +echo "DC_IP: $DC_IP" +STATUS=0 + +# run the docker container +docker \ + run --rm --env-file "${HOME}"/build_env_docker \ + --mount type=bind,src="${GOPATH}"/src/github.com/algorand/go-algorand/scripts/release/helper/,dst=/workdir \ + --mount type=bind,src="${GOPATH}"/src/github.com/algorand/go-algorand/test/e2e-go/cli/goal/expect,dst=/expectdir \ + --mount type=bind,src="${GOPATH}"/src/github.com/algorand/go-algorand/test/testdata,dst=/testdata \ + --mount type=bind,src="${KEY_DIR}",dst=/stuff \ + debian:stable \ + bash /workdir/deb_test.sh + +STATUS=$? + +echo "start_docker_debian_test completed with status: " $STATUS + +exit $STATUS + diff --git a/scripts/build_release_ubuntu_test_docker.sh b/scripts/release/helper/docker_ubuntu_test.sh similarity index 57% rename from scripts/build_release_ubuntu_test_docker.sh rename to scripts/release/helper/docker_ubuntu_test.sh index a85552e884..c7dc88c0a4 100755 --- a/scripts/build_release_ubuntu_test_docker.sh +++ b/scripts/release/helper/docker_ubuntu_test.sh @@ -4,12 +4,11 @@ # # expects docker run with: # --env-file ${HOME}/build_env_docker -# --mount type=bind,src=${HOME}/centos,dst=/stuff +# --mount type=bind,src=${HOME}/centos,dst=/root/stuff # --mount type=bind,src=${GOPATH}/src,dst=/root/go/src # --mount type=bind,src=/usr/local/go,dst=/usr/local/go -set -e -set -x +set -ex export GOPATH=${HOME}/go export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} @@ -17,32 +16,29 @@ export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} apt-get update apt-get install -y gnupg2 curl software-properties-common python3 -if [ "${TEST_UPGRADE}" == "no" ]; then - echo "upgrade test skipped" -else - apt install -y /stuff/*.deb - algod -v - if algod -v | grep -q ${FULLVERSION}; then - echo "already installed current version. wat?" - false - fi +apt install -y /root/stuff/*.deb +algod -v +if algod -v | grep -q "${FULLVERSION}" +then + echo "already installed current version. wat?" + false +fi - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +mkdir -p /root/testnode +cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode - goal node start -d /root/testnode - goal node wait -d /root/testnode -w 120 - goal node stop -d /root/testnode -fi +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 120 +goal node stop -d /root/testnode -#apt-key adv --fetch-keys https://releases.algorand.com/key.pub -apt-key add /stuff/key.pub +#apt-key add /root/stuff/key.pub +apt-key add /root/stuff/rpm.pub add-apt-repository "deb http://${DC_IP}:8111/ stable main" apt-get update apt-get install -y algorand algod -v # check that the installed version is now the current version -algod -v | grep -q ${FULLVERSION}.${CHANNEL} +algod -v | grep -q "${FULLVERSION}.${CHANNEL}" if [ ! -d /root/testnode ]; then mkdir -p /root/testnode @@ -54,3 +50,4 @@ goal node wait -d /root/testnode -w 120 goal node stop -d /root/testnode echo UBUNTU_DOCKER_TEST_OK + diff --git a/scripts/release_deb.sh b/scripts/release/helper/release_deb.sh similarity index 77% rename from scripts/release_deb.sh rename to scripts/release/helper/release_deb.sh index 912c832320..36a96023eb 100755 --- a/scripts/release_deb.sh +++ b/scripts/release/helper/release_deb.sh @@ -5,19 +5,17 @@ # To run on an ephemeral instance, mount AWS EFS somewhere and use it: # APTLY_DIR=/large/persistent/filesystem ./release_deb.sh *.deb - -set -e -set -x +set -ex if [ -z "${APTLY_DIR}" ]; then APTLY_DIR=${HOME}/.aptly fi if [ -z "${APTLY_S3_NAME}" ]; then - APTLY_S3_NAME=algorand-releases + APTLY_S3_NAME=algorand-builds fi -cat <${HOME}/.aptly.conf +cat <"${HOME}"/.aptly.conf { "rootDir": "${APTLY_DIR}", "downloadConcurrency": 4, @@ -40,7 +38,7 @@ cat <${HOME}/.aptly.conf "S3PublishEndpoints": { "algorand-releases": { "region":"us-east-1", - "bucket":"algorand-releases", + "bucket":"ben-test-release-bucket", "acl":"public-read", "prefix":"deb" } @@ -49,17 +47,20 @@ cat <${HOME}/.aptly.conf } EOF +# "bucket":"algorand-releases", + FIRSTTIME= if aptly repo create -distribution=stable -component=main algorand; then FIRSTTIME=1 fi aptly repo add algorand "$@" SNAPSHOT=algorand-$(date +%Y%m%d_%H%M%S) -aptly snapshot create ${SNAPSHOT} from repo algorand +aptly snapshot create "${SNAPSHOT}" from repo algorand if [ ! -z "${FIRSTTIME}" ]; then echo "first publish" - aptly publish snapshot -origin=Algorand -label=Algorand ${SNAPSHOT} "s3:${APTLY_S3_NAME}:" + aptly publish snapshot -origin=Algorand -label=Algorand "${SNAPSHOT}" "s3:${APTLY_S3_NAME}:" else echo "publish snapshot ${SNAPSHOT}" - aptly publish switch stable "s3:${APTLY_S3_NAME}:" ${SNAPSHOT} + aptly publish switch stable "s3:${APTLY_S3_NAME}:" "${SNAPSHOT}" fi + diff --git a/scripts/release/helper/run_ubuntu_build_test.sh b/scripts/release/helper/run_ubuntu_build_test.sh new file mode 100755 index 0000000000..d34b7d2e53 --- /dev/null +++ b/scripts/release/helper/run_ubuntu_build_test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# This script exists to give a trap atexit context for killing the httpd so that we're not waiting on that + +set -ex + +(cd "${HOME}"/dummyaptly/public && python3 "${HOME}"/go/src/github.com/algorand/go-algorand/scripts/httpd.py --pid "${HOME}"/phttpd.pid) & +trap "${HOME}"/go/src/github.com/algorand/go-algorand/scripts/kill_httpd.sh 0 + +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/root/stuff --mount type=bind,src=${HOME}/go,dst=/root/go --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:16.04 bash /root/go/src/github.com/algorand/go-algorand/scripts/release/helper/docker_ubuntu_test.sh" + +export DC_IP + +sg docker "${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/helper/docker_debian_test.sh ${HOME}/docker_test_resources" + diff --git a/scripts/debian/testDebian.exp b/scripts/release/helper/testDebian.exp similarity index 100% rename from scripts/debian/testDebian.exp rename to scripts/release/helper/testDebian.exp diff --git a/scripts/release/shutdown_ec2_instance.sh b/scripts/release/shutdown_ec2_instance.sh new file mode 100755 index 0000000000..8cc312eeb4 --- /dev/null +++ b/scripts/release/shutdown_ec2_instance.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# shellcheck disable=2164 + +AWS_REGION="$1" +GREEN_FG=$(echo -en "\e[32m") +YELLOW_FG=$(echo -en "\e[33m") +END_FG_COLOR=$(echo -en "\e[39m") +REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )" + +if [ "$AWS_REGION" = "" ] +then + echo "Missing AWS_REGION argument" + exit 1 +fi + +pushd "$REPO_ROOT"/tmp > /dev/null +SGID=$(cat sgid) +INSTANCE_ID=$(cat instance-id) +KEY_NAME=$(cat key-name) +popd > /dev/null + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for instance to terminate." +end=$((SECONDS+1200)) +PRIOR_INSTANCE_STATE= +while [ $SECONDS -lt $end ] +do + aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" --region "$AWS_REGION" > "$REPO_ROOT"/tmp/instance.json + INSTANCE_CODE=$(< "$REPO_ROOT"/tmp/instance.json jq '.TerminatingInstances[].CurrentState.Code') + INSTANCE_STATE=$(< "$REPO_ROOT"/tmp/instance.json jq '.TerminatingInstances[].CurrentState.Name') + + if [ "$INSTANCE_CODE" = "48" ] + then + echo "$GREEN_FG[$0]$END_FG_COLOR: Instance terminated." + break + fi + + if [ "$INSTANCE_STATE" != "$PRIOR_INSTANCE_STATE" ] + then + echo "$YELLOW_FG[$0]$END_FG_COLOR: Instance is in state $INSTANCE_STATE..." + PRIOR_INSTANCE_STATE="$INSTANCE_STATE" + fi + + sleep 5s +done + +if [ "$KEY_NAME" != "" ] +then + aws ec2 delete-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" +fi + +if [ "$SGID" != "" ] +then + aws ec2 delete-security-group --group-id "$SGID" --region "$AWS_REGION" +fi + +rm -rf BuilderInstanceKey.pem "$REPO_ROOT"/tmp + diff --git a/scripts/release/start_ec2_instance.sh b/scripts/release/start_ec2_instance.sh new file mode 100755 index 0000000000..b6f8b1ad7a --- /dev/null +++ b/scripts/release/start_ec2_instance.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# shellcheck disable=2164 + +AWS_REGION="$1" +# Ubuntu Server 18.04 LTS +AWS_AMI="$2" +AWS_INSTANCE_TYPE="$3" +INSTANCE_NUMBER=$RANDOM +KEY_NAME="ReleaseBuildInstanceKey_$INSTANCE_NUMBER" +KEY_NAME_FILE="ReleaseBuildInstanceKey.pem" +SECURITY_GROUP_NAME="ReleaseBuildMachineSSH_$INSTANCE_NUMBER" +CIDR="0.0.0.0/0" +RED_FG=$(echo -en "\e[31m") +GREEN_FG=$(echo -en "\e[32m") +YELLOW_FG=$(echo -en "\e[33m") +END_FG_COLOR=$(echo -en "\e[39m") +REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )" + +cleanup () { + rm -rf "$REPO_ROOT"/tmp +} + +delete_local_key () { + rm -f "$KEY_NAME_FILE" +} + +delete_key_pair () { + if ! aws ec2 delete-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" + then + exit 1 + echo "$RED_FG[$0]$END_FG_COLOR: Key pair was not deleted!" + fi +} + +delete_security_group () { + if ! aws ec2 delete-security-group --group-id "$SGID" --region "$AWS_REGION" + then + exit 1 + echo "$RED_FG[$0]$END_FG_COLOR: Security group was not deleted!" + fi +} + +manage_instance_info () { + pushd "$REPO_ROOT"/tmp > /dev/null + rm instance*.json + echo "$SGID" > sgid + echo "$INSTANCE_NAME" > instance + echo "$INSTANCE_ID" > instance-id + echo "$KEY_NAME" > key-name + popd > /dev/null + echo "$GREEN_FG[$0]$END_FG_COLOR: Created $REPO_ROOT/tmp/ dir containing instance information." +} + +if ! SGID=$(aws ec2 create-security-group --group-name "$SECURITY_GROUP_NAME" --description "Security Group for ephemeral build machine to allow port 22" --region "$AWS_REGION" | jq -r '.GroupId') +then + exit 1 +fi + +for port in {22,5022} +do + if ! aws ec2 authorize-security-group-ingress --group-name "$SECURITY_GROUP_NAME" --protocol tcp --port $port --cidr "$CIDR" --region "$AWS_REGION" + then + delete_security_group + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem opening port $port!" + exit 1 + fi +done + +delete_local_key +if ! aws ec2 create-key-pair --key-name "$KEY_NAME" --region "$AWS_REGION" | jq -r '.KeyMaterial' > "$KEY_NAME_FILE" +then + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem creating the key pair!" + delete_security_group + delete_local_key + exit 1 +else + chmod 400 "$KEY_NAME_FILE" +fi + +mkdir -p "$REPO_ROOT/tmp" + +if ! aws ec2 run-instances --image-id "$AWS_AMI" --key-name "$KEY_NAME" --security-groups "$SECURITY_GROUP_NAME" --instance-type "$AWS_INSTANCE_TYPE" --tag-specifications "ResourceType=instance,Tags=[{Key=\"Name\",Value=\"Release_Build_Ephemeral_${INSTANCE_NUMBER}\"}, {Key=\"For\",Value=\"Release_Build_Ephemeral\"}]" --block-device-mappings '{ "DeviceName": "/dev/sda1", "Ebs": { "VolumeSize": 40 } }' --count 1 --region "$AWS_REGION" > "$REPO_ROOT"/tmp/instance.json +then + echo "$RED_FG[$0]$END_FG_COLOR: There was a problem launching the instance! Deleting the security group and the key pair!" + delete_key_pair + delete_security_group + delete_local_key + cleanup + exit 1 +fi + +INSTANCE_ID=$(< "$REPO_ROOT"/tmp/instance.json jq -r '.Instances[].InstanceId') + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for instance to start." +end=$((SECONDS+90)) +PRIOR_INSTANCE_STATE= +while [ $SECONDS -lt $end ] +do + aws ec2 describe-instance-status --instance-id "$INSTANCE_ID" --region "$AWS_REGION" --include-all-instances > "$REPO_ROOT"/tmp/instance2.json + INSTANCE_CODE=$(< "$REPO_ROOT"/tmp/instance2.json jq '.InstanceStatuses[].InstanceState.Code') + INSTANCE_STATE=$(< "$REPO_ROOT"/tmp/instance2.json jq '.InstanceStatuses[].InstanceState.Name') + + if [ "$INSTANCE_CODE" == "16" ] + then + echo "$GREEN_FG[$0]$END_FG_COLOR: Instance started." + break + fi + + if [ "$INSTANCE_STATE" != "$PRIOR_INSTANCE_STATE" ] + then + echo "$YELLOW_FG[$0]$END_FG_COLOR: Instance is in state $INSTANCE_STATE..." + PRIOR_INSTANCE_STATE="$INSTANCE_STATE" +# else +# cat "$REPO_ROOT"/tmp/instance2.json + fi + + sleep 1s +done + +aws ec2 describe-instances --region "$AWS_REGION" --instance-id "$INSTANCE_ID" > "$REPO_ROOT"/tmp/instance2.json +INSTANCE_NAME=$(< "$REPO_ROOT"/tmp/instance2.json jq -r '.Reservations[].Instances[].PublicDnsName') +echo "$GREEN_FG[$0]$END_FG_COLOR: Instance name = $INSTANCE_NAME" + +manage_instance_info + +echo "$YELLOW_FG[$0]$END_FG_COLOR: Waiting for SSH connection" +end=$((SECONDS+90)) +while [ $SECONDS -lt $end ] +do + if ssh -i "$KEY_NAME_FILE" -o "StrictHostKeyChecking no" "ubuntu@$INSTANCE_NAME" "uname" + then + echo "$GREEN_FG[$0]$END_FG_COLOR: SSH connection ready" + exit 0 + fi + + sleep 1s +done + +echo "$RED_FG[$0]$END_FG_COLOR: Unable to establish SSH connection" +delete_local_key +cleanup +exit 1 + diff --git a/scripts/sign_centos_docker.sh b/scripts/sign_centos_docker.sh deleted file mode 100755 index 32df2c4ff4..0000000000 --- a/scripts/sign_centos_docker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# sign centos rpm from inside docker - -set -e -set -x - -export HOME=/root -mkdir -p ${HOME}/go -mkdir -p ${HOME}/go/bin -export GOPATH=${HOME}/go -export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} - -(cd ${HOME} && tar jxf /stuff/gnupg*.tar.bz2) -export PATH="${HOME}/gnupg2/bin:${PATH}" -export LD_LIBRARY_PATH=${HOME}/gnupg2/lib - -umask 0077 -mkdir -p ~/.gnupg -umask 0022 - -touch "${HOME}/.gnupg/gpg.conf" -if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then - echo "" -else - echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" -fi -rm -f ${HOME}/.gnupg/S.gpg-agent -(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent) - -gpg --import /stuff/key.pub -gpg --import /stuff/rpm.pub -rpmkeys --import /stuff/rpm.pub -echo "wat"|gpg -u rpm@algorand.com --clearsign - -cat <"${HOME}/.rpmmacros" -%_gpg_name Algorand RPM -%__gpg ${HOME}/gnupg2/bin/gpg -%__gpg_check_password_cmd true -EOF - -cat <"${HOME}/rpmsign.py" -import rpm -import sys -rpm.addSign(sys.argv[1], '') -EOF - -NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm|head -1) -python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" - -cp -p "${NEWEST_RPM}" /dummyrepo -createrepo --database /dummyrepo -rm -f /dummyrepo/repodata/repomd.xml.asc -gpg -u rpm@algorand.com --detach-sign --armor /dummyrepo/repodata/repomd.xml