diff --git a/scripts/build_release.sh b/scripts/build_release.sh index 8cfb1a3d47..e152deee07 100755 --- a/scripts/build_release.sh +++ b/scripts/build_release.sh @@ -17,23 +17,6 @@ date "+build_release start %Y%m%d_%H%M%S" set -e set -x -# persistent storage of repo manager scratch space is on EFS -if [ ! -z "${AWS_EFS_MOUNT}" ]; then - if mount|grep -q /data; then - echo /data already mounted - else - sudo mkdir -p /data - sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport "${AWS_EFS_MOUNT}":/ /data - # make environment for release_deb.sh - sudo mkdir -p /data/_aptly - sudo chown -R ${USER} /data/_aptly - export APTLY_DIR=/data/_aptly - fi -fi - -export GOPATH=${HOME}/go -export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH} - # a previous docker centos build can leave junk owned by root. chown and clean sudo chown -R ${USER} ${GOPATH} if [ -f ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork/Makefile ]; then @@ -58,6 +41,17 @@ export VARIATIONS="base" export NO_BUILD=true if [ -z "${RSTAMP}" ]; then RSTAMP=$(scripts/reverse_hex_timestamp) + echo RSTAMP=${RSTAMP} > "${HOME}/rstamp" +fi +# What's my default IP address? +# get the datacenter IP address for this EC2 host. +# this might equivalently be gotten from `netstat -rn` and `ifconfig -a` +if [ -z "${DC_IP}" ]; then + DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4) +fi +if [ -z "${DC_IP}" ]; then + echo "ERROR: need DC_IP to be set to your local (but not localhost) IP" + exit 1 fi # Update version file for this build @@ -71,7 +65,6 @@ fi echo ${BUILD_NUMBER} > ./buildnumber.dat git add -A git commit -m "Build ${BUILD_NUMBER}" -git push export FULLVERSION=$(./scripts/compute_build_number.sh -f) # a bash user might `source build_env` to manually continue a broken build @@ -89,6 +82,7 @@ export VARIATIONS=${VARIATIONS} RSTAMP=${RSTAMP} BUILD_NUMBER=${BUILD_NUMBER} export FULLVERSION=${FULLVERSION} +DC_IP=${DC_IP} EOF # strip leading 'export ' for docker --env-file sed 's/^export //g' < ${HOME}/build_env > ${HOME}/build_env_docker @@ -102,91 +96,96 @@ make build scripts/build_packages.sh "${PLATFORM}" -# Run RPM bulid in Centos7 Docker container -sg docker "docker build -t algocentosbuild - < scripts/centos-build.Dockerfile" - -# cleanup our libsodium build -if [ -f ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork/Makefile ]; then - (cd ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork && make distclean) -fi -rm -rf ${GOPATH}/src/github.com/algorand/go-algorand/crypto/lib -# do the RPM build -sg docker "docker run --env-file ${HOME}/build_env_docker --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go -a stdout -a stderr algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/build_release_centos_docker.sh" +# Test .deb installer -# Tag Source +mkdir -p ${HOME}/docker_test_resources +if [ ! -f "${HOME}/docker_test_resources/gnupg2.2.9_centos7_amd64.tar.bz2" ]; then + aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 ${HOME}/docker_test_resources +fi +cp -p "${HOME}/key.gpg" "${HOME}/docker_test_resources/key.pub" -TAG=${BRANCH}-${FULLVERSION} -if [ ! -z "${SIGNING_KEY_ADDR}" ]; then - git tag -s -u "${SIGNING_KEY_ADDR}" ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" +# copy previous installers into ~/docker_test_resources +cd "${HOME}/docker_test_resources" +if [ "${TEST_UPGRADE}" == "no" ]; then + echo "upgrade test disabled" else - git tag -s ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" -fi -git push origin ${TAG} - -git archive --prefix=algorand-${FULLVERSION}/ "${TAG}" | gzip > ${PKG_ROOT}/algorand_${CHANNEL}_source_${FULLVERSION}.tar.gz - -# create *.sig gpg signatures -cd ${PKG_ROOT} -for i in *.tar.gz *.deb *.rpm; do - gpg --detach-sign "${i}" -done -HASHFILE=hashes_${CHANNEL}_${OS}_${ARCH}_${FULLVERSION} -rm -f "${HASHFILE}" -touch "${HASHFILE}" -md5sum *.tar.gz *.deb *.rpm >> "${HASHFILE}" -shasum -a 256 *.tar.gz *.deb *.rpm >> "${HASHFILE}" -shasum -a 512 *.tar.gz *.deb *.rpm >> "${HASHFILE}" -gpg --detach-sign "${HASHFILE}" -gpg --clearsign "${HASHFILE}" - -echo RSTAMP=${RSTAMP} > "${HOME}/rstamp" -if [ ! -z "${S3_PREFIX}" ]; then - aws s3 sync --quiet --exclude dev\* --exclude master\* --exclude nightly\* --exclude stable\* --acl public-read ./ ${S3_PREFIX}/${CHANNEL}/${RSTAMP}_${FULLVERSION}/ + python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/get_current_installers.py "${S3_PREFIX}/${CHANNEL}" fi -# copy .rpm file to intermediate yum repo scratch space, actual publish manually later -if [ ! -d /data/yumrepo ]; then - sudo mkdir -p /data/yumrepo - sudo chown ${USER} /data/yumrepo -fi -cp -p -n *.rpm *.rpm.sig /data/yumrepo +echo "TEST_UPGRADE=${TEST_UPGRADE}" >> ${HOME}/build_env_docker + +rm -rf ${HOME}/dummyaptly +mkdir -p ${HOME}/dummyaptly +cat <${HOME}/dummyaptly.conf +{ + "rootDir": "${HOME}/dummyaptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": {}, + "S3PublishEndpoints": {}, + "SwiftPublishEndpoints": {} +} +EOF +aptly -config=${HOME}/dummyaptly.conf repo create -distribution=stable -component=main algodummy +aptly -config=${HOME}/dummyaptly.conf repo add algodummy ${HOME}/node_pkg/*.deb +SNAPSHOT=algodummy-$(date +%Y%m%d_%H%M%S) +aptly -config=${HOME}/dummyaptly.conf snapshot create ${SNAPSHOT} from repo algodummy +aptly -config=${HOME}/dummyaptly.conf publish snapshot -origin=Algorand -label=Algorand ${SNAPSHOT} -cd ${HOME} -STATUSFILE=build_status_${CHANNEL}_${FULLVERSION} -echo "ami-id:" > "${STATUSFILE}" -curl --silent http://169.254.169.254/latest/meta-data/ami-id >> "${STATUSFILE}" -cat <>"${STATUSFILE}" +(cd ${HOME}/dummyaptly/public && python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/httpd.py --pid ${HOME}/phttpd.pid) & -go version: -EOF -go version >>"${STATUSFILE}" -cat <>"${STATUSFILE}" +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:16.04 bash /root/go/src/github.com/algorand/go-algorand/scripts/build_release_ubuntu_test_docker.sh" +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=/usr/local/go,dst=/usr/local/go ubuntu:18.04 bash /root/go/src/github.com/algorand/go-algorand/scripts/build_release_ubuntu_test_docker.sh" -go env: -EOF -go env >>"${STATUSFILE}" -cat <>"${STATUSFILE}" +kill $(cat ${HOME}/phttpd.pid) -build_env: -EOF -cat <${HOME}/build_env>>"${STATUSFILE}" -cat <>"${STATUSFILE}" +date "+build_release done building ubuntu %Y%m%d_%H%M%S" -dpkg-l: -EOF -dpkg -l >>"${STATUSFILE}" -gpg --clearsign "${STATUSFILE}" -gzip "${STATUSFILE}.asc" -if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then - aws s3 cp --quiet "${STATUSFILE}.asc.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/${STATUSFILE}.asc.gz" +# Run RPM bulid in Centos7 Docker container +sg docker "docker build -t algocentosbuild - < ${GOPATH}/src/github.com/algorand/go-algorand/scripts/centos-build.Dockerfile" + +# cleanup our libsodium build +if [ -f ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork/Makefile ]; then + (cd ${GOPATH}/src/github.com/algorand/go-algorand/crypto/libsodium-fork && make distclean) fi +rm -rf ${GOPATH}/src/github.com/algorand/go-algorand/crypto/lib + +# do the RPM build, sign and validate it + +sudo rm -rf ${HOME}/dummyrepo +mkdir -p ${HOME}/dummyrepo + +cat <${HOME}/dummyrepo/algodummy.repo +[algodummy] +name=Algorand +baseurl=http://${DC_IP}:8111/ +enabled=1 +gpgcheck=1 +gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub +EOF +(cd ${HOME}/dummyrepo && python3 ${GOPATH}/src/github.com/algorand/go-algorand/scripts/httpd.py --pid ${HOME}/phttpd.pid) & + +sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME}/.gnupg/S.gpg-agent,dst=/S.gpg-agent --mount type=bind,src=${HOME}/dummyrepo,dst=/dummyrepo --mount type=bind,src=${HOME}/docker_test_resources,dst=/stuff --mount type=bind,src=${GOPATH}/src,dst=/root/go/src --mount type=bind,src=${HOME},dst=/root/subhome --mount type=bind,src=/usr/local/go,dst=/usr/local/go algocentosbuild /root/go/src/github.com/algorand/go-algorand/scripts/build_release_centos_docker.sh" + +kill $(cat ${HOME}/phttpd.pid) -# use aptly to push .deb to its serving repo -# Leave .deb publishing to manual step after we do more checks on the release artifacts. -# ${GOPATH}/src/github.com/algorand/go-algorand/scripts/release_deb.sh ${PKG_ROOT}/*deb +date "+build_release done building centos %Y%m%d_%H%M%S" -# TODO: manually post rpm to repo +# NEXT: build_release_sign.sh -date "+build_release finish %Y%m%d_%H%M%S" diff --git a/scripts/build_release_centos_docker.sh b/scripts/build_release_centos_docker.sh index 773be3477d..b733e36356 100644 --- a/scripts/build_release_centos_docker.sh +++ b/scripts/build_release_centos_docker.sh @@ -42,3 +42,80 @@ RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") trap "rm -rf ${RPMTMP}" 0 scripts/build_rpm.sh ${RPMTMP} cp -p ${RPMTMP}/*/*.rpm /root/subhome/node_pkg + +(cd ${HOME} && tar jxf /stuff/gnupg*.tar.bz2) +export PATH="${HOME}/gnupg2/bin:${PATH}" +export LD_LIBRARY_PATH=${HOME}/gnupg2/lib + +umask 0077 +mkdir -p ~/.gnupg +umask 0022 + +touch "${HOME}/.gnupg/gpg.conf" +if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then + echo "" +else + echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" +fi +rm -f ${HOME}/.gnupg/S.gpg-agent +(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent) + +gpg --import /stuff/key.pub +gpg --import ${GOPATH}/src/github.com/algorand/go-algorand/installer/rpm/RPM-GPG-KEY-Algorand + +cat <"${HOME}/.rpmmacros" +%_gpg_name Algorand RPM +%__gpg ${HOME}/gnupg2/bin/gpg +%__gpg_check_password_cmd true +EOF + +cat <"${HOME}/rpmsign.py" +import rpm +import sys +rpm.addSign(sys.argv[1], '') +EOF + +NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm|head -1) +python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" + +cp -p "${NEWEST_RPM}" /dummyrepo +createrepo --database /dummyrepo +rm -f /dummyrepo/repodata/repomd.xml.asc +gpg -u rpm@algorand.com --detach-sign --armor /dummyrepo/repodata/repomd.xml + +OLDRPM=$(ls -t /stuff/*.rpm|head -1) +if [ -f "${OLDRPM}" ]; then + yum install -y "${OLDRPM}" + algod -v + if algod -v | grep -q ${FULLVERSION}; then + echo "already installed current version. wat?" + false + fi + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 60 + goal node stop -d /root/testnode +fi + + +yum-config-manager --add-repo http://${DC_IP}:8111/algodummy.repo + +yum install -y algorand +algod -v +# check that the installed version is now the current version +algod -v | grep -q ${FULLVERSION} + +if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +fi + +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 60 +goal node stop -d /root/testnode + + +echo CENTOS_DOCKER_TEST_OK diff --git a/scripts/build_release_local.sh b/scripts/build_release_local.sh index 6f7f81d2af..953370c0c5 100644 --- a/scripts/build_release_local.sh +++ b/scripts/build_release_local.sh @@ -50,6 +50,8 @@ gpg -u dev@algorand.com --clearsign type some stuff ^D +gpg -u rpm@algorand.com --clearsign + # TODO: use simpler expression when we can rely on gpg 2.2 on ubuntu >= 18.04 #REMOTE_GPG_SOCKET=$(ssh ubuntu@${TARGET} gpgconf --list-dir agent-socket) @@ -75,7 +77,9 @@ export AWS_EFS_MOUNT= # to be prompted for GPG key password at a couple points. # It can still steal the outer terminal from within piping the output to tee. Nifty, huh? BUILDTIMESTAMP=$(cat "${HOME}/buildtimestamp") -(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release.sh" 2>&1)|tee -a "buildlog_${BUILDTIMESTAMP}" +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release_sign.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" +(bash "${HOME}/go/src/github.com/algorand/go-algorand/scripts/build_release_upload.sh" 2>&1)|tee -a "${HOME}/buildlog_${BUILDTIMESTAMP}" if [ -f "${HOME}/rstamp" ]; then . "${HOME}/rstamp" fi @@ -86,7 +90,7 @@ if [ -z "${RSTAMP}" ]; then echo "could not figure out RSTAMP, script must have failed early" exit 1 fi -gzip "buildlog_${BUILDTIMESTAMP}" +gzip "${HOME}/buildlog_${BUILDTIMESTAMP}" if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then - aws s3 cp "buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz" + aws s3 cp "${HOME}/buildlog_${BUILDTIMESTAMP}.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/buildlog_${BUILDTIMESTAMP}.gz" fi diff --git a/scripts/build_release_setup.sh b/scripts/build_release_setup.sh index 015922f97b..db75a4d7f9 100644 --- a/scripts/build_release_setup.sh +++ b/scripts/build_release_setup.sh @@ -47,7 +47,7 @@ chmod +x ${HOME}/gpgbin/remote_gpg_socket if [ "${DISTRIB_ID}" = "Ubuntu" ]; then if [ "${DISTRIB_RELEASE}" = "16.04" ]; then echo "WARNING: Ubuntu 16.04 is DEPRECATED" - sudo apt-get install -y autoconf awscli docker.io g++ fakeroot git gnupg2 gpgv2 make nfs-common python3 rpm sqlite3 + sudo apt-get install -y autoconf awscli docker.io g++ fakeroot git gnupg2 gpgv2 make nfs-common python3 rpm sqlite3 python3-boto3 cat <${HOME}/gpgbin/gpg #!/bin/bash exec /usr/bin/gpg2 "\$@" @@ -58,7 +58,7 @@ exec /usr/bin/gpgv2 "\$@" EOF chmod +x ${HOME}/gpgbin/* elif [ "${DISTRIB_RELEASE}" = "18.04" ]; then - sudo apt-get install -y autoconf awscli docker.io git gpg nfs-common python3 rpm sqlite3 + sudo apt-get install -y autoconf awscli docker.io git gpg nfs-common python3 rpm sqlite3 python3-boto3 else echo "don't know how to build on Ubuntu ${DISTRIB_RELEASE}" exit 1 @@ -103,6 +103,8 @@ fi sudo usermod -a -G docker ubuntu sg docker "docker pull centos:7" +sg docker "docker pull ubuntu:18.04" +sg docker "docker pull ubuntu:16.04" # Check out mkdir -p ${GOPATH}/src/github.com/algorand @@ -112,6 +114,8 @@ fi cd ${GOPATH}/src/github.com/algorand/go-algorand git checkout "${GIT_CHECKOUT_LABEL}" +gpg --import ${GOPATH}/src/github.com/algorand/go-algorand/installer/rpm/RPM-GPG-KEY-Algorand + # Install latest Go cd $HOME # TODO: make a config file in root of repo with single source of truth for Go major-minor version diff --git a/scripts/build_release_sign.sh b/scripts/build_release_sign.sh new file mode 100644 index 0000000000..bc87b6b5d5 --- /dev/null +++ b/scripts/build_release_sign.sh @@ -0,0 +1,35 @@ +#!/bin/bash +. ${HOME}/build_env +set -e +set -x + +cd ${GOPATH}/src/github.com/algorand/go-algorand + +# Tag Source +TAG=${BRANCH}-${FULLVERSION} +echo "TAG=${TAG}" >> ${HOME}/build_env +if [ ! -z "${SIGNING_KEY_ADDR}" ]; then + git tag -s -u "${SIGNING_KEY_ADDR}" ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" +else + git tag -s ${TAG} -m "Genesis Timestamp: $(cat ./genesistimestamp.dat)" +fi + +git archive --prefix=algorand-${FULLVERSION}/ "${TAG}" | gzip > ${PKG_ROOT}/algorand_${CHANNEL}_source_${FULLVERSION}.tar.gz + +# create *.sig gpg signatures +cd ${PKG_ROOT} +for i in *.tar.gz *.deb *.rpm; do + gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${i}" +done +HASHFILE=hashes_${CHANNEL}_${OS}_${ARCH}_${FULLVERSION} +rm -f "${HASHFILE}" +touch "${HASHFILE}" +md5sum *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 256 *.tar.gz *.deb *.rpm >> "${HASHFILE}" +shasum -a 512 *.tar.gz *.deb *.rpm >> "${HASHFILE}" +gpg -u "${SIGNING_KEY_ADDR}" --detach-sign "${HASHFILE}" +gpg -u "${SIGNING_KEY_ADDR}" --clearsign "${HASHFILE}" + +date "+build_release done signing %Y%m%d_%H%M%S" + +# NEXT: build_release_upload.sh diff --git a/scripts/build_release_ubuntu_test_docker.sh b/scripts/build_release_ubuntu_test_docker.sh new file mode 100644 index 0000000000..e721fed2f1 --- /dev/null +++ b/scripts/build_release_ubuntu_test_docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# test ubuntu install from inside docker image +# +# expects docker run with: +# --env-file ${HOME}/build_env_docker +# --mount type=bind,src=${HOME}/centos,dst=/stuff +# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src +# --mount type=bind,src=/usr/local/go,dst=/usr/local/go + +set -e +set -x + +export GOPATH=${HOME}/go +export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} + +apt-get update +apt-get install -y gnupg2 curl software-properties-common python3 + +if [ "${TEST_UPGRADE}" == "no" ]; then + echo "upgrade test skipped" +else + apt install -y /stuff/*.deb + algod -v + if algod -v | grep -q ${FULLVERSION}; then + echo "already installed current version. wat?" + false + fi + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 60 + goal node stop -d /root/testnode +fi + +#apt-key adv --fetch-keys https://releases.algorand.com/key.pub +apt-key add /stuff/key.pub +add-apt-repository "deb http://${DC_IP}:8111/ stable main" +apt-get update +apt-get install -y algorand +algod -v +# check that the installed version is now the current version +algod -v | grep -q ${FULLVERSION} + +if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode +fi + +goal node start -d /root/testnode +goal node wait -d /root/testnode -w 60 +goal node stop -d /root/testnode + +echo UBUNTU_DOCKER_TEST_OK diff --git a/scripts/build_release_upload.sh b/scripts/build_release_upload.sh new file mode 100644 index 0000000000..138ecb9a71 --- /dev/null +++ b/scripts/build_release_upload.sh @@ -0,0 +1,72 @@ +#!/bin/bash +. ${HOME}/build_env +set -e +set -x + +# persistent storage of repo manager scratch space is on EFS +if [ ! -z "${AWS_EFS_MOUNT}" ]; then + if mount|grep -q /data; then + echo /data already mounted + else + sudo mkdir -p /data + sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport "${AWS_EFS_MOUNT}":/ /data + # make environment for release_deb.sh + sudo mkdir -p /data/_aptly + sudo chown -R ${USER} /data/_aptly + export APTLY_DIR=/data/_aptly + fi +fi + +cd ${GOPATH}/src/github.com/algorand/go-algorand + +. ${HOME}/build_env +git push origin +git push origin ${TAG} + +cd ${PKG_ROOT} + +if [ ! -z "${S3_PREFIX}" ]; then + aws s3 sync --quiet --exclude dev\* --exclude master\* --exclude nightly\* --exclude stable\* --acl public-read ./ ${S3_PREFIX}/${CHANNEL}/${RSTAMP}_${FULLVERSION}/ +fi + +# copy .rpm file to intermediate yum repo scratch space, actual publish manually later +if [ ! -d /data/yumrepo ]; then + sudo mkdir -p /data/yumrepo + sudo chown ${USER} /data/yumrepo +fi +cp -p -n *.rpm *.rpm.sig /data/yumrepo + +cd ${HOME} +STATUSFILE=build_status_${CHANNEL}_${FULLVERSION} +echo "ami-id:" > "${STATUSFILE}" +curl --silent http://169.254.169.254/latest/meta-data/ami-id >> "${STATUSFILE}" +cat <>"${STATUSFILE}" + + +go version: +EOF +go version >>"${STATUSFILE}" +cat <>"${STATUSFILE}" + +go env: +EOF +go env >>"${STATUSFILE}" +cat <>"${STATUSFILE}" + +build_env: +EOF +cat <${HOME}/build_env>>"${STATUSFILE}" +cat <>"${STATUSFILE}" + +dpkg-l: +EOF +dpkg -l >>"${STATUSFILE}" +gpg --clearsign "${STATUSFILE}" +gzip "${STATUSFILE}.asc" +if [ ! -z "${S3_PREFIX_BUILDLOG}" ]; then + aws s3 cp --quiet "${STATUSFILE}.asc.gz" "${S3_PREFIX_BUILDLOG}/${RSTAMP}/${STATUSFILE}.asc.gz" +fi + +date "+build_release done uploading %Y%m%d_%H%M%S" + +# NEXT: release_deb.sh diff --git a/scripts/centos-build.Dockerfile b/scripts/centos-build.Dockerfile index 310d852227..ad6c520ebb 100644 --- a/scripts/centos-build.Dockerfile +++ b/scripts/centos-build.Dockerfile @@ -1,6 +1,6 @@ FROM centos:7 WORKDIR /root RUN yum install -y epel-release https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools +RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 ENTRYPOINT ["/bin/bash"] diff --git a/scripts/get_current_installers.py b/scripts/get_current_installers.py new file mode 100755 index 0000000000..ec3fe3da3d --- /dev/null +++ b/scripts/get_current_installers.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# pip install boto3 +# python3 get_current_installers.py s3://bucket/prefix + +import re +import sys + +import boto3 + +def get_stage_release_set(response): + prefix = None + they = [] + for x in response['Contents']: + path = x['Key'] + pre, fname = path.rsplit('/', 1) + if fname.startswith('tools_') or fname.startswith('install_') or fname.startswith('pending_'): + continue + if prefix is None: + prefix = pre + they.append(x) + elif prefix == pre: + they.append(x) + else: + break + return they + +# return (bucket,prefix) +def parse_s3_path(path): + m = re.match(r's3://([^/]+)/(.*)', path) + if m: + return m.group(1), m.group(2) + return None, None + +def main(): + bucket, prefix = parse_s3_path(sys.argv[1]) + s3 = boto3.client('s3') + staging_response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=100) + if (not staging_response.get('KeyCount')) or ('Contents' not in staging_response): + sys.stderr.write('nothing found under {}\n'.format(sys.argv[1])) + sys.exit(1) + rset = get_stage_release_set(staging_response) + for ob in rset: + okey = ob['Key'] + if okey.endswith('.rpm') or okey.endswith('.deb'): + _, fname = okey.rsplit('/', 1) + s3.download_file(bucket, okey, fname) + return + + +if __name__ == '__main__': + main() diff --git a/scripts/httpd.py b/scripts/httpd.py new file mode 100755 index 0000000000..5c191e4171 --- /dev/null +++ b/scripts/httpd.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import http.server +import os + +server_class = getattr(http.server, 'ThreadingHTTPServer', None) or getattr(http.server, 'HTTPServer') + +def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument('--pid', default=None) + ap.add_argument('--port', type=int, default=8111) + args = ap.parse_args() + + if args.pid: + with open(args.pid, 'w') as fout: + fout.write(str(os.getpid())) + server = server_class(('', args.port), http.server.SimpleHTTPRequestHandler) + server.serve_forever() + +if __name__ == '__main__': + main() diff --git a/scripts/release_deb.sh b/scripts/release_deb.sh index fea211c68d..44d7cfa0c4 100755 --- a/scripts/release_deb.sh +++ b/scripts/release_deb.sh @@ -43,12 +43,6 @@ cat <${HOME}/.aptly.conf "bucket":"algorand-releases", "acl":"public-read", "prefix":"deb" - }, - "algorand-dev-deb-repo": { - "region":"us-east-1", - "bucket":"algorand-dev-deb-repo", - "acl":"public-read", - "prefix":"deb" } }, "SwiftPublishEndpoints": {} @@ -56,7 +50,9 @@ cat <${HOME}/.aptly.conf EOF mkdir -p $GOPATH/src/github.com/aptly-dev -git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly || true +if [ ! -d $GOPATH/src/github.com/aptly-dev/aptly ]; then + git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly +fi (cd $GOPATH/src/github.com/aptly-dev/aptly && git fetch) # As of 2019-06-06 release tag v1.3.0 is 2018-May, GnuPG 2 support was added in October but they haven't tagged a new release yet. Hash below seems to work so far. (cd $GOPATH/src/github.com/aptly-dev/aptly && git checkout e2d6a53de5ee03814b3fe19a8954a09a5c2969b9)