Skip to content

Initial commit - move scripts into new release/ dir #690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
btoll wants to merge 3 commits into from
Closed

Initial commit - move scripts into new release/ dir #690

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
08b685f
Initial commit - move scripts into new release/ dir
Jan 5, 2020
5d97c69
Addressing reviewer comments
Jan 6, 2020
608d32b
try to fix build
Jan 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,6 @@ crypto/libsodium-fork/build-aux/

# doc intermediates
data/transactions/logic/*.md

*.pem

99 changes: 99 additions & 0 deletions scripts/release/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
pipeline {
environment {
AWS_ACCESS_KEY_ID = credentials("aws-access-key-id")
AWS_SECRET_ACCESS_KEY = credentials("aws-secret-access-key")
}

agent any

stages {
stage("create ec2 instance") {
steps {
sh script: 'scripts/release/start_ec2_instance.sh us-west-1 ami-0dd655843c87b6930 t2.2xlarge'

@algobolson algobolson Jan 6, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what is that ami? comment how it was picked and when? Like, it's the "us-west-1 ubuntu 18.04 as of 2019-12-27" or so.

@btoll btoll Jan 6, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add a comment.

}
}

stage("setup ec2 instance") {
steps {
sh 'aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 .'

@tsachiherman tsachiherman Jan 6, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's ok for now, but I would prefer to move this away from S3 and into github.
( like the reminder of our build sources ).

@btoll btoll Jan 6, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I'll either move this iteration or make a note to do so.

@algobolson algobolson Jan 6, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I built that compilation of gpg for centos. Centos7 ships with a version that is too old. I statically compiled a gpg that we can unpack into Centos7 and run. S3 is the right place for that archive.

sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) mkdir docker_test_resources'
sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r gnupg2.2.9_centos7_amd64.tar.bz2 ubuntu@$(cat scripts/release/tmp/instance):~/docker_test_resources/'
sh 'scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r scripts/release/setup.sh ubuntu@$(cat scripts/release/tmp/instance):~/setup.sh'
sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash setup.sh'
}
}

stage("build and package") {
steps {
/*
script {
def PASSPHRASE = input(
id: 'passphrase',
message: 'Enter the GPG passphrase',
parameters: [
[
$class: 'StringParameterDefinition',
defaultValue: '',
description: 'GPG Passphrase',
name: 'gpg_pass'
]
])

sh "bash scripts/release/socket.sh \$(cat scripts/release/tmp/instance) ${PASSPHRASE}"
}
*/

sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/build.sh'
input "GPG remote socket"
sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/package.sh'
}
}

stage("sign") {
steps {
sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/sign.sh'
}
}

stage("upload") {
steps {
script {
RSTAMP = sh(returnStdout: true, script: 'scripts/reverse_hex_timestamp')
BRANCH = sh(returnStdout: true, script: 'bash scripts/compute_branch.sh')
CHANNEL = sh(returnStdout: true, script: 'bash scripts/compute_branch_channel.sh ${BRANCH}')
FULLVERSION = sh(returnStdout: true, script: 'bash scripts/compute_build_number.sh -f')

// Bash scripts return vars with a trailing [:space:]
BRANCH = BRANCH.replaceAll("\\s","")
CHANNEL = CHANNEL.replaceAll("\\s","")
FULLVERSION = FULLVERSION.replaceAll("\\s","")

sh "rm -rf node_pkg/* && mkdir -p node_pkg/${RSTAMP}"
sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@\$(cat scripts/release/tmp/instance):~/node_pkg/* node_pkg/${RSTAMP}/"
sh "aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read node_pkg/${RSTAMP} s3://algorand-dev-deb-repo/releases/${CHANNEL}/${RSTAMP}_${FULLVERSION}/"
// Create the buildlog file.
sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/upload.sh'
// sh "aws s3 cp --quiet ubuntu@\$(cat scripts/release/tmp/instance)/build_status_${CHANNEL}_${FULLVERSION}.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/"
sh "scp -i BuilderInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@\$(cat scripts/release/tmp/instance):~/build_status_${CHANNEL}_*.asc.gz node_pkg/"
sh "aws s3 cp --quiet node_pkg/build_status_${CHANNEL}_*.asc.gz s3://algorand-devops-misc/buildlog/${RSTAMP}/"
}
}
}

/*
stage("tag") {

@winder winder Jan 7, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to have a "push button" tag, you can use an "input":
https://jenkins.io/doc/pipeline/steps/pipeline-input-step/

I think that we probably want to reverse this though, and only run the job for tagged commits.

@btoll btoll Jan 7, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That make sense. The workflow can be manually create a tag, then input that to Jenkins to pass to the build agent.

Yes, I'm already using the input step for both pausing the build and getting user input.

steps {
sh 'ssh -i BuilderInstanceKey.pem -A ubuntu@$(cat scripts/release/tmp/instance) bash go/src/github.com/algorand/go-algorand/scripts/release/tag.sh'
}
}
*/

stage("delete ec2 instance") {
steps {
input "Halt!"
sh script: "scripts/release/shutdown_ec2_instance.sh us-west-1"
}
}
}
}

54 changes: 54 additions & 0 deletions scripts/release/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
## Jenkins Release Build

The `Jenkinsfile` uses the pipeline module to define its build stages. Currently, they are:

1. create ec2 instance
1. setup ec2 instance
1. build and package
1. sign
1. upload
1. tag (TODO)
1. delete ec2 instance

The only thing that is not automated at this point is pre-setting the `gpg-agent` with the passphrase of the private key. In the middle of the `build and package` stage, Jenkins will pause and wait for the initiator of the build to do this and set up an SSH connection that will forward a Unix socket from the remote ec2 instance to the client, which in this case is most likely your laptop.

## Workflow

Take a look at the Jenkins build configuration. This will set the build in motion by downloading the project from GitHub.

## Setting up the Forwarded Connection

To complete this step, you will need to do the following:

1. Download the `BuilderInstanceKey.pem` certificate from the appropriate Jenkins workspace and `chmod 400` on it or GPG will complain. A subsequent step will assume that you moved this to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory.
1. Get the instance name from AWS, i.e., https://us-west-1.console.aws.amazon.com/ec2/home?region=us-west-1#Instances:sort=instanceState
1. Change to the `$GOPATH/src/github/algorand/go-algorand/scripts/release` directory and execute `./socket.sh`, passing it the ec2 instance name that you just got from AWS:

./socket ec2-13-57-188-227.us-west-1.compute.amazonaws.com

1. At the prompt, input the GPG passphrase (**Don't do this in a public space!!**).
1. You should now be logged into the remote machine!
1. Go back to Jenkins, hover over the build step that is currently paused, and click "Proceed".

This is all of the manual work that needs to be done.

**Note** that I'd currently like to fully automate this, but the only way to do that is to install the GPG keys on the Jenkins production maching.

> You may be wondering why it's necessary to automate the GPG bits. Well, this is to circumvent the need to somehow get the private key onto the remote machine, which we definitely don't want to do. See this explanation.

## Build Artifacts

The result of running this job will be to put the build artifacts and their detached signatures in the AWS `algorand-dev-deb-repo` bucket. The location will depend on the type of artifact, of course.

In addition, the build logs will be placed into the AWS `algorand-devops-misc` S3 bucket under `buildlog`.

## Notes

All of the `aws ...` commands are now executed by Jenkins and are defined in the `Jenkinsfile`. The reason for this is simple: Jenkins already has the AWS auth credentials, and we don't want or need to be sending them anywhere else in the cloud.

## TODO

Create the git tag.

@algobolson algobolson Jan 6, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Disagree. I think tagging should remain a manual process. Then the build system picks that up. We decide "this is 2.1.42" and then the build system builds it.

@btoll btoll Jan 6, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I have no problem with that.

Upload the deb package via `aptly`.
Add ability to specify branch and/or channel.

108 changes: 108 additions & 0 deletions scripts/release/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
#
# This script needs to be run in a terminal with a human watching to
# be prompted for GPG key password at a couple points.
#
# Externally settable env vars:
# S3_PREFIX= where to upload build artifacts (no trailing /)
# RSTAMP= `scripts/reverse_hex_timestamp`
# AWS_ACCESS_KEY_ID=
# AWS_SECRET_ACCESS_KEY=

date "+build_release start %Y%m%d_%H%M%S"

set -ex

# Anchor our repo root reference location
# /home/ubuntu/..
# /home/ubuntu/go/src/github.com/algorand/go-algorand/
#REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/..
REPO_ROOT=/home/ubuntu/go/src/github.com/algorand/go-algorand/
#if [ -f ${REPO_ROOT}/crypto/libsodium-fork/Makefile ]; then
# (cd ${REPO_ROOT}/crypto/libsodium-fork && make distclean)
#fi
#rm -rf ${REPO_ROOT}/crypto/lib

cd ${REPO_ROOT}
export RELEASE_GENESIS_PROCESS=true
PLATFORM=$(./scripts/osarchtype.sh)
PLATFORM_SPLIT=(${PLATFORM//\// })
OS=${PLATFORM_SPLIT[0]}
ARCH=${PLATFORM_SPLIT[1]}
BRANCH=$(./scripts/compute_branch.sh)
export BRANCH
CHANNEL=$(./scripts/compute_branch_channel.sh "${BRANCH}")
export CHANNEL
DEFAULTNETWORK=$(./scripts/compute_branch_network.sh)
export DEFAULTNETWORK
export PKG_ROOT=${HOME}/node_pkg
export VARIATIONS="base"
# tell underlying 'build' scripts we already built
export NO_BUILD=true

RSTAMP=$(scripts/reverse_hex_timestamp)
echo RSTAMP="${RSTAMP}" > "${HOME}/rstamp"

# What's my default IP address?
# get the datacenter IP address for this EC2 host.
# this might equivalently be gotten from `netstat -rn` and `ifconfig -a`
DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4)
if [ -z "${DC_IP}" ]; then
echo "ERROR: need DC_IP to be set to your local (but not localhost) IP"
exit 1
fi

# Update version file for this build
if [ ! -z "${BUILD_NUMBER}" ]; then
echo "using externally set BUILD_NUMBER=${BUILD_NUMBER} without incrementing"
else
if [ -e buildnumber.dat ]; then
BUILD_NUMBER=$(cat ./buildnumber.dat)
BUILD_NUMBER=$(( BUILD_NUMBER + 1 ))
else
BUILD_NUMBER=0
fi
echo ${BUILD_NUMBER} > ./buildnumber.dat
git add -A
git commit -m "Build ${BUILD_NUMBER}"
fi
FULLVERSION=$(./scripts/compute_build_number.sh -f)
export FULLVERSION

# a bash user might `source build_env` to manually continue a broken build
cat <<EOF>"${HOME}"/build_env
export RELEASE_GENESIS_PROCESS=${RELEASE_GENESIS_PROCESS}
PLATFORM=${PLATFORM}
OS=${OS}
ARCH=${ARCH}
export BRANCH=${BRANCH}
export CHANNEL=${CHANNEL}
export DEFAULTNETWORK=${DEFAULTNETWORK}
export PKG_ROOT=${PKG_ROOT}
export VARIATIONS=${VARIATIONS}
RSTAMP=${RSTAMP}
BUILD_NUMBER=${BUILD_NUMBER}
export FULLVERSION=${FULLVERSION}
DC_IP=${DC_IP}
EOF
# strip leading 'export ' for docker --env-file
sed 's/^export //g' < "${HOME}"/build_env > "${HOME}"/build_env_docker

export GOPATH=${HOME}/go
export PATH=${HOME}/gpgbin:${GOPATH}/bin:/usr/local/go/bin:${PATH}

# Build!
scripts/configure_dev.sh
make crypto/lib/libsodium.a

@tsachiherman tsachiherman Jan 6, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why ? doesn't make build creates that automatically ?

@btoll btoll Jan 6, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll have to check with Brian to see if there's a reason to build this.

@algobolson algobolson Jan 7, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this shouldn't be necessary, here and a couple other places, make build should take care of building libsodium, but I guess at one point in the past this worked around a finicky build. Probably we should try without it and see if we don't need it and maybe this time we can take the time to fix it right, or comment here why it's a good workaround

@tsachiherman tsachiherman Jan 7, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Brian is correct. At some point in the past it was needed.
Today, assuming that you're building on a clean environment every time, there shouldn't be a need to do that. ( but please do correct me if I'm wrong )

@btoll btoll Jan 8, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've found that if I don't include this right before we make the rpm package that it won't build.

For example, if I comment out the code here:

# definitely rebuild libsodium which could link to external C libraries
#if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then
#    (cd ${REPO_DIR}/crypto/libsodium-fork && make distclean)
#fi
#rm -rf ${REPO_DIR}/crypto/lib
#make crypto/lib/libsodium.a

make build

RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
trap 'rm -rf ${RPMTMP}' 0
"${REPO_DIR}/scripts/release/helper/build_rpm.sh" "${RPMTMP}"

I get the following build errors:

crypto/lib/libsodium.a(libsodium_la-utils.o): In function `explicit_bzero':
/usr/include/x86_64-linux-gnu/bits/string_fortified.h:83: undefined reference to `__explicit_bzero_chk'
/usr/include/x86_64-linux-gnu/bits/string_fortified.h:83: undefined reference to `__explicit_bzero_chk'
/usr/include/x86_64-linux-gnu/bits/string_fortified.h:83: undefined reference to `__explicit_bzero_chk'
collect2: error: ld returned 1 exit status
make: *** [buildsrc] Error 2
build_release done packaging centos 20200108_033715

@algobolson algobolson Jan 8, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, inside the rpm build inside the centos docker container I know we need the cleanup lines. The outer ubuntu build and the inner centos build are using the same checkout to ensure they have exactly the same source version. (We might be able to get rid of the entire 'centos build' now, we should do testing that current builds built on ubuntu18 run on ubuntu16 and centos7; this should work because of Tsachi's changes to make the build fully static)

The question I had about the build scripts was can we just skip the line make crypto/lib/libsodium.a that shows up in a few places.

make build

export BUILD_DEB=1
scripts/build_packages.sh "${PLATFORM}"

# build docker release package
cd ${REPO_ROOT}/docker/release
sg docker "./build_algod_docker.sh ${HOME}/node_pkg/node_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.tar.gz"
cd ${REPO_ROOT}/scripts

# NEXT: package.sh

122 changes: 122 additions & 0 deletions scripts/release/helper/build_release_centos_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/usr/bin/env bash
# shellcheck disable=2012
#
# build centos rpm from inside docker
#
# mount src from outside
# --mount type=bind,src=${GOPATH}/src,dst=/root/go/src
#
# mount golang install from outside
# --mount type=bind,src=/usr/local/go,dst=/usr/local/go
#
# output copied to /root/subhome/node_pkg
# --mount type=bind,src=${HOME},dst=/root/subhome

set -ex

export HOME=/root
export GOPATH=${HOME}/go
export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}

# Anchor our repo root reference location
#REPO_DIR="$( cd "$(dirname "$0")" ; pwd -P )"/..
REPO_DIR=/root/go/src/github.com/algorand/go-algorand

${REPO_DIR}/scripts/configure_dev-deps.sh

cd ${REPO_DIR}

# definitely rebuild libsodium which could link to external C libraries
if [ -f ${REPO_DIR}/crypto/libsodium-fork/Makefile ]; then
(cd ${REPO_DIR}/crypto/libsodium-fork && make distclean)
fi
rm -rf ${REPO_DIR}/crypto/lib
make crypto/lib/libsodium.a

make build

#export NO_BUILD=1

RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp")
trap 'rm -rf ${RPMTMP}' 0
"${REPO_DIR}/scripts/release/helper/build_rpm.sh" "${RPMTMP}"
cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg

(cd ${HOME} && tar jxf /root/stuff/gnupg*.tar.bz2)
export PATH="${HOME}/gnupg2/bin:${PATH}"
export LD_LIBRARY_PATH=${HOME}/gnupg2/lib

umask 0077
mkdir -p "${HOME}/.gnupg"
umask 0022
touch "${HOME}/.gnupg/gpg.conf"
if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then
echo ""
else
echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf"
fi
rm -f ${HOME}/.gnupg/S.gpg-agent
(cd ~/.gnupg && ln -s /S.gpg-agent S.gpg-agent)

gpg --import /root/stuff/key.pub
gpg --import /root/stuff/rpm.pub
gpg --import ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand
rpmkeys --import /root/stuff/rpm.pub
echo "wat" | gpg -u rpm@algorand.com --clearsign

cat <<EOF>"${HOME}/.rpmmacros"
%_gpg_name ALGORAND RPM <rpm@algorand.com>
%__gpg ${HOME}/gnupg2/bin/gpg
%__gpg_check_password_cmd true
EOF

cat <<EOF>"${HOME}/rpmsign.py"
import rpm
import sys
rpm.addSign(sys.argv[1], '')
EOF

NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm | head -1)
python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}"

cp -p "${NEWEST_RPM}" /root/dummyrepo
createrepo --database /root/dummyrepo
rm -f /root/dummyrepo/repodata/repomd.xml.asc
gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml

OLDRPM=$(ls -t /root/stuff/*.rpm | head -1)
if [ -f "${OLDRPM}" ]; then
yum install -y "${OLDRPM}"
algod -v
if algod -v | grep -q "${FULLVERSION}"
then
echo "already installed current version. wat?"
false
fi

mkdir -p /root/testnode
cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode

goal node start -d /root/testnode
goal node wait -d /root/testnode -w 120
goal node stop -d /root/testnode
fi

yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo"

yum install -y algorand
algod -v
# check that the installed version is now the current version
algod -v | grep -q "${FULLVERSION}.${CHANNEL}"

if [ ! -d /root/testnode ]; then
mkdir -p /root/testnode
cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode
fi

goal node start -d /root/testnode
goal node wait -d /root/testnode -w 120
goal node stop -d /root/testnode

echo CENTOS_DOCKER_TEST_OK

Loading