diff --git a/Dockerfile.binaries b/Dockerfile.binaries new file mode 100644 index 0000000000..05855c02a1 --- /dev/null +++ b/Dockerfile.binaries @@ -0,0 +1,56 @@ +# Purpose: +# ------- +# This docker image is used by the ./cloudbuild-binaries.yaml CI flow to +# cross-compile geth for different platforms. +# At the time of writing this is only linux-{386/amd64} but there are two +# pending update that will add darwin-{386,amd64} and windows-amd64 support. +# +# How to test changes to this image locally: +# ----------------------------------------- +# First build the image: +# docker build -f Dockerfile.binaries -t gcr.io/celo-testnet/geth-xgo-builder:$USER . +# +# Running this image depends on a series of environment variables: +# BUILD_TARGETS Comma separated list of platforms to build for, +# passed as an arg to xgo. eg: "linux/386,linux/amd64" +# TAG_NAME Name of the tag associated with the current commit +# BRANCH_NAME Name of the branch +# REPO_NAME Name of the repo +# COMMIT_SHA Full SHA of the commit +# COMMIT_TIMESTAMP Commit timestamp +# TODO: currently this is not accurately passed see +# discussion in PR celo-blockchain# +# CLOUDBUILD True/False +# CI True/False +# These two are used to comply with how geth handles build +# environments internal/build/env.go where we have added a +# branch for Cloudbuild +# You also need to mount: +# $(pwd)/build/bin:/build - where binaries will be written +# $(pwd)/build/archives:/archives - where the archives (final release artfeact) will be written +# $(pwd):/go/src/github.com/ethereum/go-ethereum - the source code +# +# With all of the above in place the container needs to execute two commands: +# (see ./cloudbuild-binaries.yaml for example of the full command) +# 1. Create the binaries: +# go run build/ci.go xgo --alltools -- -targets=$BUILD_TARGETS -v -dest /build +# 2. Create release archives: +# go run build/ci.go xgo-archive -targets=$_BUILD_TARGETS -in /build -out /archives +# +# This will result in build archives stored in ./build/archives +# In the CI flow these are then uploaded to cloud storage as artefacts. + +# Build Geth binaries in the xgo builder container +FROM techknowlogick/xgo:go-1.13.x +# techknowlogic/xgo is a fork of karalabe/xgo updated to ubunut-18, it is more maintained +# by the community and allows us to backport mingw in order to build for windows +# See discussion in PR celo-blockchain# about downsides of this image + +# We need a newer version of mingw, backported to Bionic +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt install -y --no-install-recommends software-properties-common apt-utils +RUN add-apt-repository -y ppa:mati865/mingw-w64 +RUN apt update && apt -y upgrade + +RUN mkdir -p /go/src/github.com/ethereum/go-ethereum +WORKDIR /go/src/github.com/ethereum/go-ethereum diff --git a/build/ci.go b/build/ci.go index 2f385e0590..c28c0db955 100644 --- a/build/ci.go +++ b/build/ci.go @@ -23,17 +23,18 @@ Usage: go run build/ci.go Available commands are: - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - lint -- runs certain pre-selected linters - archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts - importkeys -- imports signing keys from env - debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package - nsis -- creates a Windows NSIS installer - aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive - xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework - xgo [ -alltools ] [ options ] -- cross builds according to options - purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore + install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + lint -- runs certain pre-selected linters + archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts + importkeys -- imports signing keys from env + debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package + nsis -- creates a Windows NSIS installer + aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive + xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework + xgo [ -alltools ] [ options ] -- cross builds according to options + xgo-archive [ -targets linux/amd64,linux/386... ][ -type zip|tar ][ -in dir ][ -out dir ] -- archives build artifacts from cross-compilation + purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). @@ -192,6 +193,8 @@ func main() { doXCodeFramework(os.Args[2:]) case "xgo": doXgo(os.Args[2:]) + case "xgo-archive": + doXgoArchive(os.Args[2:]) case "purge": doPurge(os.Args[2:]) default: @@ -498,8 +501,8 @@ func maybeSkipArchive(env build.Environment) { log.Printf("skipping because this is a cron job") os.Exit(0) } - if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) + if env.Branch != "master" && !strings.HasPrefix(env.Branch, "release/") { + log.Printf("skipping because branch %q is not on the whitelist", env.Branch) os.Exit(0) } } @@ -1112,6 +1115,82 @@ func xgoTool(args []string) *exec.Cmd { return cmd } +// Archive cross compilation artifacts + +func doXgoArchive(cmdline []string) { + var ( + targetsFlag = flag.String("targets", "", `List of targets int the same format as xgo`) + archiveType = flag.String("type", "tar", `Archive type tar|zip`) + inputDir = flag.String("in", "", `Directory with all build binaries`) + outputDir = flag.String("out", "", `Output directory`) + ext string + ) + + flag.CommandLine.Parse(cmdline) + targets := strings.Split(*targetsFlag, ",") + switch *archiveType { + case "zip": + ext = ".zip" + case "tar": + ext = ".tar.gz" + default: + log.Fatal("unknown archive type: ", archiveType) + } + + + var ( + env = build.Env() + ) + maybeSkipArchive(env) + + for _, target := range targets { + // linux/amd64 => geth-linux-amd64 + var ( + targetBinSegment = strings.Replace(target, "/", "-", 1) + basegeth = targetBinSegment + "-" + params.ArchiveVersion(env.Commit) + geth = filepath.Join(*outputDir, "geth-" + basegeth + ext) + alltools = filepath.Join(*outputDir, "geth-alltools-" + basegeth + ext) + ) + + if err := build.WriteArchive(geth, xgoGethArchiveFiles(targetBinSegment, *inputDir)); err != nil { + log.Fatal(err) + } + if err := build.WriteArchive(alltools, xgoAllToolsArchiveFiles(targetBinSegment, *inputDir)); err != nil { + log.Fatal(err) + } + } +} + +func xgoGethArchiveFiles(target string, dir string) []string { + return []string{ + "COPYING", + executableXgoPath("geth", target, dir), + } +} + +func xgoAllToolsArchiveFiles(target string, dir string) []string { + return []string{ + "COPYING", + executableXgoPath("abigen", target, dir), + executableXgoPath("bootnode", target, dir), + executableXgoPath("evm", target, dir), + executableXgoPath("geth", target, dir), + executableXgoPath("rlpdump", target, dir), + executableXgoPath("wnode", target, dir), + executableXgoPath("clef", target, dir), + executableXgoPath("blspopchecker", target, dir), + } +} + +func executableXgoPath(exec, target, dir string) string { + filename := exec + "-" + target + if strings.HasPrefix(target, "windows") { + filename = filename + ".exe" + } + + return filepath.Join(dir, filename) +} + // Binary distribution cleanups func doPurge(cmdline []string) { diff --git a/cloudbuild-binaries.yaml b/cloudbuild-binaries.yaml new file mode 100644 index 0000000000..766caca409 --- /dev/null +++ b/cloudbuild-binaries.yaml @@ -0,0 +1,62 @@ +# See ./Dockerfile.binaries for more information w.r.t this CI flow +steps: +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'sh' + args: + - '-c' + - 'docker pull us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest || exit 0' +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'sh' + args: + - '-c' + - 'docker build . + --cache-from us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest + -t us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA + -t us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest + -f Dockerfile.binaries' +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'sh' + args: + - '-c' + - 'docker run --rm + -v $(pwd)/build/bin:/build + -v $(pwd)/build/archives:/archives + -v $(pwd):/go/src/github.com/ethereum/go-ethereum + --entrypoint /bin/sh + --env BUILD_TARGETS=$_BUILD_TARGETS + --env TAG_NAME=$TAG_NAME + --env BRANCH_NAME=$BRANCH_NAME + --env REPO_NAME=$REPO_NAME + --env COMMIT_SHA=$COMMIT_SHA + --env COMMIT_TIMESTAMP=$(date +%s) + --env CLOUDBUILD=True + --env CI=True + us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA + -c "go run build/ci.go xgo --alltools -- -targets=$_BUILD_TARGETS -v -dest /build"' +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'sh' + args: + - '-c' + - 'docker run --rm + -v $(pwd)/build/bin:/build + -v $(pwd)/build/archives:/archives + -v $(pwd):/go/src/github.com/ethereum/go-ethereum + --entrypoint /bin/sh + --env BUILD_TARGETS=$_BUILD_TARGETS + --env TAG_NAME=$TAG_NAME + --env BRANCH_NAME=$BRANCH_NAME + --env REPO_NAME=$REPO_NAME + --env COMMIT_SHA=$COMMIT_SHA + --env COMMIT_TIMESTAMP=$(date +%s) + --env CLOUDBUILD=True + --env CI=True + us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA + -c "go run build/ci.go xgo-archive -targets=$_BUILD_TARGETS -in /build -out /archives"' +artifacts: + objects: + location: 'gs://$_BUCKET/$BRANCH_NAME/' + paths: ['./build/archives/*'] +images: +- 'us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA' +- 'us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest' +timeout: 2700s diff --git a/internal/build/env.go b/internal/build/env.go index fcf7dc055a..5ddbcbc850 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -91,6 +91,24 @@ func Env() Environment { IsCronJob: os.Getenv("APPVEYOR_SCHEDULED_BUILD") == "True", IsMusl: os.Getenv("MUSL") == "true", } + case os.Getenv("CI") == "True" && os.Getenv("CLOUDBUILD") == "True": + commit := os.Getenv("COMMIT_SHA") + date, err := strconv.ParseInt(strings.TrimSpace(os.Getenv("COMMIT_TIMESTAMP")), 10, 64) + if err != nil { + panic(fmt.Sprintf("failed to parse git commit date: %v", err)) + } + return Environment{ + Name: "cloudbuild", + Repo: os.Getenv("REPO_NAME"), + Commit: commit, + Date: time.Unix(date, 0).Format("20060102"), + Branch: os.Getenv("BRANCH_NAME"), + Tag: os.Getenv("TAG_NAME"), + Buildnum: os.Getenv("BUILD_ID"), + IsPullRequest: os.Getenv("_PR_NUMBER") != "", + IsCronJob: false, + IsMusl: os.Getenv("MUSL") == "true", + } default: return LocalEnv() }