Setup binary release CI pipeline

Dockerfile.binaries

@@ -0,0 +1,56 @@
+# Purpose:
+# -------
+# This docker image is used by the ./cloudbuild-binaries.yaml CI flow to
+# cross-compile geth for different platforms.
+# At the time of writing this is only linux-{386/amd64} but there are two
+# pending update that will add darwin-{386,amd64} and windows-amd64 support.
+#
+# How to test changes to this image locally:
+# -----------------------------------------
+# First build the image:
+# docker build -f Dockerfile.binaries -t gcr.io/celo-testnet/geth-xgo-builder:$USER .
+#
+# Running this image depends on a series of environment variables:
+#   BUILD_TARGETS       Comma separated list of platforms to build for,
+#                       passed as an arg to xgo. eg: "linux/386,linux/amd64"
+#   TAG_NAME            Name of the tag associated with the current commit
+#   BRANCH_NAME         Name of the branch
+#   REPO_NAME           Name of the repo
+#   COMMIT_SHA          Full SHA of the commit
+#   COMMIT_TIMESTAMP    Commit timestamp
+#                       TODO: currently this is not accurately passed see
+#                             discussion in PR celo-blockchain#<number>
+#   CLOUDBUILD          True/False
+#   CI                  True/False
+#                       These two are used to comply with how geth handles build
+#                       environments internal/build/env.go where we have added a
+#                       branch for Cloudbuild
+# You also need to mount:
+#    $(pwd)/build/bin:/build - where binaries will be written
+#    $(pwd)/build/archives:/archives - where the archives (final release artfeact) will be written
+#    $(pwd):/go/src/github.com/ethereum/go-ethereum - the source code
+#
+# With all of the above in place the container needs to execute two commands:
+# (see ./cloudbuild-binaries.yaml for example of the full command)
+# 1. Create the binaries:
+#    go run build/ci.go xgo --alltools -- -targets=$BUILD_TARGETS -v -dest /build
+# 2. Create release archives:
+#    go run build/ci.go xgo-archive -targets=$_BUILD_TARGETS -in /build -out /archives
+#
+# This will result in build archives stored in ./build/archives
+# In the CI flow these are then uploaded to cloud storage as artefacts.
+
+# Build Geth binaries in the xgo builder container
+FROM techknowlogick/xgo:go-1.13.x
+# techknowlogic/xgo is a fork of karalabe/xgo updated to ubunut-18, it is more maintained
+# by the community and allows us to backport mingw in order to build for windows
+# See discussion in PR celo-blockchain#<number> about downsides of this image
+
+# We need a newer version of mingw, backported to Bionic
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update && apt install -y  --no-install-recommends software-properties-common apt-utils
+RUN add-apt-repository -y ppa:mati865/mingw-w64
+RUN apt update && apt -y upgrade
+
+RUN mkdir -p /go/src/github.com/ethereum/go-ethereum
+WORKDIR /go/src/github.com/ethereum/go-ethereum

build/ci.go

@@ -23,17 +23,18 @@ Usage: go run build/ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
-   test       [ -coverage ] [ packages... ]                                                    -- runs the tests
-   lint                                                                                        -- runs certain pre-selected linters
-   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
-   importkeys                                                                                  -- imports signing keys from env
-   debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
-   nsis                                                                                        -- creates a Windows NSIS installer
-   aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
-   xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
-   xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
-   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
+   install     [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
+   test        [ -coverage ] [ packages... ]                                                    -- runs the tests
+   lint                                                                                         -- runs certain pre-selected linters
+   archive     [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
+   importkeys                                                                                   -- imports signing keys from env
+   debsrc      [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
+   nsis                                                                                         -- creates a Windows NSIS installer
+   aar         [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
+   xcode       [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
+   xgo         [ -alltools ] [ options ]                                                        -- cross builds according to options
+   xgo-archive [ -targets linux/amd64,linux/386... ][ -type zip|tar ][ -in dir ][ -out dir ]	-- archives build artifacts from cross-compilation
+   purge       [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -192,6 +193,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "xgo-archive":
+		doXgoArchive(os.Args[2:])
 	case "purge":
 		doPurge(os.Args[2:])
 	default:
@@ -498,8 +501,8 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a cron job")
 		os.Exit(0)
 	}
-	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
-		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
+	if env.Branch != "master" && !strings.HasPrefix(env.Branch, "release/") {
+		log.Printf("skipping because branch %q is not on the whitelist", env.Branch)
 		os.Exit(0)
 	}
 }
@@ -1112,6 +1115,82 @@ func xgoTool(args []string) *exec.Cmd {
 	return cmd
 }
 
+// Archive cross compilation artifacts
+
+func doXgoArchive(cmdline []string) {
+	var (
+		targetsFlag = flag.String("targets", "", `List of targets int the same format as xgo`)
+		archiveType = flag.String("type", "tar", `Archive type tar|zip`)
+		inputDir = flag.String("in", "", `Directory with all build binaries`)
+		outputDir = flag.String("out", "", `Output directory`)
+		ext string
+	)
+
+	flag.CommandLine.Parse(cmdline)
+	targets := strings.Split(*targetsFlag, ",")
+	switch *archiveType {
+	case "zip":
+		ext = ".zip"
+	case "tar":
+		ext = ".tar.gz"
+	default:
+		log.Fatal("unknown archive type: ", archiveType)
+	}
+
+
+	var (
+		env = build.Env()
+	)
+	maybeSkipArchive(env)
+
+	for _, target := range targets {
+		// linux/amd64 => geth-linux-amd64
+		var (
+			targetBinSegment = strings.Replace(target, "/", "-", 1)
+			basegeth = targetBinSegment + "-" + params.ArchiveVersion(env.Commit)
+			geth     = filepath.Join(*outputDir, "geth-" + basegeth + ext)
+			alltools = filepath.Join(*outputDir, "geth-alltools-" + basegeth + ext)
+		)
+
+		if err := build.WriteArchive(geth, xgoGethArchiveFiles(targetBinSegment, *inputDir)); err != nil {
+			log.Fatal(err)
+		}
+		if err := build.WriteArchive(alltools, xgoAllToolsArchiveFiles(targetBinSegment, *inputDir)); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func xgoGethArchiveFiles(target string, dir string) []string {
+	return []string{
+		"COPYING",
+		executableXgoPath("geth", target, dir),
+	}
+}
+
+func xgoAllToolsArchiveFiles(target string, dir string) []string {
+	return []string{
+		"COPYING",
+		executableXgoPath("abigen", target, dir),
+		executableXgoPath("bootnode", target, dir),
+		executableXgoPath("evm", target, dir),
+		executableXgoPath("geth", target, dir),
+		executableXgoPath("rlpdump", target, dir),
+		executableXgoPath("wnode", target, dir),
+		executableXgoPath("clef", target, dir),
+		executableXgoPath("blspopchecker", target, dir),
+	}
+}
+
+func executableXgoPath(exec, target, dir string) string {
+	filename := exec + "-" + target
+	if strings.HasPrefix(target, "windows") {
+		filename = filename + ".exe"
+	}
+
+	return filepath.Join(dir, filename)
+}
+
 // Binary distribution cleanups
 
 func doPurge(cmdline []string) {

cloudbuild-binaries.yaml

@@ -0,0 +1,62 @@
+# See ./Dockerfile.binaries for more information w.r.t this CI flow
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'sh'
+  args:
+    - '-c'
+    - 'docker pull us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest || exit 0'
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'sh'
+  args:
+    - '-c'
+    - 'docker build .
+        --cache-from us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest
+        -t us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA
+        -t us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest
+        -f Dockerfile.binaries'
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'sh'
+  args:
+    - '-c'
+    - 'docker run --rm
+        -v $(pwd)/build/bin:/build
+        -v $(pwd)/build/archives:/archives
+        -v $(pwd):/go/src/github.com/ethereum/go-ethereum
+        --entrypoint /bin/sh
+        --env BUILD_TARGETS=$_BUILD_TARGETS
+        --env TAG_NAME=$TAG_NAME
+        --env BRANCH_NAME=$BRANCH_NAME
+        --env REPO_NAME=$REPO_NAME
+        --env COMMIT_SHA=$COMMIT_SHA
+        --env COMMIT_TIMESTAMP=$(date +%s)
+        --env CLOUDBUILD=True
+        --env CI=True
+        us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA
+        -c "go run build/ci.go xgo --alltools -- -targets=$_BUILD_TARGETS -v -dest /build"'
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'sh'
+  args:
+    - '-c'
+    - 'docker run --rm
+        -v $(pwd)/build/bin:/build
+        -v $(pwd)/build/archives:/archives
+        -v $(pwd):/go/src/github.com/ethereum/go-ethereum
+        --entrypoint /bin/sh
+        --env BUILD_TARGETS=$_BUILD_TARGETS
+        --env TAG_NAME=$TAG_NAME
+        --env BRANCH_NAME=$BRANCH_NAME
+        --env REPO_NAME=$REPO_NAME
+        --env COMMIT_SHA=$COMMIT_SHA
+        --env COMMIT_TIMESTAMP=$(date +%s)
+        --env CLOUDBUILD=True
+        --env CI=True
+        us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA
+        -c "go run build/ci.go xgo-archive -targets=$_BUILD_TARGETS -in /build -out /archives"'
+artifacts:
+  objects:
+    location: 'gs://$_BUCKET/$BRANCH_NAME/'
+    paths: ['./build/archives/*']
+images:
+- 'us.gcr.io/$PROJECT_ID/geth-xgo-builder:$COMMIT_SHA'
+- 'us.gcr.io/$PROJECT_ID/geth-xgo-builder:latest'
+timeout: 2700s

internal/build/env.go

@@ -91,6 +91,24 @@ func Env() Environment {
 			IsCronJob:     os.Getenv("APPVEYOR_SCHEDULED_BUILD") == "True",
 			IsMusl:        os.Getenv("MUSL") == "true",
 		}
+	case os.Getenv("CI") == "True" && os.Getenv("CLOUDBUILD") == "True":
+		commit := os.Getenv("COMMIT_SHA")
+		date, err := strconv.ParseInt(strings.TrimSpace(os.Getenv("COMMIT_TIMESTAMP")), 10, 64)
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse git commit date: %v", err))
+		}
+		return Environment{
+			Name:          "cloudbuild",
+			Repo:          os.Getenv("REPO_NAME"),
+			Commit:        commit,
+			Date:          time.Unix(date, 0).Format("20060102"),
+			Branch:        os.Getenv("BRANCH_NAME"),
+			Tag:           os.Getenv("TAG_NAME"),
+			Buildnum:      os.Getenv("BUILD_ID"),
+			IsPullRequest: os.Getenv("_PR_NUMBER") != "",
+			IsCronJob:     false,
+			IsMusl:        os.Getenv("MUSL") == "true",
+		}
 	default:
 		return LocalEnv()
 	}