Skip to content

Commit

Permalink
cloud build: initial set of shared files
Browse files Browse the repository at this point in the history
The approach taken here extends the existing support for
cross-compiling binaries on the build host and specifying the Go
compiler: Go is installed if needed (as in Prow testing), binaries are
build on the host, then one image is created for each platform, and
finally those are combined into a single multi-architecture image.
  • Loading branch information
pohly committed May 28, 2020
1 parent 9084fec commit bd41690
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 0 deletions.
63 changes: 63 additions & 0 deletions build.make
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,69 @@ build: $(CMDS:%=build-%)
container: $(CMDS:%=container-%)
push: $(CMDS:%=push-%)

# Additional parameters are needed when pushing to a local registry,
# see https://github.com/docker/buildx/issues/94.
# However, that then runs into https://github.com/docker/cli/issues/2396.
#
# What works for local testing is:
# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
DOCKER_BUILDX_CREATE_ARGS ?=

# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
# Docker Buildx is included in Docker 19.03.
#
# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
# the tag for the resulting multiarch image.
push-multiarch-%: check-pull-base-ref build-%
set -ex; \
DOCKER_CLI_EXPERIMENTAL=enabled; \
export DOCKER_CLI_EXPERIMENTAL; \
docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
trap "docker buildx rm multiarchimage-buildertest" EXIT; \
dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
pushMultiArch () { \
tag=$$1; \
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
docker buildx build --push \
--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
--platform=$$os/$$arch \
--file $$(eval echo \$${dockerfile_$$os}) \
--build-arg binary=./bin/$*$$suffix \
--label revision=$(REV) \
.; \
done; \
images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
docker manifest push -p $(IMAGE_NAME):$$tag; \
}; \
if [ $(PULL_BASE_REF) = "master" ]; then \
: "creating or overwriting canary image"; \
pushMultiArch canary; \
elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
: "creating or overwriting canary image for release branch"; \
release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
pushMultiArch $$release_canary_tag; \
elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
: "creating release image"; \
pushMultiArch $(PULL_BASE_REF); \
else \
: "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \
exit 1; \
fi

.PHONY: check-pull-base-ref
check-pull-base-ref:
if ! [ "$(PULL_BASE_REF)" ]; then \
echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
exit 1; \
fi

push-multiarch: $(CMDS:%=push-multiarch-%)

clean:
-rm -rf bin

Expand Down
6 changes: 6 additions & 0 deletions cloudbuild.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#! /bin/bash

# shellcheck disable=SC1091
. release-tools/prow.sh

gcr_cloud_build
44 changes: 44 additions & 0 deletions cloudbuild.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# A configuration file for multi-arch image building with the Google cloud build service.
#
# Repos using this file must:
# - import csi-release-tools
# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml
# - add a .cloudbuild.sh which can be a custom file or a symlink
# to release-tools/cloudbuild.sh
# - accept "binary" as build argument in their Dockerfile(s) (see
# https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6)
# because binaries will get built for different architectures and then
# get copied from the built host into the container image
#
# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
# for more details on image pushing process in Kubernetes.

# This must be specified in seconds. If omitted, defaults to 600s (10 mins).
timeout: 1200s
# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
# or any new substitutions added in the future.
options:
substitution_option: ALLOW_LOOSE
steps:
# The image must contain bash and curl. Ideally it should also contain
# the desired version of Go (currently defined in release-tools/travis.yml),
# but that just speeds up the build and is not required.
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
entrypoint: ./.cloudbuild.sh
env:
- GIT_TAG=${_GIT_TAG}
- PULL_BASE_REF=${_PULL_BASE_REF}
- REGISTRY_NAME=gcr.io/${_STAGING_PROJECT}
- HOME=/root
substitutions:
# _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
# can be used as a substitution.
_GIT_TAG: '12345'
# _PULL_BASE_REF will contain the ref that was pushed to trigger this build -
# a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
_PULL_BASE_REF: 'master'
# The default gcr.io staging project for Kubernetes-CSI
# (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL).
# Might be overridden in the Prow build job for a repo which wants
# images elsewhere.
_STAGING_PROJECT: 'k8s-staging-csi'
14 changes: 14 additions & 0 deletions prow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1189,3 +1189,17 @@ main () {

return "$ret"
}

# This function can be called by a repo's top-level cloudbuild.sh:
# it handles environment set up in the GCR cloud build and then
# invokes "make push-multiarch" to do the actual image building.
gcr_cloud_build () {
# Register gcloud as a Docker credential helper.
# Required for "docker buildx build --push".
gcloud auth configure-docker

# Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value.
REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v')

run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}"
}

0 comments on commit bd41690

Please sign in to comment.