Added multiarch build support for teleport-operator

Makefile

@@ -16,7 +16,6 @@ VERSION=11.0.0-dev
 DOCKER_IMAGE_QUAY ?= quay.io/gravitational/teleport
 DOCKER_IMAGE_ECR ?= public.ecr.aws/gravitational/teleport
 DOCKER_IMAGE_STAGING ?= 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport
-DOCKER_IMAGE_OPERATOR_STAGING ?= 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-operator
 
 
 GOPATH ?= $(shell go env GOPATH)
@@ -1055,23 +1054,6 @@ publish-ci: image-ci
 	fi
 	if [ -f e/Makefile ]; then $(MAKE) -C e publish-ci; fi
 
-# Docker image build for Teleport Operator
-.PHONY: image-operator-ci
-image-operator-ci:
-	make -C operator docker-build IMG="$(DOCKER_IMAGE_OPERATOR_STAGING):$(VERSION)"
-
-# DOCKER_CLI_EXPERIMENTAL=enabled is set to allow inspecting the manifest for present images.
-# https://docs.docker.com/engine/reference/commandline/cli/#experimental-features
-# The internal staging images use amazon ECR's immutable repository settings. This makes overwrites impossible currently.
-# This can cause issues when drone tagging pipelines must be re-run due to failures.
-# Currently the work around for this is to not attempt to push to the image when it already exists.
-.PHONY: publish-operator-ci
-publish-operator-ci: image-operator-ci
-	@if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$(DOCKER_IMAGE_OPERATOR_STAGING):$(VERSION)" >/dev/null 2>&1; then \
-		echo "$(DOCKER_IMAGE_OPERATOR_STAGING):$(VERSION) already exists. ";                                                         \
-	else                                                                                                                             \
-		docker push "$(DOCKER_IMAGE_OPERATOR_STAGING):$(VERSION)";                                                                   \
-	fi
 
 .PHONY: print-version
 print-version:

dronegen/common.go

@@ -33,6 +33,9 @@ const (
 	// ProductionRegistryQuay is the production image registry that hosts images on quay.io. Will be deprecated in the future.
 	// See RFD 73 - https://github.com/gravitational/teleport/blob/c18c09f5d562dd46a509154eab4295ad39decc3c/rfd/0073-public-image-registry.md
 	ProductionRegistryQuay = "quay.io"
+
+	// Go version used by internal tools
+	GoVersion = "1.18"
 )
 
 var (
@@ -97,12 +100,20 @@ func pushTriggerForBranch(branches ...string) trigger {
 	return t
 }
 
+func cronTrigger(cronJobNames []string) trigger {
+	return trigger{
+		Cron: triggerRef{Include: cronJobNames},
+		Repo: triggerRef{Include: []string{"gravitational/teleport"}},
+	}
+}
+
 func cloneRepoCommands(cloneDirectory, commit string) []string {
 	return []string{
 		fmt.Sprintf("mkdir -pv %q", cloneDirectory),
 		fmt.Sprintf("cd %q", cloneDirectory),
-		`git init && git remote add origin ${DRONE_REMOTE_URL}`,
-		`git fetch origin --tags`,
+		"git init",
+		"git remote add origin ${DRONE_REMOTE_URL}",
+		"git fetch origin --tags",
 		fmt.Sprintf("git checkout -qf %q", commit),
 	}
 }
@@ -207,6 +218,19 @@ func dockerService(v ...volumeRef) service {
 	}
 }
 
+// Starts a container registry service at `drone-docker-registry:5000`
+// This can be pushed/pulled to via `docker push/pull drone-docker-registry:5000/image:tag`
+func dockerRegistryService() service {
+	// The name of this service must match k8s.io/apimachinery/pkg/util/validation `IsDNS1123Subdomain`
+	// so that it is resolvable
+	// See https://github.com/drone-runners/drone-runner-kube/blob/master/engine/compiler/compiler.go#L398
+	// for details
+	return service{
+		Name:  "drone-docker-registry",
+		Image: "registry:2",
+	}
+}
+
 // dockerVolumes returns a slice of volumes
 // It includes the Docker socket volume by default, plus any extra volumes passed in
 func dockerVolumes(v ...volume) []volume {
@@ -255,17 +279,23 @@ func waitForDockerStep() step {
 	}
 }
 
-func verifyValidPromoteRunSteps(checkoutPath, commit string, isParallelismEnabled bool) []step {
-	tagStep := verifyTaggedStep()
-	cloneStep := cloneRepoStep(checkoutPath, commit)
-	verifyStep := verifyNotPrereleaseStep(checkoutPath)
-
-	if isParallelismEnabled {
-		cloneStep.DependsOn = []string{tagStep.Name}
-		verifyStep.DependsOn = []string{cloneStep.Name}
+// waitForDockerStep returns a step which checks that the Docker registry is ready
+func waitForDockerRegistryStep() step {
+	return step{
+		Name:  "Wait for docker registry",
+		Image: "alpine",
+		Commands: []string{
+			"apk add curl",
+			`timeout 30s /bin/sh -c 'while [ "$(curl -s -o /dev/null -w %{http_code} http://drone-docker-registry:5000/)" != "200" ]; do sleep 1; done'`,
+		},
 	}
+}
 
-	return []step{tagStep, cloneStep, verifyStep}
+func verifyValidPromoteRunSteps() []step {
+	tagStep := verifyTaggedStep()
+	verifyStep := verifyNotPrereleaseStep()
+
+	return []step{tagStep, verifyStep}
 }
 
 func verifyTaggedStep() step {
@@ -287,13 +317,15 @@ func cloneRepoStep(clonePath, commit string) step {
 	}
 }
 
-func verifyNotPrereleaseStep(checkoutPath string) step {
+func verifyNotPrereleaseStep() step {
+	clonePath := "/tmp/repo"
 	return step{
 		Name:  "Check if tag is prerelease",
-		Image: "golang:1.18-alpine",
-		Commands: []string{
-			fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")),
+		Image: fmt.Sprintf("golang:%s-alpine", GoVersion),
+		Commands: append(
+			cloneRepoCommands(clonePath, "${DRONE_TAG}"),
+			fmt.Sprintf("cd %q", path.Join(clonePath, "build.assets", "tooling")),
 			"go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)",
-		},
+		),
 	}
 }

dronegen/container_image_products.go

@@ -0,0 +1,232 @@
+// Copyright 2021 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"path"
+	"regexp"
+)
+
+// Describes a Gravitational "product", where a "product" is a piece of software
+// that we provide to our customers via container repositories.
+type Product struct {
+	Name                 string
+	DockerfilePath       string
+	WorkingDirectory     string                                  // Working directory to use for "docker build".
+	DockerfileTarget     string                                  // Optional. Defines a dockerfile target to stop at on build.
+	SupportedArchs       []string                                // ISAs that the builder should produce
+	SetupSteps           []step                                  // Product-specific steps that must be ran before building an image.
+	DockerfileArgBuilder func(arch string) []string              // Generator that returns "docker build --arg" strings
+	ImageBuilder         func(repo string, tag *ImageTag) *Image // Generator that returns an Image struct that defines what "docker build" should produce
+	GetRequiredStepNames func(arch string) []string              // Generator that returns the name of the steps that "docker build" should wait for
+}
+
+func NewTeleportOperatorProduct(cloneDirectory string) *Product {
+	name := "teleport-operator"
+	return &Product{
+		Name:             name,
+		DockerfilePath:   path.Join(cloneDirectory, "operator", "Dockerfile"),
+		WorkingDirectory: cloneDirectory,
+		SupportedArchs:   []string{"amd64", "arm", "arm64"},
+		ImageBuilder: func(repo string, tag *ImageTag) *Image {
+			return &Image{
+				Repo: repo,
+				Name: name,
+				Tag:  tag,
+			}
+		},
+		DockerfileArgBuilder: func(arch string) []string {
+			gccPackage := ""
+			compilerName := ""
+			switch arch {
+			case "x86_64", "amd64":
+				gccPackage = "gcc-x86-64-linux-gnu"
+				compilerName = "x86_64-linux-gnu-gcc"
+			case "i686", "i386":
+				gccPackage = "gcc-multilib-i686-linux-gnu"
+				compilerName = "i686-linux-gnu-gcc"
+			case "arm64", "aarch64":
+				gccPackage = "gcc-aarch64-linux-gnu"
+				compilerName = "aarch64-linux-gnu-gcc"
+			// We may want to add additional arm ISAs in the future to support devices without hardware FPUs
+			case "armhf":
+			case "arm":
+				gccPackage = "gcc-arm-linux-gnueabihf"
+				compilerName = "arm-linux-gnueabihf-gcc"
+			}
+
+			return []string{
+				fmt.Sprintf("COMPILER_PACKAGE=%s", gccPackage),
+				fmt.Sprintf("COMPILER_NAME=%s", compilerName),
+			}
+		},
+	}
+}
+
+func (p *Product) getBaseImage(arch string, version *ReleaseVersion) *Image {
+	return &Image{
+		Name: p.Name,
+		Tag: &ImageTag{
+			ShellBaseValue:   version.ShellVersion,
+			DisplayBaseValue: version.MajorVersion,
+			Arch:             arch,
+		},
+	}
+}
+
+func (p *Product) GetLocalRegistryImage(arch string, version *ReleaseVersion) *Image {
+	image := p.getBaseImage(arch, version)
+	image.Repo = localRegistry
+
+	return image
+}
+
+func (p *Product) GetStagingRegistryImage(arch string, version *ReleaseVersion, stagingRepo *ContainerRepo) *Image {
+	image := p.getBaseImage(arch, version)
+	image.Repo = stagingRepo.RegistryDomain
+
+	return image
+}
+
+func (p *Product) buildSteps(version *ReleaseVersion, setupStepNames []string, flags *TriggerFlags) []step {
+	steps := make([]step, 0)
+
+	stagingRepo := GetStagingContainerRepo(flags.UseUniqueStagingTag)
+	productionRepos := GetProductionContainerRepos()
+
+	for _, setupStep := range p.SetupSteps {
+		setupStep.DependsOn = append(setupStep.DependsOn, setupStepNames...)
+		steps = append(steps, setupStep)
+		setupStepNames = append(setupStepNames, setupStep.Name)
+	}
+
+	archBuildStepDetails := make([]*buildStepOutput, 0, len(p.SupportedArchs))
+
+	for _, supportedArch := range p.SupportedArchs {
+		// Include steps for building images from scratch
+		if flags.ShouldBuildNewImages {
+			archBuildStep, archBuildStepDetail := p.createBuildStep(supportedArch, version)
+
+			archBuildStep.DependsOn = append(archBuildStep.DependsOn, setupStepNames...)
+			if p.GetRequiredStepNames != nil {
+				archBuildStep.DependsOn = append(archBuildStep.DependsOn, p.GetRequiredStepNames(supportedArch)...)
+			}
+
+			steps = append(steps, archBuildStep)
+			archBuildStepDetails = append(archBuildStepDetails, archBuildStepDetail)
+		} else {
+			// Generate build details that point to staging images
+			archBuildStepDetails = append(archBuildStepDetails, &buildStepOutput{
+				StepName:   "",
+				BuiltImage: p.GetStagingRegistryImage(supportedArch, version, stagingRepo),
+				Version:    version,
+				Product:    p,
+			})
+		}
+	}
+
+	for _, containerRepo := range getReposToPublishTo(productionRepos, stagingRepo, flags) {
+		steps = append(steps, containerRepo.buildSteps(archBuildStepDetails)...)
+	}
+
+	return steps
+}
+
+func getReposToPublishTo(productionRepos []*ContainerRepo, stagingRepo *ContainerRepo, flags *TriggerFlags) []*ContainerRepo {
+	stagingRepos := []*ContainerRepo{stagingRepo}
+
+	if flags.ShouldAffectProductionImages {
+		if !flags.ShouldBuildNewImages {
+			// In this case the images will be pulled from staging and therefor should not be re-published
+			// to staging
+			return productionRepos
+		}
+
+		return append(stagingRepos, productionRepos...)
+	}
+
+	return stagingRepos
+}
+
+func (p *Product) GetBuildStepName(arch string, version *ReleaseVersion) string {
+	telportImageName := p.GetLocalRegistryImage(arch, version)
+	return fmt.Sprintf("Build %s image %q", p.Name, telportImageName.GetDisplayName())
+}
+
+func cleanBuilderName(builderName string) string {
+	var invalidBuildxCharExpression = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
+	return invalidBuildxCharExpression.ReplaceAllString(builderName, "-")
+}
+
+func (p *Product) createBuildStep(arch string, version *ReleaseVersion) (step, *buildStepOutput) {
+	localRegistryImage := p.GetLocalRegistryImage(arch, version)
+	builderName := cleanBuilderName(fmt.Sprintf("%s-builder", localRegistryImage.GetDisplayName()))
+
+	buildxConfigFileDir := path.Join("/tmp", builderName)
+	buildxConfigFilePath := path.Join(buildxConfigFileDir, "buildkitd.toml")
+
+	buildxCreateCommand := "docker buildx create"
+	buildxCreateCommand += fmt.Sprintf(" --driver %q", "docker-container")
+	// This is set so that buildx can reach the local registry
+	buildxCreateCommand += fmt.Sprintf(" --driver-opt %q", "network=host")
+	buildxCreateCommand += fmt.Sprintf(" --name %q", builderName)
+	buildxCreateCommand += fmt.Sprintf(" --config %q", buildxConfigFilePath)
+
+	buildCommand := "docker buildx build"
+	buildCommand += " --push"
+	buildCommand += fmt.Sprintf(" --builder %q", builderName)
+	if p.DockerfileTarget != "" {
+		buildCommand += fmt.Sprintf(" --target %q", p.DockerfileTarget)
+	}
+	buildCommand += fmt.Sprintf(" --platform %q", "linux/"+arch)
+	buildCommand += fmt.Sprintf(" --tag %q", localRegistryImage.GetShellName())
+	buildCommand += fmt.Sprintf(" --file %q", p.DockerfilePath)
+	if p.DockerfileArgBuilder != nil {
+		for _, buildArg := range p.DockerfileArgBuilder(arch) {
+			buildCommand += fmt.Sprintf(" --build-arg %q", buildArg)
+		}
+	}
+	buildCommand += " " + p.WorkingDirectory
+
+	step := step{
+		Name:    p.GetBuildStepName(arch, version),
+		Image:   "docker",
+		Volumes: dockerVolumeRefs(),
+		Environment: map[string]value{
+			"DOCKER_BUILDKIT": {
+				raw: "1",
+			},
+		},
+		Commands: []string{
+			"docker run --privileged --rm tonistiigi/binfmt --install all",
+			fmt.Sprintf("mkdir -pv %q && cd %q", p.WorkingDirectory, p.WorkingDirectory),
+			fmt.Sprintf("mkdir -pv %q", buildxConfigFileDir),
+			fmt.Sprintf("echo '[registry.%q]' > %q", localRegistry, buildxConfigFilePath),
+			fmt.Sprintf("echo '  http = true' >> %q", buildxConfigFilePath),
+			buildxCreateCommand,
+			buildCommand,
+			fmt.Sprintf("docker buildx rm %q", builderName),
+			fmt.Sprintf("rm -rf %q", buildxConfigFileDir),
+		},
+	}
+
+	return step, &buildStepOutput{
+		StepName:   step.Name,
+		BuiltImage: localRegistryImage,
+		Version:    version,
+		Product:    p,
+	}
+}