Attempts to make Drone play nicer with Docker Hub

Makefile

@@ -10,8 +10,8 @@
 # Naming convention:
 #   Stable releases:   "1.0.0"
 #   Pre-releases:      "1.0.0-alpha.1", "1.0.0-beta.2", "1.0.0-rc.3"
-#   Master/dev branch: "1.0.0-dev"
-VERSION=13.0.0-dev
+#   Master/dev branch: "1.0.0-de"
+VERSION=13.0.0-dev.drone.tcsc.014
 
 DOCKER_IMAGE ?= teleport
 
@@ -952,12 +952,13 @@ $(VERSRC): Makefile
 # 		- build binaries with 'make release'
 # 		- run `make tag` and use its output to 'git tag' and 'git push --tags'
 .PHONY: update-tag
+update-tag: TAG_REMOTE ?= origin
 update-tag:
 	@test $(VERSION)
 	git tag $(GITTAG)
 	git tag api/$(GITTAG)
 	(cd e && git tag $(GITTAG) && git push origin $(GITTAG))
-	git push origin $(GITTAG) && git push origin api/$(GITTAG)
+	git push $(TAG_REMOTE) $(GITTAG) && git push $(TAG_REMOTE) api/$(GITTAG)
 
 .PHONY: test-package
 test-package: remove-temp-files

dronegen/aws.go

@@ -93,6 +93,7 @@ func kubernetesAssumeAwsRoleStep(s kubernetesRoleSettings) step {
 	return step{
 		Name:  s.name,
 		Image: "amazon/aws-cli",
+		Pull:  "if-not-exists",
 		Environment: map[string]value{
 			"AWS_ACCESS_KEY_ID":     s.awsAccessKeyID,
 			"AWS_SECRET_ACCESS_KEY": s.awsSecretAccessKey,
@@ -125,6 +126,7 @@ func kubernetesUploadToS3Step(s kubernetesS3Settings) step {
 	return step{
 		Name:  "Upload to S3",
 		Image: "amazon/aws-cli",
+		Pull:  "if-not-exists",
 		Environment: map[string]value{
 			"AWS_S3_BUCKET": {fromSecret: "AWS_S3_BUCKET"},
 			"AWS_REGION":    {raw: s.region},

dronegen/buildbox.go

@@ -69,7 +69,8 @@ func buildboxPipelineStep(buildboxName string, fips bool) step {
 	return step{
 		Name:    "Build and push " + buildboxName,
 		Image:   "docker",
-		Volumes: []volumeRef{volumeRefDocker, volumeRefAwsConfig},
+		Pull:    "if-not-exists",
+		Volumes: dockerVolumeRefs(volumeRefAwsConfig),
 		Commands: []string{
 			`apk add --no-cache make aws-cli`,
 			`chown -R $UID:$GID /go`,
@@ -101,7 +102,7 @@ func buildboxPipeline() pipeline {
 	// only on master for now; add the release branch name when forking a new release series.
 	p.Trigger = pushTriggerForBranch("master", "branch/*")
 	p.Workspace = workspace{Path: "/go/src/github.com/gravitational/teleport"}
-	p.Volumes = []volume{volumeDocker, volumeAwsConfig}
+	p.Volumes = dockerVolumes(volumeAwsConfig)
 	p.Services = []service{
 		dockerService(),
 	}

dronegen/common.go

@@ -86,6 +86,25 @@ var (
 		Name: "awsconfig",
 		Path: "/root/.aws",
 	}
+
+	// volumeDockerConfig is a temporary volume for storing docker
+	// credentials for use with the Docker-in-Docker service we use
+	// to isolate the host machines docker daemon from the one used
+	// during the build. Mount this any tome you use `volumeDocker`
+	//
+	// Drone claims to destroy the the temp volumes after a workflow
+	// has run, so it should be safe to write credentials etc.
+	volumeDockerConfig = volume{
+		Name: "dockerconfig",
+		Temp: &volumeTemp{},
+	}
+
+	// volumeRefDockerConfig is how you reference the docker config
+	// volume in a workflow step
+	volumeRefDockerConfig = volumeRef{
+		Name: "dockerconfig",
+		Path: "/root/.docker",
+	}
 )
 
 var buildboxVersion value
@@ -245,13 +264,13 @@ func dockerRegistryService() service {
 // dockerVolumes returns a slice of volumes
 // It includes the Docker socket volume by default, plus any extra volumes passed in
 func dockerVolumes(v ...volume) []volume {
-	return append(v, volumeDocker)
+	return append(v, volumeDocker, volumeDockerConfig)
 }
 
 // dockerVolumeRefs returns a slice of volumeRefs
 // It includes the Docker socket volumeRef as a default, plus any extra volumeRefs passed in
 func dockerVolumeRefs(v ...volumeRef) []volumeRef {
-	return append(v, volumeRefDocker)
+	return append(v, volumeRefDocker, volumeRefDockerConfig)
 }
 
 // releaseMakefileTarget gets the correct Makefile target for a given arch/fips/centos combo
@@ -283,10 +302,16 @@ func waitForDockerStep() step {
 	return step{
 		Name:  "Wait for docker",
 		Image: "docker",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			`timeout 30s /bin/sh -c 'while [ ! -S /var/run/docker.sock ]; do sleep 1; done'`,
+			`printenv DOCKERHUB_PASSWORD | docker login -u="$DOCKERHUB_USERNAME" --password-stdin`,
+		},
+		Volumes: dockerVolumeRefs(),
+		Environment: map[string]value{
+			"DOCKERHUB_USERNAME": {fromSecret: "DOCKERHUB_USERNAME"},
+			"DOCKERHUB_PASSWORD": {fromSecret: "DOCKERHUB_READONLY_TOKEN"},
 		},
-		Volumes: []volumeRef{volumeRefDocker},
 	}
 }
 
@@ -295,6 +320,7 @@ func waitForDockerRegistryStep() step {
 	return step{
 		Name:  "Wait for docker registry",
 		Image: "alpine",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			"apk add curl",
 			fmt.Sprintf(`timeout 30s /bin/sh -c 'while [ "$(curl -s -o /dev/null -w %%{http_code} http://%s/)" != "200" ]; do sleep 1; done'`, LocalRegistrySocket),
@@ -306,6 +332,7 @@ func verifyTaggedStep() step {
 	return step{
 		Name:  "Verify build is tagged",
 		Image: "alpine:latest",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			"[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)",
 		},
@@ -317,6 +344,7 @@ func cloneRepoStep(clonePath, commit string) step {
 	return step{
 		Name:     "Check out code",
 		Image:    "alpine/git:latest",
+		Pull:     "if-not-exists",
 		Commands: cloneRepoCommands(clonePath, commit),
 	}
 }

dronegen/gha.go

@@ -62,6 +62,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline {
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -70,6 +71,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline {
 		{
 			Name:  "Delegate build to GitHub",
 			Image: fmt.Sprintf("golang:%s-alpine", GoVersion),
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GHA_APP_KEY": {fromSecret: "GITHUB_WORKFLOW_APP_PRIVATE_KEY"},
 			},

dronegen/main.go

@@ -39,6 +39,22 @@ func main() {
 	pipelines = append(pipelines, buildContainerImagePipelines()...)
 	pipelines = append(pipelines, publishReleasePipeline())
 
+	// Inject the Drone-level dockerhub credentials into all non-exec
+	// pipelines. Drone will then use the docker credentials file in
+	// the named secret as its credentials when pulling images from
+	// dockerhub.
+	//
+	// Exec pipelines to not have the `image_pull_secrets` option, as
+	// their steps are invoked directly on the host runner and not
+	// into a per-step container.
+	for pidx := range pipelines {
+		p := &pipelines[pidx]
+		if p.Type == "exec" {
+			continue
+		}
+		p.ImagePullSecrets = append(p.ImagePullSecrets, "DOCKERHUB_CREDENTIALS")
+	}
+
 	if err := writePipelines(".drone.yml", pipelines); err != nil {
 		fmt.Println("failed writing drone pipelines:", err)
 		os.Exit(1)

dronegen/push.go

@@ -126,14 +126,15 @@ func pushPipeline(b buildType) pipeline {
 	}
 	p.Trigger = triggerPush
 	p.Workspace = workspace{Path: "/go"}
-	p.Volumes = []volume{volumeDocker}
+	p.Volumes = []volume{volumeDocker, volumeDockerConfig}
 	p.Services = []service{
 		dockerService(),
 	}
 	p.Steps = []step{
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -143,8 +144,9 @@ func pushPipeline(b buildType) pipeline {
 		{
 			Name:        "Build artifacts",
 			Image:       "docker",
+			Pull:        "if-not-exists",
 			Environment: pushEnvironment,
-			Volumes:     []volumeRef{volumeRefDocker},
+			Volumes:     []volumeRef{volumeRefDocker, volumeRefDockerConfig},
 			Commands:    pushBuildCommands(b),
 		},
 		sendErrorToSlackStep(),

dronegen/relcli.go

@@ -44,11 +44,7 @@ func relcliPipeline(trigger trigger, name string, stepName string, command strin
 	}
 
 	p.Services = []service{dockerService(volumeRefTmpfs)}
-	p.Volumes = []volume{
-		volumeDocker,
-		volumeTmpfs,
-		volumeAwsConfig,
-	}
+	p.Volumes = dockerVolumes(volumeTmpfs, volumeAwsConfig)
 
 	return p
 }
@@ -62,6 +58,7 @@ func pullRelcliStep(awsConfigVolumeRef volumeRef) step {
 		},
 		Volumes: []volumeRef{
 			volumeRefDocker,
+			volumeRefDockerConfig,
 			volumeRefAwsConfig,
 		},
 		Commands: []string{
@@ -85,6 +82,7 @@ func executeRelcliStep(name string, command string) step {
 		},
 		Volumes: []volumeRef{
 			volumeRefDocker,
+			volumeRefDockerConfig,
 			volumeRefTmpfs,
 			volumeRefAwsConfig,
 		},

dronegen/tag.go

@@ -274,14 +274,15 @@ func tagPipeline(b buildType) pipeline {
 	p.Trigger = triggerTag
 	p.DependsOn = []string{tagCleanupPipelineName}
 	p.Workspace = workspace{Path: "/go"}
-	p.Volumes = []volume{volumeAwsConfig, volumeDocker}
+	p.Volumes = dockerVolumes(volumeAwsConfig)
 	p.Services = []service{
 		dockerService(),
 	}
 	p.Steps = []step{
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -291,13 +292,15 @@ func tagPipeline(b buildType) pipeline {
 		{
 			Name:        "Build artifacts",
 			Image:       "docker",
+			Pull:        "if-not-exists",
 			Environment: tagEnvironment,
-			Volumes:     []volumeRef{volumeRefDocker},
+			Volumes:     []volumeRef{volumeRefDocker, volumeRefDockerConfig},
 			Commands:    tagBuildCommands(b),
 		},
 		{
 			Name:     "Copy artifacts",
 			Image:    "docker",
+			Pull:     "if-not-exists",
 			Commands: tagCopyArtifactCommands(b),
 		},
 		kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
@@ -317,6 +320,7 @@ func tagPipeline(b buildType) pipeline {
 		{
 			Name:     "Register artifacts",
 			Image:    "docker",
+			Pull:     "if-not-exists",
 			Commands: tagCreateReleaseAssetCommands(b, "", extraQualifications),
 			Environment: map[string]value{
 				"RELEASES_CERT": {fromSecret: "RELEASES_CERT"},
@@ -465,12 +469,10 @@ func tagPackagePipeline(packageType string, b buildType) pipeline {
 		environment["OSS_TARBALL_PATH"] = value{raw: "/go/artifacts"}
 	}
 
-	packageDockerVolumes := []volume{
-		volumeDocker,
-		volumeAwsConfig,
-	}
+	packageDockerVolumes := dockerVolumes(volumeAwsConfig)
 	packageDockerVolumeRefs := []volumeRef{
 		volumeRefDocker,
+		volumeRefDockerConfig,
 		volumeRefAwsConfig,
 	}
 	packageDockerService := dockerService()

dronegen/types.go

@@ -29,20 +29,21 @@ import (
 type pipeline struct {
 	comment string
 
-	Kind        string           `yaml:"kind"`
-	Type        string           `yaml:"type"`
-	Name        string           `yaml:"name"`
-	Environment map[string]value `yaml:"environment,omitempty"`
-	Trigger     trigger          `yaml:"trigger"`
-	Workspace   workspace        `yaml:"workspace,omitempty"`
-	Platform    platform         `yaml:"platform,omitempty"`
-	Node        map[string]value `yaml:"node,omitempty"`
-	Clone       clone            `yaml:"clone,omitempty"`
-	DependsOn   []string         `yaml:"depends_on,omitempty"`
-	Concurrency concurrency      `yaml:"concurrency,omitempty"`
-	Steps       []step           `yaml:"steps"`
-	Services    []service        `yaml:"services,omitempty"`
-	Volumes     []volume         `yaml:"volumes,omitempty"`
+	Kind             string           `yaml:"kind"`
+	Type             string           `yaml:"type"`
+	Name             string           `yaml:"name"`
+	Environment      map[string]value `yaml:"environment,omitempty"`
+	Trigger          trigger          `yaml:"trigger"`
+	Workspace        workspace        `yaml:"workspace,omitempty"`
+	Platform         platform         `yaml:"platform,omitempty"`
+	Node             map[string]value `yaml:"node,omitempty"`
+	Clone            clone            `yaml:"clone,omitempty"`
+	DependsOn        []string         `yaml:"depends_on,omitempty"`
+	Concurrency      concurrency      `yaml:"concurrency,omitempty"`
+	Steps            []step           `yaml:"steps"`
+	Services         []service        `yaml:"services,omitempty"`
+	Volumes          []volume         `yaml:"volumes,omitempty"`
+	ImagePullSecrets []string         `yaml:"image_pull_secrets,omitempty"`
 }
 
 func newKubePipeline(name string) pipeline {
@@ -169,6 +170,7 @@ type volumeRef struct {
 type step struct {
 	Name        string              `yaml:"name"`
 	Image       string              `yaml:"image,omitempty"`
+	Pull        string              `yaml:"pull,omitempty"`
 	Commands    []string            `yaml:"commands,omitempty"`
 	Environment map[string]value    `yaml:"environment,omitempty"`
 	Volumes     []volumeRef         `yaml:"volumes,omitempty"`

examples/chart/teleport-cluster/Chart.yaml

@@ -1,4 +1,4 @@
-.version: &version "13.0.0-dev"
+.version: &version "13.0.0-dev.drone.tcsc.014"
 
 name: teleport-cluster
 apiVersion: v2

examples/chart/teleport-cluster/charts/teleport-operator/Chart.yaml

@@ -1,4 +1,4 @@
-.version: &version "13.0.0-dev"
+.version: &version "13.0.0-dev.drone.tcsc.014"
 
 name: teleport-operator
 apiVersion: v2

examples/chart/teleport-kube-agent/Chart.yaml

@@ -1,4 +1,4 @@
-.version: &version "13.0.0-dev"
+.version: &version "13.0.0-dev.drone.tcsc.014"
 
 name: teleport-kube-agent
 apiVersion: v2