[v11] APT/YUM publishing fixes

.drone.yml

@@ -7286,16 +7286,6 @@ steps:
   - git init && git remote add origin ${DRONE_REMOTE_URL}
   - git fetch origin --tags
   - git checkout -qf "${DRONE_TAG}"
-  depends_on:
-  - Verify build is tagged
-- name: Check if tag is prerelease
-  image: golang:1.18-alpine
-  commands:
-  - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
-  - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
-    a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
-  depends_on:
-  - Check out code
 - name: Assume Download AWS Role
   image: amazon/aws-cli
   commands:
@@ -7323,12 +7313,11 @@ steps:
   depends_on:
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Download artifacts for "${DRONE_TAG}"
   image: amazon/aws-cli
   commands:
   - mkdir -pv "$ARTIFACT_PATH"
-  - rm -rf "$ARTIFACT_PATH/*"
+  - rm -rf "$ARTIFACT_PATH"/*
   - aws s3 sync --no-progress --delete --exclude "*" --include "*.deb*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/
     "$ARTIFACT_PATH"
   environment:
@@ -7339,9 +7328,9 @@ steps:
   - name: awsconfig
     path: /root/.aws
   depends_on:
+  - Assume Download AWS Role
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Assume Upload AWS Role
   image: amazon/aws-cli
   commands:
@@ -7367,9 +7356,19 @@ steps:
   - name: awsconfig
     path: /root/.aws
   depends_on:
+  - Download artifacts for "${DRONE_TAG}"
+  - Verify build is tagged
+  - Check out code
+- name: Check if tag is prerelease
+  image: golang:1.18-alpine
+  commands:
+  - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
+  - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
+    a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
+  depends_on:
+  - Assume Upload AWS Role
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Publish debs to APT repos for "${DRONE_TAG}"
   image: golang:1.18.4-bullseye
   commands:
@@ -7403,10 +7402,9 @@ steps:
   - name: awsconfig
     path: /root/.aws
   depends_on:
-  - Download artifacts for "${DRONE_TAG}"
+  - Check if tag is prerelease
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 volumes:
 - name: apt-persistence
   claim:
@@ -7483,16 +7481,6 @@ steps:
   - git init && git remote add origin ${DRONE_REMOTE_URL}
   - git fetch origin --tags
   - git checkout -qf "${DRONE_TAG}"
-  depends_on:
-  - Verify build is tagged
-- name: Check if tag is prerelease
-  image: golang:1.18-alpine
-  commands:
-  - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
-  - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
-    a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
-  depends_on:
-  - Check out code
 - name: Assume Download AWS Role
   image: amazon/aws-cli
   commands:
@@ -7520,12 +7508,11 @@ steps:
   depends_on:
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Download artifacts for "${DRONE_TAG}"
   image: amazon/aws-cli
   commands:
   - mkdir -pv "$ARTIFACT_PATH"
-  - rm -rf "$ARTIFACT_PATH/*"
+  - rm -rf "$ARTIFACT_PATH"/*
   - aws s3 sync --no-progress --delete --exclude "*" --include "*.rpm*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/
     "$ARTIFACT_PATH"
   environment:
@@ -7536,9 +7523,9 @@ steps:
   - name: awsconfig
     path: /root/.aws
   depends_on:
+  - Assume Download AWS Role
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Assume Upload AWS Role
   image: amazon/aws-cli
   commands:
@@ -7557,16 +7544,26 @@ steps:
     AWS_ACCESS_KEY_ID:
       from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID
     AWS_ROLE:
-      from_secret: YUM_REPO_NEW_ROLE
+      from_secret: YUM_REPO_NEW_AWS_ROLE
     AWS_SECRET_ACCESS_KEY:
       from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY
   volumes:
   - name: awsconfig
     path: /root/.aws
   depends_on:
+  - Download artifacts for "${DRONE_TAG}"
+  - Verify build is tagged
+  - Check out code
+- name: Check if tag is prerelease
+  image: golang:1.18-alpine
+  commands:
+  - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"
+  - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is
+    a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)
+  depends_on:
+  - Assume Upload AWS Role
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 - name: Publish rpms to YUM repos for "${DRONE_TAG}"
   image: golang:1.18.4-bullseye
   commands:
@@ -7601,10 +7598,9 @@ steps:
   - name: awsconfig
     path: /root/.aws
   depends_on:
-  - Download artifacts for "${DRONE_TAG}"
+  - Check if tag is prerelease
   - Verify build is tagged
   - Check out code
-  - Check if tag is prerelease
 volumes:
 - name: yum-persistence
   claim:
@@ -8729,6 +8725,6 @@ steps:
     WORKSPACE_DIR: /tmp/build-darwin-amd64-connect
 ---
 kind: signature
-hmac: b1c9a7bbb841e00e8cf6a36bb2b6f91a72cce77f9b23c47c6d72cad432087711
+hmac: 9cea74329da2b26a902270d647876f08302522d666567528d821b2f28a2877a4
 
 ...

dronegen/common.go

@@ -251,19 +251,6 @@ func waitForDockerStep() step {
 	}
 }
 
-func verifyValidPromoteRunSteps(checkoutPath, commit string, isParallelismEnabled bool) []step {
-	tagStep := verifyTaggedStep()
-	cloneStep := cloneRepoStep(checkoutPath, commit)
-	verifyStep := verifyNotPrereleaseStep(checkoutPath)
-
-	if isParallelismEnabled {
-		cloneStep.DependsOn = []string{tagStep.Name}
-		verifyStep.DependsOn = []string{cloneStep.Name}
-	}
-
-	return []step{tagStep, cloneStep, verifyStep}
-}
-
 func verifyTaggedStep() step {
 	return step{
 		Name:  "Verify build is tagged",

dronegen/os_repos.go

@@ -189,7 +189,10 @@ func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipel
 	p.Trigger = triggerPromote
 	p.Trigger.Repo.Include = []string{"gravitational/teleport"}
 
-	setupSteps := verifyValidPromoteRunSteps(checkoutPath, commitName, true)
+	setupSteps := []step{
+		verifyTaggedStep(),
+		cloneRepoStep(checkoutPath, commitName),
+	}
 
 	setupStepNames := make([]string, 0, len(setupSteps))
 	for _, setupStep := range setupSteps {
@@ -352,12 +355,6 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str
 	}
 	toolSetupCommands = append(toolSetupCommands, optpb.setupCommands...)
 
-	downloadStepName := fmt.Sprintf("Download artifacts for %q", version)
-	buildStepDependencies := []string{}
-	if enableParallelism {
-		buildStepDependencies = append(buildStepDependencies, downloadStepName)
-	}
-
 	assumeDownloadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
 		awsRoleSettings: awsRoleSettings{
 			awsAccessKeyID:     value{fromSecret: "AWS_ACCESS_KEY_ID"},
@@ -368,86 +365,99 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str
 		name:         "Assume Download AWS Role",
 	})
 
+	downloadStep := step{
+		Name:  fmt.Sprintf("Download artifacts for %q", version),
+		Image: "amazon/aws-cli",
+		Environment: map[string]value{
+			"AWS_S3_BUCKET": {
+				fromSecret: "AWS_S3_BUCKET",
+			},
+			"ARTIFACT_PATH": {
+				raw: optpb.artifactPath,
+			},
+		},
+		Volumes: []volumeRef{volumeRefAwsConfig},
+		Commands: []string{
+			"mkdir -pv \"$ARTIFACT_PATH\"",
+			// Clear out old versions from previous steps
+			"rm -rf \"$ARTIFACT_PATH\"/*",
+			strings.Join(
+				[]string{
+					"aws s3 sync",
+					"--no-progress",
+					"--delete",
+					"--exclude \"*\"",
+					fmt.Sprintf("--include \"*.%s*\"", optpb.packageType),
+					fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder),
+					"\"$ARTIFACT_PATH\"",
+				},
+				" ",
+			),
+		},
+	}
+
 	assumeUploadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
 		awsRoleSettings: optpb.bucketSecrets.awsRoleSettings,
 		configVolume:    volumeRefAwsConfig,
 		name:            "Assume Upload AWS Role",
 	})
 
-	return []step{
-		assumeDownloadRoleStep,
-		{
-			Name:  downloadStepName,
-			Image: "amazon/aws-cli",
-			Environment: map[string]value{
-				"AWS_S3_BUCKET": {
-					fromSecret: "AWS_S3_BUCKET",
-				},
-				"ARTIFACT_PATH": {
-					raw: optpb.artifactPath,
-				},
-			},
-			Volumes: []volumeRef{volumeRefAwsConfig},
-			Commands: []string{
-				"mkdir -pv \"$ARTIFACT_PATH\"",
-				// Clear out old versions from previous steps
-				"rm -rf \"$ARTIFACT_PATH/*\"",
+	verifyNotPrereleaseStep := verifyNotPrereleaseStep(codePath)
+
+	buildAndUploadStep := step{
+		Name:        fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version),
+		Image:       "golang:1.18.4-bullseye",
+		Environment: optpb.environmentVars,
+		Commands: append(
+			toolSetupCommands,
+			[]string{
+				"mkdir -pv -m0700 \"$GNUPGHOME\"",
+				"echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME",
+				"chown -R root:root \"$GNUPGHOME\"",
+				fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")),
+				fmt.Sprintf("export VERSION=%q", version),
+				"export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured
 				strings.Join(
-					[]string{
-						"aws s3 sync",
-						"--no-progress",
-						"--delete",
-						"--exclude \"*\"",
-						fmt.Sprintf("--include \"*.%s*\"", optpb.packageType),
-						fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder),
-						"\"$ARTIFACT_PATH\"",
-					},
+					append(
+						[]string{
+							// This just makes the (long) command a little more readable
+							"go run ./cmd/build-os-package-repos",
+							optpb.packageManagerName,
+							"-bucket \"$REPO_S3_BUCKET\"",
+							"-local-bucket-path \"$BUCKET_CACHE_PATH\"",
+							"-artifact-version \"$VERSION\"",
+							"-release-channel \"$RELEASE_CHANNEL\"",
+							"-artifact-path \"$ARTIFACT_PATH\"",
+							"-log-level 4", // Set this to 5 for debug logging
+						},
+						optpb.extraArgs...,
+					),
 					" ",
 				),
+			}...,
+		),
+		Volumes: []volumeRef{
+			{
+				Name: optpb.volumeName,
+				Path: optpb.pvcMountPoint,
 			},
+			volumeRefTmpfs,
+			volumeRefAwsConfig,
 		},
+	}
+
+	if enableParallelism {
+		downloadStep.DependsOn = []string{assumeDownloadRoleStep.Name}
+		assumeUploadRoleStep.DependsOn = []string{downloadStep.Name}
+		verifyNotPrereleaseStep.DependsOn = []string{assumeUploadRoleStep.Name}
+		buildAndUploadStep.DependsOn = []string{verifyNotPrereleaseStep.Name}
+	}
+
+	return []step{
+		assumeDownloadRoleStep,
+		downloadStep,
 		assumeUploadRoleStep,
-		{
-			Name:        fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version),
-			Image:       "golang:1.18.4-bullseye",
-			Environment: optpb.environmentVars,
-			Commands: append(
-				toolSetupCommands,
-				[]string{
-					"mkdir -pv -m0700 \"$GNUPGHOME\"",
-					"echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME",
-					"chown -R root:root \"$GNUPGHOME\"",
-					fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")),
-					fmt.Sprintf("export VERSION=%q", version),
-					"export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured
-					strings.Join(
-						append(
-							[]string{
-								// This just makes the (long) command a little more readable
-								"go run ./cmd/build-os-package-repos",
-								optpb.packageManagerName,
-								"-bucket \"$REPO_S3_BUCKET\"",
-								"-local-bucket-path \"$BUCKET_CACHE_PATH\"",
-								"-artifact-version \"$VERSION\"",
-								"-release-channel \"$RELEASE_CHANNEL\"",
-								"-artifact-path \"$ARTIFACT_PATH\"",
-								"-log-level 4", // Set this to 5 for debug logging
-							},
-							optpb.extraArgs...,
-						),
-						" ",
-					),
-				}...,
-			),
-			Volumes: []volumeRef{
-				{
-					Name: optpb.volumeName,
-					Path: optpb.pvcMountPoint,
-				},
-				volumeRefTmpfs,
-				volumeRefAwsConfig,
-			},
-			DependsOn: buildStepDependencies,
-		},
+		verifyNotPrereleaseStep,
+		buildAndUploadStep,
 	}
 }

dronegen/yum.go

@@ -36,7 +36,7 @@ func getYumPipelineBuilder() *OsPackageToolPipelineBuilder {
 			"YUM_REPO_NEW_AWS_S3_BUCKET",
 			"YUM_REPO_NEW_AWS_ACCESS_KEY_ID",
 			"YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY",
-			"YUM_REPO_NEW_ROLE",
+			"YUM_REPO_NEW_AWS_ROLE",
 		),
 	)