From 00682df0cf97d9cda6a2bb9f1b6be2e02869fea1 Mon Sep 17 00:00:00 2001 From: Walt Della Date: Thu, 20 Oct 2022 11:02:26 -0700 Subject: [PATCH 1/4] Serialize apt/yum promote pipelines These were running in parallel, but we want them to run serially. Therefore, we add a dependency between each step and its previous step. --- .drone.yml | 10 ++- dronegen/os_repos.go | 153 ++++++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 78 deletions(-) diff --git a/.drone.yml b/.drone.yml index 558a8330ea269..30c1ed8e5aee2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7354,6 +7354,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: + - Assume Download AWS Role - Verify build is tagged - Check if tag is prerelease - Check out code @@ -7382,6 +7383,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: + - Download artifacts for "${DRONE_TAG}" - Verify build is tagged - Check if tag is prerelease - Check out code @@ -7418,7 +7420,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: - - Download artifacts for "${DRONE_TAG}" + - Assume Upload AWS Role - Verify build is tagged - Check if tag is prerelease - Check out code @@ -7555,6 +7557,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: + - Assume Download AWS Role - Verify build is tagged - Check if tag is prerelease - Check out code @@ -7583,6 +7586,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: + - Download artifacts for "${DRONE_TAG}" - Verify build is tagged - Check if tag is prerelease - Check out code @@ -7620,7 +7624,7 @@ steps: - name: awsconfig path: /root/.aws depends_on: - - Download artifacts for "${DRONE_TAG}" + - Assume Upload AWS Role - Verify build is tagged - Check if tag is prerelease - Check out code @@ -11575,6 +11579,6 @@ volumes: temp: {} --- kind: signature -hmac: 47934b14a6ca045c0beb0357b731373852ce4d490bd3d248db2d45f30025e1e8 +hmac: 14604baa2d8e374b8c2ae4323aadf1e0ff93c54098dbfd3c1196ef5aa6426d84 ... diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go index 3d3c69b71bbf1..9af45899457d3 100644 --- a/dronegen/os_repos.go +++ b/dronegen/os_repos.go @@ -343,12 +343,6 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str } toolSetupCommands = append(toolSetupCommands, optpb.setupCommands...) - downloadStepName := fmt.Sprintf("Download artifacts for %q", version) - buildStepDependencies := []string{} - if enableParallelism { - buildStepDependencies = append(buildStepDependencies, downloadStepName) - } - assumeDownloadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{ awsRoleSettings: awsRoleSettings{ awsAccessKeyID: value{fromSecret: "AWS_ACCESS_KEY_ID"}, @@ -359,86 +353,95 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str name: "Assume Download AWS Role", }) + downloadStep := step{ + Name: fmt.Sprintf("Download artifacts for %q", version), + Image: "amazon/aws-cli", + Environment: map[string]value{ + "AWS_S3_BUCKET": { + fromSecret: "AWS_S3_BUCKET", + }, + "ARTIFACT_PATH": { + raw: optpb.artifactPath, + }, + }, + Volumes: []volumeRef{volumeRefAwsConfig}, + Commands: []string{ + "mkdir -pv \"$ARTIFACT_PATH\"", + // Clear out old versions from previous steps + "rm -rf \"$ARTIFACT_PATH/*\"", + strings.Join( + []string{ + "aws s3 sync", + "--no-progress", + "--delete", + "--exclude \"*\"", + fmt.Sprintf("--include \"*.%s*\"", optpb.packageType), + fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder), + "\"$ARTIFACT_PATH\"", + }, + " ", + ), + }, + } + assumeUploadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{ awsRoleSettings: optpb.bucketSecrets.awsRoleSettings, configVolume: volumeRefAwsConfig, name: "Assume Upload AWS Role", }) - return []step{ - assumeDownloadRoleStep, - { - Name: downloadStepName, - Image: "amazon/aws-cli", - Environment: map[string]value{ - "AWS_S3_BUCKET": { - fromSecret: "AWS_S3_BUCKET", - }, - "ARTIFACT_PATH": { - raw: optpb.artifactPath, - }, - }, - Volumes: []volumeRef{volumeRefAwsConfig}, - Commands: []string{ - "mkdir -pv \"$ARTIFACT_PATH\"", - // Clear out old versions from previous steps - "rm -rf \"$ARTIFACT_PATH/*\"", + buildAndUploadStep := step{ + Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version), + Image: fmt.Sprintf("golang:%s-bullseye", GoVersion), + Environment: optpb.environmentVars, + Commands: append( + toolSetupCommands, + []string{ + "mkdir -pv -m0700 \"$GNUPGHOME\"", + "echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME", + "chown -R root:root \"$GNUPGHOME\"", + fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")), + fmt.Sprintf("export VERSION=%q", version), + "export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured strings.Join( - []string{ - "aws s3 sync", - "--no-progress", - "--delete", - "--exclude \"*\"", - fmt.Sprintf("--include \"*.%s*\"", optpb.packageType), - fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder), - "\"$ARTIFACT_PATH\"", - }, + append( + []string{ + // This just makes the (long) command a little more readable + "go run ./cmd/build-os-package-repos", + optpb.packageManagerName, + "-bucket \"$REPO_S3_BUCKET\"", + "-local-bucket-path \"$BUCKET_CACHE_PATH\"", + "-artifact-version \"$VERSION\"", + "-release-channel \"$RELEASE_CHANNEL\"", + "-artifact-path \"$ARTIFACT_PATH\"", + "-log-level 4", // Set this to 5 for debug logging + }, + optpb.extraArgs..., + ), " ", ), + }..., + ), + Volumes: []volumeRef{ + { + Name: optpb.volumeName, + Path: optpb.pvcMountPoint, }, + volumeRefTmpfs, + volumeRefAwsConfig, }, + } + + if enableParallelism { + downloadStep.DependsOn = []string{assumeDownloadRoleStep.Name} + assumeUploadRoleStep.DependsOn = []string{downloadStep.Name} + buildAndUploadStep.DependsOn = []string{assumeUploadRoleStep.Name} + } + + return []step{ + assumeDownloadRoleStep, + downloadStep, assumeUploadRoleStep, - { - Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version), - Image: fmt.Sprintf("golang:%s-bullseye", GoVersion), - Environment: optpb.environmentVars, - Commands: append( - toolSetupCommands, - []string{ - "mkdir -pv -m0700 \"$GNUPGHOME\"", - "echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME", - "chown -R root:root \"$GNUPGHOME\"", - fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")), - fmt.Sprintf("export VERSION=%q", version), - "export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured - strings.Join( - append( - []string{ - // This just makes the (long) command a little more readable - "go run ./cmd/build-os-package-repos", - optpb.packageManagerName, - "-bucket \"$REPO_S3_BUCKET\"", - "-local-bucket-path \"$BUCKET_CACHE_PATH\"", - "-artifact-version \"$VERSION\"", - "-release-channel \"$RELEASE_CHANNEL\"", - "-artifact-path \"$ARTIFACT_PATH\"", - "-log-level 4", // Set this to 5 for debug logging - }, - optpb.extraArgs..., - ), - " ", - ), - }..., - ), - Volumes: []volumeRef{ - { - Name: optpb.volumeName, - Path: optpb.pvcMountPoint, - }, - volumeRefTmpfs, - volumeRefAwsConfig, - }, - DependsOn: buildStepDependencies, - }, + buildAndUploadStep, } } From 3d601306157b9e7c644e9a6127b547ca90b75367 Mon Sep 17 00:00:00 2001 From: Walt Della Date: Thu, 20 Oct 2022 11:18:51 -0700 Subject: [PATCH 2/4] Allow dev build promotes to proceed in deb/rpm pipelines This helps test a couple more changes from this pipeline when cutting a dev build. Particularly, we saw the download and role assumption steps fail in https://github.com/gravitational/teleport/pull/17334, and this change would have allowed us to catch that error during testing. --- .drone.yml | 74 ++++++++++++++++++++++---------------------- dronegen/os_repos.go | 12 ++++--- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/.drone.yml b/.drone.yml index 30c1ed8e5aee2..e6fed6d75a27f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7289,19 +7289,6 @@ steps: commands: - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' && exit 1)' -- name: Check if tag is prerelease - image: golang:1.18-alpine - commands: - - apk add git - - mkdir -pv "/tmp/repo" - - cd "/tmp/repo" - - git init - - git remote add origin ${DRONE_REMOTE_URL} - - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" - - cd "/tmp/repo/build.assets/tooling" - - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) - name: Check out code image: alpine/git:latest commands: @@ -7337,7 +7324,6 @@ steps: path: /root/.aws depends_on: - Verify build is tagged - - Check if tag is prerelease - Check out code - name: Download artifacts for "${DRONE_TAG}" image: amazon/aws-cli @@ -7356,7 +7342,6 @@ steps: depends_on: - Assume Download AWS Role - Verify build is tagged - - Check if tag is prerelease - Check out code - name: Assume Upload AWS Role image: amazon/aws-cli @@ -7385,7 +7370,23 @@ steps: depends_on: - Download artifacts for "${DRONE_TAG}" - Verify build is tagged - - Check if tag is prerelease + - Check out code +- name: Check if tag is prerelease + image: golang:1.18-alpine + commands: + - apk add git + - mkdir -pv "/tmp/repo" + - cd "/tmp/repo" + - git init + - git remote add origin ${DRONE_REMOTE_URL} + - git fetch origin --tags + - git checkout -qf "${DRONE_TAG}" + - cd "/tmp/repo/build.assets/tooling" + - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is + a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) + depends_on: + - Assume Upload AWS Role + - Verify build is tagged - Check out code - name: Publish debs to APT repos for "${DRONE_TAG}" image: golang:1.18-bullseye @@ -7420,9 +7421,8 @@ steps: - name: awsconfig path: /root/.aws depends_on: - - Assume Upload AWS Role - - Verify build is tagged - Check if tag is prerelease + - Verify build is tagged - Check out code volumes: - name: apt-persistence @@ -7492,19 +7492,6 @@ steps: commands: - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' && exit 1)' -- name: Check if tag is prerelease - image: golang:1.18-alpine - commands: - - apk add git - - mkdir -pv "/tmp/repo" - - cd "/tmp/repo" - - git init - - git remote add origin ${DRONE_REMOTE_URL} - - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" - - cd "/tmp/repo/build.assets/tooling" - - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) - name: Check out code image: alpine/git:latest commands: @@ -7540,7 +7527,6 @@ steps: path: /root/.aws depends_on: - Verify build is tagged - - Check if tag is prerelease - Check out code - name: Download artifacts for "${DRONE_TAG}" image: amazon/aws-cli @@ -7559,7 +7545,6 @@ steps: depends_on: - Assume Download AWS Role - Verify build is tagged - - Check if tag is prerelease - Check out code - name: Assume Upload AWS Role image: amazon/aws-cli @@ -7588,7 +7573,23 @@ steps: depends_on: - Download artifacts for "${DRONE_TAG}" - Verify build is tagged - - Check if tag is prerelease + - Check out code +- name: Check if tag is prerelease + image: golang:1.18-alpine + commands: + - apk add git + - mkdir -pv "/tmp/repo" + - cd "/tmp/repo" + - git init + - git remote add origin ${DRONE_REMOTE_URL} + - git fetch origin --tags + - git checkout -qf "${DRONE_TAG}" + - cd "/tmp/repo/build.assets/tooling" + - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is + a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) + depends_on: + - Assume Upload AWS Role + - Verify build is tagged - Check out code - name: Publish rpms to YUM repos for "${DRONE_TAG}" image: golang:1.18-bullseye @@ -7624,9 +7625,8 @@ steps: - name: awsconfig path: /root/.aws depends_on: - - Assume Upload AWS Role - - Verify build is tagged - Check if tag is prerelease + - Verify build is tagged - Check out code volumes: - name: yum-persistence @@ -11579,6 +11579,6 @@ volumes: temp: {} --- kind: signature -hmac: 14604baa2d8e374b8c2ae4323aadf1e0ff93c54098dbfd3c1196ef5aa6426d84 +hmac: 45ba388eb8c6898edca566511da05dea51daeca458fed5bcb5baa83177795a5b ... diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go index 9af45899457d3..bb130256bcf86 100644 --- a/dronegen/os_repos.go +++ b/dronegen/os_repos.go @@ -195,10 +195,10 @@ func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipel p.Trigger = triggerPromote p.Trigger.Repo.Include = []string{"gravitational/teleport"} - setupSteps := append( - verifyValidPromoteRunSteps(), + setupSteps := []step{ + verifyTaggedStep(), cloneRepoStep(checkoutPath, commitName), - ) + } setupStepNames := make([]string, 0, len(setupSteps)) for _, setupStep := range setupSteps { @@ -390,6 +390,8 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str name: "Assume Upload AWS Role", }) + verifyNotPrereleaseStep := verifyNotPrereleaseStep() + buildAndUploadStep := step{ Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version), Image: fmt.Sprintf("golang:%s-bullseye", GoVersion), @@ -435,13 +437,15 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str if enableParallelism { downloadStep.DependsOn = []string{assumeDownloadRoleStep.Name} assumeUploadRoleStep.DependsOn = []string{downloadStep.Name} - buildAndUploadStep.DependsOn = []string{assumeUploadRoleStep.Name} + verifyNotPrereleaseStep.DependsOn = []string{assumeUploadRoleStep.Name} + buildAndUploadStep.DependsOn = []string{verifyNotPrereleaseStep.Name} } return []step{ assumeDownloadRoleStep, downloadStep, assumeUploadRoleStep, + verifyNotPrereleaseStep, buildAndUploadStep, } } From 53dad692480d286b63721a2e45520174e267cbab Mon Sep 17 00:00:00 2001 From: Walt Della Date: Thu, 20 Oct 2022 11:30:17 -0700 Subject: [PATCH 3/4] Fix globbing bug This bug does not appear to affect anything currently. However it should be fixed in case the rm is important at some point in the future. The bug is: when a wildcard is inside quotes, it is treated as a literal filename. So rm -rf "$ARTIFACT_PATH/*" tries to remove the file named '*' instead of trying to remove everything in artifact path. --- .drone.yml | 6 +++--- dronegen/os_repos.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index e6fed6d75a27f..b82aaaae53477 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7329,7 +7329,7 @@ steps: image: amazon/aws-cli commands: - mkdir -pv "$ARTIFACT_PATH" - - rm -rf "$ARTIFACT_PATH/*" + - rm -rf "$ARTIFACT_PATH"/* - aws s3 sync --no-progress --delete --exclude "*" --include "*.deb*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ "$ARTIFACT_PATH" environment: @@ -7532,7 +7532,7 @@ steps: image: amazon/aws-cli commands: - mkdir -pv "$ARTIFACT_PATH" - - rm -rf "$ARTIFACT_PATH/*" + - rm -rf "$ARTIFACT_PATH"/* - aws s3 sync --no-progress --delete --exclude "*" --include "*.rpm*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ "$ARTIFACT_PATH" environment: @@ -11579,6 +11579,6 @@ volumes: temp: {} --- kind: signature -hmac: 45ba388eb8c6898edca566511da05dea51daeca458fed5bcb5baa83177795a5b +hmac: bee65ecafb73bb1999f8ceca2c75b205dd5e8d0c8701e69d0a28db87615039cf ... diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go index bb130256bcf86..7479bfcb49f01 100644 --- a/dronegen/os_repos.go +++ b/dronegen/os_repos.go @@ -368,7 +368,7 @@ func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version str Commands: []string{ "mkdir -pv \"$ARTIFACT_PATH\"", // Clear out old versions from previous steps - "rm -rf \"$ARTIFACT_PATH/*\"", + "rm -rf \"$ARTIFACT_PATH\"/*", strings.Join( []string{ "aws s3 sync", From aad6822fc7a87eee9eb88ad1b544bf590f459806 Mon Sep 17 00:00:00 2001 From: Walt Della Date: Thu, 13 Oct 2022 13:16:48 -0700 Subject: [PATCH 4/4] Swap YUM_REPO_NEW_ROLE to YUM_REPO_NEW_AWS_ROLE All other roles environment variables end in AWS_ROLE, and consistency is our friend here. --- .drone.yml | 4 ++-- dronegen/yum.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index b82aaaae53477..0b6ebbf251031 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7564,7 +7564,7 @@ steps: AWS_ACCESS_KEY_ID: from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID AWS_ROLE: - from_secret: YUM_REPO_NEW_ROLE + from_secret: YUM_REPO_NEW_AWS_ROLE AWS_SECRET_ACCESS_KEY: from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY volumes: @@ -11579,6 +11579,6 @@ volumes: temp: {} --- kind: signature -hmac: bee65ecafb73bb1999f8ceca2c75b205dd5e8d0c8701e69d0a28db87615039cf +hmac: e70373fddb9ac5bff144dfcacff1c8d60ece46429a4f70c724bad0307bf03740 ... diff --git a/dronegen/yum.go b/dronegen/yum.go index 0a8960d0b1262..2cbbb44a3fa63 100644 --- a/dronegen/yum.go +++ b/dronegen/yum.go @@ -36,7 +36,7 @@ func getYumPipelineBuilder() *OsPackageToolPipelineBuilder { "YUM_REPO_NEW_AWS_S3_BUCKET", "YUM_REPO_NEW_AWS_ACCESS_KEY_ID", "YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY", - "YUM_REPO_NEW_ROLE", + "YUM_REPO_NEW_AWS_ROLE", ), )