diff --git a/.drone.yml b/.drone.yml index 3c0ae9778448e..f6902d3830484 100644 --- a/.drone.yml +++ b/.drone.yml @@ -516,7 +516,7 @@ image_pull_secrets: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/gha.go (main.ghaBuildPipeline) +# Generated at dronegen/gha.go (main.ghaMultiBuildPipeline) ################################################ kind: pipeline @@ -1208,7 +1208,7 @@ image_pull_secrets: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/gha.go (main.ghaBuildPipeline) +# Generated at dronegen/gha.go (main.ghaMultiBuildPipeline) ################################################ kind: pipeline @@ -4313,7 +4313,7 @@ image_pull_secrets: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/gha.go (main.ghaBuildPipeline) +# Generated at dronegen/gha.go (main.ghaMultiBuildPipeline) ################################################ kind: pipeline @@ -4595,7 +4595,7 @@ image_pull_secrets: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/gha.go (main.ghaBuildPipeline) +# Generated at dronegen/gha.go (main.ghaMultiBuildPipeline) ################################################ kind: pipeline @@ -6509,42 +6509,12 @@ image_pull_secrets: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go (main.buildNeverTriggerPipeline) +# Generated at dronegen/gha.go (main.ghaMultiBuildPipeline) ################################################ kind: pipeline type: kubernetes -name: migrate-apt-new-repos -trigger: - event: - include: - - custom - repo: - include: - - non-existent-repository - branch: - include: - - non-existent-branch -clone: - disable: true -steps: -- name: Placeholder - image: alpine:latest - commands: - - echo "This command, step, and pipeline never runs" -image_pull_secrets: -- DOCKERHUB_CREDENTIALS - ---- -################################################ -# Generated using dronegen, do not edit by hand! -# Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go (main.(*OsPackageToolPipelineBuilder).buildBaseOsPackagePipeline) -################################################ - -kind: pipeline -type: kubernetes -name: publish-apt-new-repos +name: publish-os-package-repos trigger: event: include: @@ -6554,21 +6524,14 @@ trigger: - production repo: include: - - gravitational/teleport - - gravitational/teleport-private + - gravitational/* workspace: path: /go clone: disable: true steps: -- name: Verify build is tagged - image: alpine:latest - pull: if-not-exists - commands: - - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' - && exit 1)' - name: Check out code - image: alpine/git:latest + image: docker:git pull: if-not-exists commands: - mkdir -pv "/go/src/github.com/gravitational/teleport" @@ -6576,361 +6539,51 @@ steps: - git init - git remote add origin ${DRONE_REMOTE_URL} - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" -- name: Assume Download AWS Role - image: amazon/aws-cli - pull: if-not-exists - commands: - - aws sts get-caller-identity - - |- - printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \ - $(aws sts assume-role \ - --role-arn "$AWS_ROLE" \ - --role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \ - --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ - --output text) \ - > /root/.aws/credentials - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - - aws sts get-caller-identity --profile default - environment: - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID - AWS_ROLE: - from_secret: AWS_ROLE - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Verify build is tagged - - Check out code -- name: Download artifacts for "${DRONE_TAG}" - image: amazon/aws-cli - commands: - - mkdir -pv "$ARTIFACT_PATH" - - rm -rf "$ARTIFACT_PATH"/* - - if [ "${DRONE_REPO_PRIVATE}" = true ]; then ENT_FILTER="*ent"; fi - - FILTER="${ENT_FILTER}*.deb*" - - aws s3 sync --no-progress --delete --exclude "*" --include "$FILTER" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ - "$ARTIFACT_PATH" - environment: - ARTIFACT_PATH: /go/artifacts - AWS_S3_BUCKET: - from_secret: AWS_S3_BUCKET - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Assume Download AWS Role - - Verify build is tagged - - Check out code -- name: Assume Upload AWS Role - image: amazon/aws-cli - pull: if-not-exists - commands: - - aws sts get-caller-identity - - |- - printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \ - $(aws sts assume-role \ - --role-arn "$AWS_ROLE" \ - --role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \ - --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ - --output text) \ - > /root/.aws/credentials - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - - aws sts get-caller-identity --profile default + - git checkout -qf "${DRONE_COMMIT_SHA}" + - mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa && + chmod 600 /root/.ssh/id_rsa + - ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts + - git submodule update --init e + - mkdir -pv /go/cache + - rm -f /root/.ssh/id_rsa environment: - AWS_ACCESS_KEY_ID: - from_secret: APT_REPO_NEW_AWS_ACCESS_KEY_ID - AWS_ROLE: - from_secret: APT_REPO_NEW_AWS_ROLE - AWS_SECRET_ACCESS_KEY: - from_secret: APT_REPO_NEW_AWS_SECRET_ACCESS_KEY - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Download artifacts for "${DRONE_TAG}" - - Verify build is tagged - - Check out code -- name: Check if tag is prerelease + GITHUB_PRIVATE_KEY: + from_secret: GITHUB_PRIVATE_KEY +- name: Determine if release should go to development or production image: golang:1.18-alpine commands: - - apk add git - - mkdir -pv "/tmp/repo" - - cd "/tmp/repo" - - git init - - git remote add origin ${DRONE_REMOTE_URL} - - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" - - cd "/tmp/repo/build.assets/tooling" - - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) - depends_on: - - Assume Upload AWS Role - - Verify build is tagged - - Check out code -- name: Publish debs to APT repos for "${DRONE_TAG}" - image: golang:1.18.4-bullseye - commands: - - apt update - - apt install -y aptly - - mkdir -pv -m0700 "$GNUPGHOME" - - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME - - chown -R root:root "$GNUPGHOME" - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - export VERSION="${DRONE_TAG}" - - export RELEASE_CHANNEL="stable" - - go run ./cmd/build-os-package-repos apt -bucket "$REPO_S3_BUCKET" -local-bucket-path - "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" - -artifact-path "$ARTIFACT_PATH" -log-level 4 -aptly-root-dir "$APTLY_ROOT_DIR" - environment: - APTLY_ROOT_DIR: /mnt/aptly - ARTIFACT_PATH: /go/artifacts - AWS_REGION: us-west-2 - BUCKET_CACHE_PATH: /tmp/bucket - DEBIAN_FRONTEND: noninteractive - GNUPGHOME: /tmpfs/gnupg - GPG_RPM_SIGNING_ARCHIVE: - from_secret: GPG_RPM_SIGNING_ARCHIVE - REPO_S3_BUCKET: - from_secret: APT_REPO_NEW_AWS_S3_BUCKET - volumes: - - name: apt-persistence - path: /mnt - - name: tmpfs - path: /tmpfs - - name: awsconfig - path: /root/.aws - depends_on: - - Check if tag is prerelease - - Verify build is tagged - - Check out code -volumes: -- name: apt-persistence - claim: - name: drone-s3-aptrepo-pvc -- name: tmpfs - temp: - medium: memory -- name: awsconfig - temp: {} -image_pull_secrets: -- DOCKERHUB_CREDENTIALS - ---- -################################################ -# Generated using dronegen, do not edit by hand! -# Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go (main.buildNeverTriggerPipeline) -################################################ - -kind: pipeline -type: kubernetes -name: migrate-yum-new-repos -trigger: - event: - include: - - custom - repo: - include: - - non-existent-repository - branch: - include: - - non-existent-branch -clone: - disable: true -steps: -- name: Placeholder - image: alpine:latest - commands: - - echo "This command, step, and pipeline never runs" -image_pull_secrets: -- DOCKERHUB_CREDENTIALS - ---- -################################################ -# Generated using dronegen, do not edit by hand! -# Use 'make dronegen' to update. -# Generated at dronegen/os_repos.go (main.(*OsPackageToolPipelineBuilder).buildBaseOsPackagePipeline) -################################################ - -kind: pipeline -type: kubernetes -name: publish-yum-new-repos -trigger: - event: - include: - - promote - target: - include: - - production - repo: - include: - - gravitational/teleport - - gravitational/teleport-private -workspace: - path: /go -clone: - disable: true -steps: -- name: Verify build is tagged - image: alpine:latest - pull: if-not-exists - commands: - - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' - && exit 1)' -- name: Check out code - image: alpine/git:latest - pull: if-not-exists - commands: - - mkdir -pv "/go/src/github.com/gravitational/teleport" - - cd "/go/src/github.com/gravitational/teleport" - - git init - - git remote add origin ${DRONE_REMOTE_URL} - - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" -- name: Assume Download AWS Role - image: amazon/aws-cli - pull: if-not-exists - commands: - - aws sts get-caller-identity - - |- - printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \ - $(aws sts assume-role \ - --role-arn "$AWS_ROLE" \ - --role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \ - --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ - --output text) \ - > /root/.aws/credentials - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - - aws sts get-caller-identity --profile default - environment: - AWS_ACCESS_KEY_ID: - from_secret: AWS_ACCESS_KEY_ID - AWS_ROLE: - from_secret: AWS_ROLE - AWS_SECRET_ACCESS_KEY: - from_secret: AWS_SECRET_ACCESS_KEY - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Verify build is tagged - - Check out code -- name: Download artifacts for "${DRONE_TAG}" - image: amazon/aws-cli - commands: - - mkdir -pv "$ARTIFACT_PATH" - - rm -rf "$ARTIFACT_PATH"/* - - if [ "${DRONE_REPO_PRIVATE}" = true ]; then ENT_FILTER="*ent"; fi - - FILTER="${ENT_FILTER}*.rpm*" - - aws s3 sync --no-progress --delete --exclude "*" --include "$FILTER" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ - "$ARTIFACT_PATH" - environment: - ARTIFACT_PATH: /go/artifacts - AWS_S3_BUCKET: - from_secret: AWS_S3_BUCKET - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Assume Download AWS Role - - Verify build is tagged - - Check out code -- name: Assume Upload AWS Role - image: amazon/aws-cli + - mkdir -pv "/go/vars" + - (go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "promote" || echo + "build") > "/go/vars/release-environment.txt" +- name: Publish Teleport to stable/${DRONE_TAG} apt repo + image: golang:1.18-alpine pull: if-not-exists commands: - - aws sts get-caller-identity - - |- - printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\n" \ - $(aws sts assume-role \ - --role-arn "$AWS_ROLE" \ - --role-session-name $(echo "drone-${DRONE_REPO}-${DRONE_BUILD_NUMBER}" | sed "s|/|-|g") \ - --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ - --output text) \ - > /root/.aws/credentials - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - - aws sts get-caller-identity --profile default + - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -series-run -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master + -input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")" + -input "package-name-filter=$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")" + -input "package-to-test=teleport-ent" -input "release-channel=stable" -input "repo-type=apt" + -input "version-channel=${DRONE_TAG}" ' environment: - AWS_ACCESS_KEY_ID: - from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID - AWS_ROLE: - from_secret: YUM_REPO_NEW_AWS_ROLE - AWS_SECRET_ACCESS_KEY: - from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY - volumes: - - name: awsconfig - path: /root/.aws - depends_on: - - Download artifacts for "${DRONE_TAG}" - - Verify build is tagged - - Check out code -- name: Check if tag is prerelease + GHA_APP_KEY: + from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY +- name: Publish Teleport to stable/${DRONE_TAG} yum repo image: golang:1.18-alpine + pull: if-not-exists commands: - - apk add git - - mkdir -pv "/tmp/repo" - - cd "/tmp/repo" - - git init - - git remote add origin ${DRONE_REMOTE_URL} - - git fetch origin --tags - - git checkout -qf "${DRONE_TAG}" - - cd "/tmp/repo/build.assets/tooling" - - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is - a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78) - depends_on: - - Assume Upload AWS Role - - Verify build is tagged - - Check out code -- name: Publish rpms to YUM repos for "${DRONE_TAG}" - image: golang:1.18.4-bullseye - commands: - - apt update - - apt install -y createrepo-c - - mkdir -pv "$CACHE_DIR" - - mkdir -pv -m0700 "$GNUPGHOME" - - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME - - chown -R root:root "$GNUPGHOME" - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - - export VERSION="${DRONE_TAG}" - - export RELEASE_CHANNEL="stable" - - go run ./cmd/build-os-package-repos yum -bucket "$REPO_S3_BUCKET" -local-bucket-path - "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" - -artifact-path "$ARTIFACT_PATH" -log-level 4 -cache-dir "$CACHE_DIR" + - 'go run ./cmd/gh-trigger-workflow -owner ${DRONE_REPO_OWNER} -repo teleport.e + -tag-workflow -series-run -timeout 12h0m0s -workflow deploy-packages.yaml -workflow-ref=refs/heads/master + -input "artifact-tag=${DRONE_TAG}" -input "environment=$(cat "/go/vars/release-environment.txt")" + -input "package-name-filter=$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")" + -input "package-to-test=teleport-ent" -input "release-channel=stable" -input "repo-type=yum" + -input "version-channel=${DRONE_TAG}" ' environment: - ARTIFACT_PATH: /go/artifacts - AWS_REGION: us-west-2 - BUCKET_CACHE_PATH: /mnt/bucket - CACHE_DIR: /mnt/createrepo_cache - DEBIAN_FRONTEND: noninteractive - GNUPGHOME: /tmpfs/gnupg - GPG_RPM_SIGNING_ARCHIVE: - from_secret: GPG_RPM_SIGNING_ARCHIVE - REPO_S3_BUCKET: - from_secret: YUM_REPO_NEW_AWS_S3_BUCKET - volumes: - - name: yum-persistence - path: /mnt - - name: tmpfs - path: /tmpfs - - name: awsconfig - path: /root/.aws - depends_on: - - Check if tag is prerelease - - Verify build is tagged - - Check out code -volumes: -- name: yum-persistence - claim: - name: drone-s3-yumrepo-pvc -- name: tmpfs - temp: - medium: memory -- name: awsconfig - temp: {} + GHA_APP_KEY: + from_secret: GITHUB_WORKFLOW_APP_PRIVATE_KEY image_pull_secrets: - DOCKERHUB_CREDENTIALS @@ -19262,8 +18915,7 @@ clone: depends_on: - promote-build - teleport-container-images-branch-promote -- publish-apt-new-repos -- publish-yum-new-repos +- publish-os-package-repos steps: - name: Check if commit is tagged image: alpine @@ -19373,6 +19025,6 @@ image_pull_secrets: - DOCKERHUB_CREDENTIALS --- kind: signature -hmac: 3224c6e70706c2a83d73cb32d45447d59f3c931ca9942cf4a50a5a061e855618 +hmac: 496201bc9c20a753db18ed878ff8fdcd8a8df8e88c7fe6671846850cd5ac25bf ... diff --git a/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go b/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go deleted file mode 100644 index 66e0e60ed9545..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go +++ /dev/null @@ -1,241 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "io/fs" - "path/filepath" - "strings" - "time" - - "github.com/davecgh/go-spew/spew" - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" -) - -type AptRepoTool struct { - config *AptConfig - aptly *Aptly - gpg *GPG - s3Manager *S3manager - supportedOSs map[string][]string -} - -// Instantiates a new apt repo tool instance and performs any required setup/config. -func NewAptRepoTool(config *AptConfig, supportedOSs map[string][]string) (*AptRepoTool, error) { - aptly, err := NewAptly(config.aptlyPath) - if err != nil { - return nil, trace.Wrap(err, "failed to create a new aptly instance") - } - - gpg, err := NewGPG() - if err != nil { - return nil, trace.Wrap(err, "failed to create a new GPG instance") - } - - s3Manager, err := NewS3Manager(config.S3Config) - if err != nil { - return nil, trace.Wrap(err, "failed to create a new s3manager instance") - } - - return &AptRepoTool{ - aptly: aptly, - config: config, - gpg: gpg, - s3Manager: s3Manager, - supportedOSs: supportedOSs, - }, nil -} - -// Runs the tool, creating and updating APT repos based upon the current configuration. -func (art *AptRepoTool) Run() error { - start := time.Now() - logrus.Infoln("Starting APT repo build process...") - logrus.Debugf("Using config: %+v", spew.Sdump(art.config)) - - isFirstRun, err := art.aptly.IsFirstRun() - if err != nil { - return trace.Wrap(err, "failed to check if Aptly needs (re)built") - } - - if isFirstRun { - logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from APT repository...") - - err = art.s3Manager.DownloadExistingRepo() - if err != nil { - return trace.Wrap(err, "failed to sync existing repo from S3 bucket") - } - - _, err = art.recreateExistingRepos(art.config.localBucketPath) - if err != nil { - return trace.Wrap(err, "failed to recreate existing repos") - } - } else { - logrus.Debugf("Not first run of tool, skipping Aptly repository rebuild process") - } - - // Note: this logic will only push the artifact into the `art.supportedOSs` repos. - // This behavior is intended to allow deprecating old OS versions in the future - // without removing the associated repos entirely. - artifactRepos, err := art.getArtifactRepos() - if err != nil { - return trace.Wrap(err, "failed to create repos") - } - - err = art.importNewDebs(artifactRepos) - if err != nil { - return trace.Wrap(err, "failed to import new debs") - } - - err = art.publishRepos() - if err != nil { - return trace.Wrap(err, "failed to publish repos") - } - - // Both Hashicorp and Docker publish their key to this path - err = art.gpg.WritePublicKeyToFile(filepath.Join(art.aptly.rootDir, "public", "gpg")) - if err != nil { - return trace.Wrap(err, "failed to write GPG public key") - } - - art.s3Manager.ChangeLocalBucketPath(filepath.Join(art.aptly.rootDir, "public")) - err = art.s3Manager.UploadBuiltRepo() - if err != nil { - return trace.Wrap(err, "failed to sync changes to S3 bucket") - } - - // Future work: add literals to config? - err = art.s3Manager.UploadRedirectURL("index.html", "https://goteleport.com/docs/installation/#linux") - if err != nil { - return trace.Wrap(err, "failed to redirect index page to Teleport docs") - } - - logrus.Infof("APT repo build process completed in %s", time.Since(start).Round(time.Millisecond)) - return nil -} - -func (art *AptRepoTool) publishRepos() error { - // Pull in all Aptly repos, not just the latest ones to ensure they all get built into APT repos correctly - repos, err := art.aptly.GetAllRepos() - if err != nil { - return trace.Wrap(err, "failed to get all Aptly repos") - } - - // Build a map keyed by os info with value of all repos that support the os in the key - // This will be used to structure the publish command - logrus.Debugf("Categorizing repos according to OS info: %v", RepoNames(repos)) - categorizedRepos := make(map[string][]*Repo) - for _, r := range repos { - if osRepos, ok := categorizedRepos[r.OSInfo()]; ok { - categorizedRepos[r.OSInfo()] = append(osRepos, r) - } else { - categorizedRepos[r.OSInfo()] = []*Repo{r} - } - } - logrus.Debugf("Categorized repos: %v", categorizedRepos) - - for osInfo, osRepoList := range categorizedRepos { - if len(osRepoList) < 1 { - continue - } - - err := art.aptly.PublishRepos(osRepoList, osRepoList[0].os, osRepoList[0].osVersion) - if err != nil { - return trace.Wrap(err, "failed to publish for os %q", osInfo) - } - } - - return nil -} - -func (art *AptRepoTool) recreateExistingRepos(localPublishedPath string) ([]*Repo, error) { - logrus.Infoln("Recreating previously published repos...") - createdRepos, err := art.aptly.CreateReposFromPublishedPath(localPublishedPath) - if err != nil { - return nil, trace.Wrap(err, "failed to recreate existing repos") - } - - for _, repo := range createdRepos { - err := art.aptly.ImportDebsFromExistingRepo(repo) - if err != nil { - return nil, trace.Wrap(err, "failed to import debs from existing repo %q", repo.Name()) - } - } - - logrus.Infof("Recreated and imported pre-existing artifacts for %d repos", len(createdRepos)) - return createdRepos, nil -} - -func (art *AptRepoTool) getArtifactRepos() ([]*Repo, error) { - logrus.Infoln("Creating or getting Aptly repos for artifact requirements...") - - artifactRepos, err := art.aptly.CreateReposFromArtifactRequirements(art.supportedOSs, - art.config.releaseChannel, semver.Major(art.config.artifactVersion)) - if err != nil { - return nil, trace.Wrap(err, "failed to create or get repos from artifact requirements") - } - - logrus.Infof("Created or got %d artifact Aptly repos", len(artifactRepos)) - return artifactRepos, nil -} - -func (art *AptRepoTool) importNewDebs(repos []*Repo) error { - logrus.Debugf("Importing new debs into %d repos: %q", len(repos), strings.Join(RepoNames(repos), "\", \"")) - err := filepath.WalkDir(art.config.artifactPath, - func(debPath string, d fs.DirEntry, err error) error { - return art.importNewDebsWalker(debPath, d, err, repos) - }, - ) - if err != nil { - return trace.Wrap(err, "failed to find and import debs") - } - - return nil -} - -// This should not be used outside of importNewDebs -func (art *AptRepoTool) importNewDebsWalker(debPath string, d fs.DirEntry, err error, repos []*Repo) error { - if err != nil { - return trace.Wrap(err, "failure while searching %s for debs", debPath) - } - - if d.IsDir() { - return nil - } - - fileName := d.Name() - if filepath.Ext(fileName) != ".deb" { - return nil - } - - // Import new artifacts into all repos that match the artifact's requirements - for _, repo := range repos { - // Other checks could be added here to ensure that a given deb gets added to the correct repo - // such as name or parent directory, facilitating os-specific artifacts - if repo.majorVersion != semver.Major(art.config.artifactVersion) || repo.releaseChannel != art.config.releaseChannel { - continue - } - - err = art.aptly.ImportDeb(repo.Name(), debPath) - if err != nil { - return trace.Wrap(err, "failed to import deb from %s", debPath) - } - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/aptly.go b/build.assets/tooling/cmd/build-os-package-repos/aptly.go deleted file mode 100644 index 12f7f42fe0de1..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/aptly.go +++ /dev/null @@ -1,676 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bufio" - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "regexp" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" -) - -// This provides wrapper functions for the Aptly command. Aptly is written in Go but it doesn't appear -// to have a good binary API to use, only a CLI tool and REST API. - -type Aptly struct { - rootDir string -} - -// Instantiates Aptly, performing any system configuration needed. -func NewAptly(rootDir string) (*Aptly, error) { - a := &Aptly{ - rootDir: rootDir, - } - - err := a.ensureDefaultConfigExists() - if err != nil { - return nil, trace.Wrap(err, "failed to ensure Aptly default config exists") - } - - err = a.updateConfiguration() - if err != nil { - return nil, trace.Wrap(err, "failed to load Aptly configuration") - } - - return a, nil -} - -func (*Aptly) ensureDefaultConfigExists() error { - // If the default config doesn't exist then it will be created the first time an Aptly command is - // ran, which messes up the output. - // Note: it is important to not use any repo-related commands here as they have a side effect of - // also creating the Aptly rootDir structure which is usually undesirable here - _, err := BuildAndRunCommand("aptly", "config", "show") - if err != nil { - return trace.Wrap(err, "failed to create default Aptly config") - } - - return nil -} - -func (a *Aptly) updateConfiguration() error { - aptlyConfigMap, err := loadAptlyConfigMap() - if err != nil { - return trace.Wrap(err, "failed to load Aptly config map") - } - - // Additional config can be handled here if needed in the future - aptlyConfigMap["rootDir"] = a.rootDir - - logrus.Debugf("Built Aptly config: %v", aptlyConfigMap) - saveAptlyConfigMap(aptlyConfigMap) - - configOutput, err := BuildAndRunCommand("aptly", "config", "show") - if err != nil { - return trace.Wrap(err, "failed to check Aptly config") - } - logrus.Debugln("Aptly config on disk:") - logrus.Debugf("%v", configOutput) - - return nil -} - -func saveAptlyConfigMap(aptlyConfigMap map[string]interface{}) error { - aptlyConfigData, err := json.MarshalIndent(aptlyConfigMap, "", " ") - if err != nil { - return trace.Wrap(err, "failed to marshal Aptly config with data %v", aptlyConfigMap) - } - - aptlyConfigPath, err := getAptlyConfigPath() - if err != nil { - return trace.Wrap(err, "failed to get Aptly config path") - } - - err = os.WriteFile(aptlyConfigPath, aptlyConfigData, 0644) - if err != nil { - return trace.Wrap(err, "failed to write Aptly config to %q with data %q", aptlyConfigPath, string(aptlyConfigData)) - } - - return nil -} - -func loadAptlyConfigMap() (map[string]interface{}, error) { - aptlyConfigPath, err := getAptlyConfigPath() - if err != nil { - return nil, trace.Wrap(err, "failed to get Aptly config path") - } - - aptlyConfigData, err := os.ReadFile(aptlyConfigPath) - if err != nil { - return nil, trace.Wrap(err, "failed to read Aptly config file at %q", aptlyConfigPath) - } - - var aptlyConfigMap map[string]interface{} - if err := json.Unmarshal(aptlyConfigData, &aptlyConfigMap); err != nil { - return nil, trace.Wrap(err, "failed to unmarshal Aptly config from file %q with data %q", aptlyConfigPath, string(aptlyConfigData)) - } - - return aptlyConfigMap, nil -} - -// This follows the config logic specified here: https://www.aptly.info/doc/configuration/ -func getAptlyConfigPath() (string, error) { - userHomeDir, err := os.UserHomeDir() - if err != nil { - return "", trace.Wrap(err, "failed to get user home directory path") - } - - userAptlyConfigPath := path.Join(userHomeDir, ".aptly.conf") - _, err = os.Stat(userAptlyConfigPath) - if err == nil { - return userAptlyConfigPath, nil - } - - if !errors.Is(err, os.ErrNotExist) { - return "", trace.Wrap(err, "failed to check if %q exists", userAptlyConfigPath) - } - - systemAptlyConfigPath := "/etc/aptly.conf" - _, err = os.Stat(systemAptlyConfigPath) - if err == nil { - return systemAptlyConfigPath, nil - } - - if !errors.Is(err, os.ErrNotExist) { - return "", trace.Wrap(err, "failed to check if %q exists", systemAptlyConfigPath) - } - - return "", trace.Errorf("Aptly config not found at %q or %q", userAptlyConfigPath, systemAptlyConfigPath) -} - -func (a *Aptly) IsFirstRun() (bool, error) { - _, err := os.Stat(a.rootDir) - if err == nil { - return false, nil - } - - if errors.Is(err, os.ErrNotExist) { - return true, nil - } - - return false, trace.Wrap(err, "failed to check if %q exists", a.rootDir) -} - -// Creates the provided repo `r` via Aptly. Returns true if the repo was created, false otherewise. -func (a *Aptly) CreateRepoIfNotExists(r *Repo) (bool, error) { - logrus.Debugf("Creating repo %q if it doesn't already exist..", r.Name()) - doesRepoExist, err := a.DoesRepoExist(r) - if err != nil { - return false, trace.Wrap(err, "failed to check whether or not the repo %q already exists", r.Name()) - } - - if doesRepoExist { - logrus.Debugf("Repo %q already exists, skipping creation", r.Name()) - return false, nil - } - - distributionArg := fmt.Sprintf("-distribution=%s", r.osVersion) - componentArg := fmt.Sprintf("-component=%s/%s", r.releaseChannel, r.majorVersion) - _, err = BuildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name()) - if err != nil { - return false, trace.Wrap(err, "failed to create repo %q", r.Name()) - } - - logrus.Debugf("Created repo %q", r.Name()) - return true, nil -} - -// Checks to see if the Aptly described by repo `r` exists. Returns true if it exists, false otherwise. -func (a *Aptly) DoesRepoExist(r *Repo) (bool, error) { - logrus.Debugf("Checking if repo %q exists...", r.Name()) - - existingRepoNames, err := a.GetExistingRepoNames() - if err != nil { - return false, trace.Wrap(err, "failed to get existing repo names") - } - - return slices.Contains(existingRepoNames, r.Name()), nil -} - -// Gets a list of the name of Aptly repos that already exists. -func (a *Aptly) GetExistingRepoNames() ([]string, error) { - logrus.Debugln("Getting a list of pre-existing repos...") - // The output of the command will be simiar to: - // ``` - // - // ... - // - // ``` - output, err := BuildAndRunCommand("aptly", "repo", "list", "-raw") - if err != nil { - return nil, trace.Wrap(err, "failed to get a list of existing repos") - } - - // Split the command output by new line - parsedRepoNames := strings.Split(output, "\n") - - // The names may have whitespace and the command may print an extra blank line, so we remove those here - var validRepoNames []string - for _, parsedRepoName := range parsedRepoNames { - if trimmedRepoName := strings.TrimSpace(parsedRepoName); trimmedRepoName != "" { - validRepoNames = append(validRepoNames, trimmedRepoName) - } - } - - logrus.Debugf("Found %d repos: %q", len(validRepoNames), strings.Join(validRepoNames, "\", \"")) - return validRepoNames, nil -} - -// Imports a deb at `debPath` into the Aptly repo of name `repoName`. -// If `debPath` is a folder, the folder will be searched recursively for *.deb files -// which are then imported into the repo. -func (a *Aptly) ImportDeb(repoName string, debPath string) error { - logrus.Infof("Importing deb(s) from %q into repo %q...", debPath, repoName) - - _, err := BuildAndRunCommand("aptly", "repo", "add", repoName, debPath) - if err != nil { - return trace.Wrap(err, "failed to add %q to repo %q", debPath, repoName) - } - - return nil -} - -// This function imports deb files from a preexisting published repo, typically created from a previous run of this tool. -func (a *Aptly) ImportDebsFromExistingRepo(repo *Repo) error { - logrus.Infof("Importing pre-existing debs from repo %q...", repo.Name()) - publishedRepoAbsolutePath, err := repo.PublishedRepoAbsolutePath() - if err != nil { - return trace.Wrap(err, "failed to get the absolute path of the published repo %q", repo.Name()) - } - - logrus.Debugf("Looking in %q for Packages files...", publishedRepoAbsolutePath) - err = filepath.WalkDir(publishedRepoAbsolutePath, - func(packagesPath string, d fs.DirEntry, err error) error { - if err != nil { - return trace.Wrap(err, "failure while searching %s for Packages files", packagesPath) - } - - if d.IsDir() { - return nil - } - - if d.Name() != "Packages" { - return nil - } - - logrus.Debugf("Matched %q as a Packages file, attempting to import listed debs into %q...", packagesPath, repo.Name()) - err = a.importDebsFromPackagesFile(repo, packagesPath) - if err != nil { - return trace.Wrap(err, "failed to import debs into repo %q from packages file %q", repo.Name(), packagesPath) - } - - return nil - }, - ) - - if err != nil { - return trace.Wrap(err, "failed to find and import debs from existing repo %q at published path %q", repo.Name(), publishedRepoAbsolutePath) - } - - return nil -} - -func (a *Aptly) importDebsFromPackagesFile(repo *Repo, packagesPath string) error { - logrus.Debugf("Importing debs from %q into %q", packagesPath, repo.Name()) - debRelativeFilePaths, err := parsePackagesFile(packagesPath) - if err != nil { - return trace.Wrap(err, "failed to parse packages file %q for deb file paths", packagesPath) - } - - logrus.Debugf("Found %d debs listed in %q: %q", len(debRelativeFilePaths), packagesPath, strings.Join(debRelativeFilePaths, "\", \"")) - for _, debRelativeFilePath := range debRelativeFilePaths { - debPath := path.Join(repo.publishedSourcePath, repo.os, debRelativeFilePath) - logrus.Debugf("Constructed deb absolute path %q", debPath) - err = a.ImportDeb(repo.Name(), debPath) - if err != nil { - return trace.Wrap(err, "failed to import deb into repo %q from %q", repo.Name(), debPath) - } - } - - return nil -} - -func parsePackagesFile(packagesPath string) ([]string, error) { - logrus.Debugf("Parsing packages file %q", packagesPath) - file, err := os.Open(packagesPath) - if err != nil { - logrus.Fatal(err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - debRelativeFilePaths := make([]string, 0, 1) // If the package file exists then it should contain at least one deb file path - for scanner.Scan() { - line := scanner.Text() - // Skip empty lines - if line == "" { - continue - } - - key, value, err := parsePackagesFileLine(line) - if err != nil { - return nil, trace.Wrap(err, "failed to parse line %q in packages file %q", line, packagesPath) - } - - if key != "Filename" { - continue - } - - logrus.Debugf("Found deb file listed at relative path %q", value) - debRelativeFilePaths = append(debRelativeFilePaths, value) - } - - if err := scanner.Err(); err != nil { - return nil, trace.Wrap(err, "received error while reading %q", packagesPath) - } - - return debRelativeFilePaths, nil -} - -func parsePackagesFileLine(line string) (string, string, error) { - splitLine := strings.SplitN(line, ": ", 2) - if len(splitLine) != 2 { - return "", "", trace.Errorf("packages file line %q is malformed", line) - } - - key := splitLine[0] - value := splitLine[1] - - if key == "" { - return "", "", trace.Errorf("packages file line %q contains an empty key", line) - } - - if value == "" { - return "", "", trace.Errorf("packages file line %q contains an empty value", line) - } - - return key, value, nil -} - -// Publishes the Aptly repos defined in the `repos` slice to the `repoOS` subpath. -func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) error { - repoNames := RepoNames(repos) - logrus.Infof("Publishing repos for OS %q: %q...", repoOS, strings.Join(repoNames, "\", \"")) - - // Trying to publish to an already published OS/OS version will fail, and dropping a published - // OS/OS version when no new components (release channel/major version) are added is - // computationally expensive - areSomeUnpublished, areSomePublished, err := a.getRepoSlicePublishedState(repos) - if err != nil { - return trace.Wrap(err, "failed to determine if repos for have been published or not") - } - - logrus.Debugln("Repo OS/OS version combo publish state:") - logrus.Debugf("Are some unpublished: %v", areSomeUnpublished) - logrus.Debugf("Are some published: %v", areSomePublished) - logrus.Debugf("Repos: %v", RepoNames(repos)) - - // If all repos have been published - if areSomePublished && !areSomeUnpublished { - // Update rather than republish - _, err := BuildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS) - if err != nil { - return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) - } - - return nil - } - - // If some have been published and some have not - // This will occur if there is a new major release, a OS version is supported, or a new release channel is added - if areSomePublished && areSomeUnpublished { - // Drop the currently published APT repo so that it can be rebuilt from scratch - _, err := BuildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS) - if err != nil { - return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) - } - } - - // If all repos have not been published (or were just unpublished/dropped) - // Build the command args and publish all the repos - args := []string{"publish", "repo"} - if len(repos) > 1 { - componentsArgument := fmt.Sprintf("-component=%s", strings.Repeat(",", len(repos)-1)) - args = append(args, componentsArgument) - } - args = append(args, repoNames...) - args = append(args, repoOS) - - // Full command is `aptly publish repo -component=<, repeating len(repos) - 1 times> ` - _, err = BuildAndRunCommand("aptly", args...) - if err != nil { - return trace.Wrap(err, "failed to publish repos") - } - - return nil -} - -// This function determines if `repos` contains repos that have an OS/OS version combo -// that have not yet been published, and if it contains repos that have an OS/OS version -// combo that have been published. -// -// Returns: -// -// 1. true if `repos` contains at least one repo who's OS/OS version combo has not been -// published yet -// -// 2. true if `repos` contains at least one repo who's OS/OS version combo has been -// published -// -// Example: -// -// `getRepoSlicePublishedState([, -// ])` will return -// -// 1. `true, false, nil` if a repo for debian/trixie, debian/buster has not been published yet -// -// 2. `false, true, nil` if a repo for debian/trixie, debian/buster has been published -// -// 3. `true, true, nil` if a repo for debian/trixie has not been published yet but debian/buster -// has, or vice versa -func (a *Aptly) getRepoSlicePublishedState(repos []*Repo) (bool, bool, error) { - publishedRepoNames, err := a.GetPublishedRepoNames() - if err != nil { - return false, false, trace.Wrap(err, "failed to get a list of published repos' names") - } - - containsUnpublishedRepo := false - containsPublishedRepo := false - for _, repo := range repos { - repoName := repo.Name() - hasRepoBeenPublished := slices.Contains(publishedRepoNames, repoName) - logrus.Debugf("Repo %q has been published: %v", repoName, hasRepoBeenPublished) - containsUnpublishedRepo = containsUnpublishedRepo || !hasRepoBeenPublished - containsPublishedRepo = containsPublishedRepo || hasRepoBeenPublished - - // No need to keep checking if they're both already true - if containsUnpublishedRepo && containsPublishedRepo { - break - } - } - - // failsafe in case of some underlying logic change - if !containsUnpublishedRepo && !containsPublishedRepo { - return false, false, trace.Errorf("something went very wrong here") - } - - return containsUnpublishedRepo, containsPublishedRepo, nil -} - -func (a *Aptly) GetPublishedRepoNames() ([]string, error) { - logrus.Debugln("Getting a list of published repos...") - // The output of the command will be simiar to: - // ``` - // Published repositories: - // * / [, ..., ] publishes {: []}, ..., {: []} - // ... - // * / [, ..., ] publishes {: []}, ..., {: []} - // * / [, ..., ] publishes {: []}, ..., {: []} - // ... - // * / [, ..., ] publishes {: []}, ..., {: []} - // ``` - // - // If no repos have been published then the output will be similar to: - // ``` - // No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`. - // ``` - // Note that the `-raw` argument is not used here as it does not provide sufficient information - output, err := BuildAndRunCommand("aptly", "publish", "list") - if err != nil { - return nil, trace.Wrap(err, "failed to get a list of published repos") - } - - // Split the command output by new line - publishedRepoLines := strings.Split(output, "\n") - // In all cases the first line should exist and not be parsed - publishedRepoLines = publishedRepoLines[1:] - - repoNameRegexStr := ": \\[(.+?)\\]" - repoNameRegex, err := regexp.Compile(repoNameRegexStr) - if err != nil { - return nil, trace.Wrap(err, "failed to compile repo name regex %q", repoNameRegexStr) - } - - var publishedRepoNames []string - for _, publishedRepoLine := range publishedRepoLines { - // The names may have whitespace and the command may print an extra blank line, so we remove those here - if trimmedRepoLine := strings.TrimSpace(publishedRepoLine); trimmedRepoLine != "" { - // Additional parsing should go here if needed in the future - repoNameMatches := repoNameRegex.FindAllStringSubmatch(publishedRepoLine, -1) - if repoNameMatches == nil { - return nil, trace.Errorf("failed to match repo names in line %q with regex %q", publishedRepoLine, repoNameRegexStr) - } - - for _, repoNameMatch := range repoNameMatches { - // `repoNameRegexStr` is written such that there will be exactly one match and one group in repoNameMatch - // for example repoNameMatch could be [": [debian-bookwork-stable-v6]", "debian-bookwork-stable-v6"] - publishedRepoName := repoNameMatch[1] - publishedRepoNames = append(publishedRepoNames, publishedRepoName) - } - } - } - - logrus.Debugf("Found %d published repos: %q", len(publishedRepoNames), publishedRepoNames) - return publishedRepoNames, nil -} - -// Creates Aptly repos from a local path that has previously published Apt repos created by this tool. -// Returns a list of repo objects describing the created Aptly repos. -func (a *Aptly) CreateReposFromPublishedPath(localPublishedPath string) ([]*Repo, error) { - // The file tree that we care about here will be of the following structure: - // `///dists////...` - logrus.Infof("Recreating previously published repos from %q...", localPublishedPath) - - createdRepos := []*Repo{} - - osSubdirectories, err := getSubdirectories(localPublishedPath) - if err != nil { - return nil, trace.Wrap(err, "failed to get OS subdirectories in %s", localPublishedPath) - } - - for _, os := range osSubdirectories { - osVersionParentDirectory := filepath.Join(localPublishedPath, os, "dists") - osVersionSubdirectories, err := getSubdirectories(osVersionParentDirectory) - if err != nil { - return nil, trace.Wrap(err, "failed to get OS version subdirectories in %s", localPublishedPath) - } - - for _, osVersion := range osVersionSubdirectories { - releaseChannelParentDirectory := filepath.Join(osVersionParentDirectory, osVersion) - releaseChannelSubdirectories, err := getSubdirectories(releaseChannelParentDirectory) - if err != nil { - return nil, trace.Wrap(err, "failed to get release channel subdirectories in %s", localPublishedPath) - } - - for _, releaseChannel := range releaseChannelSubdirectories { - majorVersionParentDirectory := filepath.Join(releaseChannelParentDirectory, releaseChannel) - majorVersionSubdirectories, err := getSubdirectories(majorVersionParentDirectory) - if err != nil { - return nil, trace.Wrap(err, "failed to get major version subdirectories in %s", localPublishedPath) - } - - for _, majorVersion := range majorVersionSubdirectories { - r := &Repo{ - os: os, - osVersion: osVersion, - releaseChannel: releaseChannel, - majorVersion: majorVersion, - publishedSourcePath: localPublishedPath, - } - - wasRepoCreated, err := a.CreateRepoIfNotExists(r) - if err != nil { - return nil, trace.Wrap(err, "failed to create repo %q", r.Name()) - } - - if wasRepoCreated { - createdRepos = append(createdRepos, r) - } - } - } - } - } - - logrus.Infof("Recreated %d repos", len(createdRepos)) - return createdRepos, nil -} - -// Creates or gets Aptly repos for all permutations of the provided requirements. -// Returns a list of repo objects describing the Aptly repos, regardless of if they -// already existed. -// -// supportedOSInfo should be a dictionary keyed by OS name, with values being a list of -// supported OS version codenames. -func (a *Aptly) CreateReposFromArtifactRequirements(supportedOSInfo map[string][]string, - releaseChannel string, majorVersion string) ([]*Repo, error) { - logrus.Infoln("Creating new repos from artifact requirements:") - logrus.Infof("Supported OSs: %+v", supportedOSInfo) - logrus.Infof("Release channel: %q", releaseChannel) - logrus.Infof("Artifact major version: %q", majorVersion) - - artifactRequirementRepos := []*Repo{} - for os, osVersions := range supportedOSInfo { - for _, osVersion := range osVersions { - r := &Repo{ - os: os, - osVersion: osVersion, - releaseChannel: releaseChannel, - majorVersion: majorVersion, - } - - _, err := a.CreateRepoIfNotExists(r) - if err != nil { - return nil, trace.Wrap(err, "failed to create repo %q", r.Name()) - } - - artifactRequirementRepos = append(artifactRequirementRepos, r) - } - } - - return artifactRequirementRepos, nil -} - -// Returns a list of all Aptly reported repos -func (a *Aptly) GetAllRepos() ([]*Repo, error) { - repoNames, err := a.GetExistingRepoNames() - if err != nil { - return nil, trace.Wrap(err, "failed to get existing repo names") - } - - repos := make([]*Repo, len(repoNames)) - for i, repoName := range repoNames { - repo, err := NewRepoFromName(repoName) - if err != nil { - return nil, trace.Wrap(err, "failed to build repo struct for repo name %q", repoName) - } - - repos[i] = repo - } - - return repos, nil -} - -func getSubdirectories(basePath string) ([]string, error) { - logrus.Debugf("Getting subdirectories of %q...", basePath) - files, err := os.ReadDir(basePath) - if err != nil { - return nil, trace.Wrap(err, "failed to read directory %q", basePath) - } - - subdirectories := []string{} - for _, file := range files { - if !file.IsDir() { - continue - } - - subdirectory := file.Name() - logrus.Debugf("Found subdirectory %q", subdirectory) - subdirectories = append(subdirectories, subdirectory) - } - - return subdirectories, nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/command_executor.go b/build.assets/tooling/cmd/build-os-package-repos/command_executor.go deleted file mode 100644 index 78a9bb8b7342d..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/command_executor.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "os/exec" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" -) - -// Builds an runs a command with the provided arguments. Extensively logs command -// details to the debug log. Returns stdout and stderr combined, along with an -// error iff one occurred. -func BuildAndRunCommand(command string, args ...string) (string, error) { - cmd := exec.Command(command, args...) - logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '")) - output, err := cmd.CombinedOutput() - - if output != nil { - logrus.Debugf("Command output: %s", string(output)) - } - - if err != nil { - var exitError *exec.ExitError - if errors.As(err, &exitError) { - exitCode := exitError.ExitCode() - logrus.Debugf("Command exited with exit code %d", exitCode) - } else { - logrus.Debugln("Command failed without an exit code") - } - return "", trace.Wrap(err, "Command failed, see debug output for additional details") - } - - logrus.Debugln("Command exited successfully") - return string(output), nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/config.go b/build.assets/tooling/cmd/build-os-package-repos/config.go deleted file mode 100644 index df3357f2dee57..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/config.go +++ /dev/null @@ -1,282 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" -) - -const StableChannelFlagValue string = "stable" - -type LoggerConfig struct { - logLevel uint - logJSON bool -} - -func NewLoggerConfigWithFlagset(fs *flag.FlagSet) *LoggerConfig { - lc := &LoggerConfig{} - fs.UintVar(&lc.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose") - fs.BoolVar(&lc.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging") - - return lc -} - -func (lc *LoggerConfig) Check() error { - if err := lc.validateLogLevel(); err != nil { - return trace.Wrap(err, "failed to validate the log level flag") - } - - return nil -} - -func (lc *LoggerConfig) validateLogLevel() error { - if lc.logLevel > 6 { - return trace.BadParameter("the log-level flag should be between 0 and 6") - } - - return nil -} - -type S3Config struct { - bucketName string - localBucketPath string - maxConcurrentSyncs int -} - -func NewS3ConfigWithFlagset(fs *flag.FlagSet) *S3Config { - s3c := &S3Config{} - fs.StringVar(&s3c.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from") - fs.StringVar(&s3c.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to") - fs.IntVar(&s3c.maxConcurrentSyncs, "max-concurrent-syncs", 16, "The maximum number of S3 bucket syncs that may run in parallel (-1 for unlimited, 16 default)") - - return s3c -} - -func (s3c *S3Config) Check() error { - if err := s3c.validateBucketName(); err != nil { - return trace.Wrap(err, "failed to validate the bucket name flag") - } - if err := s3c.validateLocalBucketPath(); err != nil { - return trace.Wrap(err, "failed to validate the local bucket path flag") - } - if err := s3c.validateMaxConcurrentSyncs(); err != nil { - return trace.Wrap(err, "failed to validate the max concurrent syncs flag") - } - - return nil -} - -func (s3c *S3Config) validateBucketName() error { - if s3c.bucketName == "" { - return trace.BadParameter("the bucket flag should not be empty") - } - - return nil -} - -func (s3c *S3Config) validateLocalBucketPath() error { - if s3c.localBucketPath == "" { - return trace.BadParameter("the local-bucket-path flag should not be empty") - } - - if stat, err := os.Stat(s3c.localBucketPath); err == nil && !stat.IsDir() { - return trace.BadParameter("the local bucket path points to a file instead of a directory") - } - - return nil -} - -func (s3c *S3Config) validateMaxConcurrentSyncs() error { - if s3c.maxConcurrentSyncs < -1 { - return trace.BadParameter("the max-concurrent-syncs flag must be greater than -1") - } - - return nil -} - -// This type is common to all other config types -type Config struct { - *LoggerConfig - *S3Config - artifactPath string - artifactVersion string - printHelp bool - releaseChannel string -} - -func NewConfigWithFlagset(fs *flag.FlagSet) *Config { - c := &Config{} - c.LoggerConfig = NewLoggerConfigWithFlagset(fs) - c.S3Config = NewS3ConfigWithFlagset(fs) - - fs.StringVar(&c.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb or *.rpm files to add to the repos") - fs.StringVar(&c.artifactVersion, "artifact-version", "", "The version of the artifacts that will be added to the repos") - fs.Visit(func(f *flag.Flag) { - if f.Name == "-h" || f.Name == "--help" { - c.printHelp = true - } - }) - fs.StringVar(&c.releaseChannel, "release-channel", "", "The release channel of the repos that the artifacts should be added to") - - return c -} - -func (c *Config) Check() error { - if err := c.LoggerConfig.Check(); err != nil { - return trace.Wrap(err, "failed to validate logger config") - } - - if err := c.S3Config.Check(); err != nil { - return trace.Wrap(err, "failed to validate S3 config") - } - - if err := c.validateArtifactPath(); err != nil { - return trace.Wrap(err, "failed to validate the artifact path flag") - } - if err := c.validateArtifactVersion(); err != nil { - return trace.Wrap(err, "failed to validate the artifact version flag") - } - if err := c.validateReleaseChannel(); err != nil { - return trace.Wrap(err, "failed to validate the release channel flag") - } - - return nil -} - -func (c *Config) validateArtifactPath() error { - if c.artifactPath == "" { - return trace.BadParameter("the artifact-path flag should not be empty") - } - - if stat, err := os.Stat(c.artifactPath); os.IsNotExist(err) { - return trace.BadParameter("the artifact-path %q does not exist", c.artifactPath) - } else if !stat.IsDir() { - return trace.BadParameter("the artifact-path %q is not a directory", c.artifactPath) - } - - return nil -} - -func (c *Config) validateArtifactVersion() error { - if c.artifactVersion == "" { - return trace.BadParameter("the artifact-version flag should not be empty") - } - - if !semver.IsValid(c.artifactVersion) { - return trace.BadParameter("the artifact-version flag does not contain a valid semver version string") - } - - return nil -} - -func (c *Config) validateReleaseChannel() error { - if c.releaseChannel == "" { - return trace.BadParameter("the release-channel flag should not be empty") - } - - // Not sure what other channels we'd want to support, but they should be listed here - validReleaseChannels := []string{StableChannelFlagValue} - - for _, validReleaseChannel := range validReleaseChannels { - if c.releaseChannel == validReleaseChannel { - return nil - } - } - - return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ",")) -} - -// APT-specific config -type AptConfig struct { - *Config - aptlyPath string -} - -func NewAptConfigWithFlagSet(fs *flag.FlagSet) (*AptConfig, error) { - ac := &AptConfig{} - ac.Config = NewConfigWithFlagset(fs) - - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, trace.Wrap(err, "failed to get user's home directory path") - } - - fs.StringVar(&ac.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)") - - return ac, nil -} - -func (ac *AptConfig) validateAptlyPath() error { - if ac.aptlyPath == "" { - return trace.BadParameter("the aptly-root-dir flag should not be empty") - } - - return nil -} - -func (ac *AptConfig) Check() error { - if err := ac.Config.Check(); err != nil { - return trace.Wrap(err, "failed to validate common config") - } - - if err := ac.validateAptlyPath(); err != nil { - return trace.Wrap(err, "failed to validate the aptly-root-dir path flag") - } - - return nil -} - -// YUM-specific config -type YumConfig struct { - *Config - cacheDir string -} - -func NewYumConfigWithFlagSet(fs *flag.FlagSet) *YumConfig { - yc := &YumConfig{} - yc.Config = NewConfigWithFlagset(fs) - - fs.StringVar(&yc.cacheDir, "cache-dir", "/tmp/createrepo/cache", "The createrepo checksum caching directory (see https://linux.die.net/man/8/createrepo for details") - - return yc -} - -func (yc *YumConfig) validateCacheDir() error { - if yc.cacheDir == "" { - return trace.BadParameter("the cache-dir flag should not be empty") - } - - return nil -} - -func (yc *YumConfig) Check() error { - if err := yc.Config.Check(); err != nil { - return trace.Wrap(err, "failed to validate common config") - } - - if err := yc.validateCacheDir(); err != nil { - return trace.Wrap(err, "failed to validate the cache-dir path flag") - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/createrepo.go b/build.assets/tooling/cmd/build-os-package-repos/createrepo.go deleted file mode 100644 index 5c3a208926891..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/createrepo.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - "os/exec" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" -) - -type CreateRepo struct { - cacheDir string - binaryName string -} - -// Instantiates createrepo, ensuring all system requirements for performing createrepo operations -// have been met -func NewCreateRepo(cacheDir string) (*CreateRepo, error) { - cr := &CreateRepo{ - cacheDir: cacheDir, - // `createrepo_c` is the "new" (as in 9 years old) replacement for `createrepo` - // This can be replace with "createrepo" in the unlikely chance that there is - // a problem - binaryName: "createrepo_c", - } - - err := cr.ensureBinaryExists() - if err != nil { - return nil, trace.Wrap(err, "failed to ensure CreateRepo binary exists") - } - - // Ensure the cache dir exists - err = os.MkdirAll(cr.cacheDir, 0660) - if err != nil { - return nil, trace.Wrap(err, "failed to ensure %q exists", cr.cacheDir) - } - - return cr, nil -} - -func (cr *CreateRepo) ensureBinaryExists() error { - _, err := exec.LookPath(cr.binaryName) - if err != nil { - return trace.Wrap(err, "failed to verify that %q binary exists", cr.binaryName) - } - - return nil -} - -func (cr *CreateRepo) CreateOrUpdateRepo(repoPath string) error { - // --cachedir --update - logrus.Debugf("Updating repo metadata for repo at %q", repoPath) - - args := []string{ - "--cachedir", - cr.cacheDir, - "--update", - repoPath, - } - - _, err := BuildAndRunCommand(cr.binaryName, args...) - if err != nil { - return trace.Wrap(err, "createrepo create/update command failed on path %q with cache directory %q", repoPath, cr.cacheDir) - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/gpg.go b/build.assets/tooling/cmd/build-os-package-repos/gpg.go deleted file mode 100644 index 531649d4d45fc..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/gpg.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" -) - -type GPG struct{} - -// Instantiates GPG, ensuring all system requirements for using GPG are fulfilled -func NewGPG() (*GPG, error) { - g := &GPG{} - - err := g.ensureFirstRunHasOccurred() - if err != nil { - return nil, trace.Wrap(err, "failed to setup GPG") - } - - err = g.ensureSecretKeyExists() - if err != nil { - return nil, trace.Wrap(err, "failed to ensure a secret key exists") - } - - return g, nil -} - -// The first time GPG is run for a user with any "meaningful" arguments it will -// generate several files and log what it created to stdout. These logs can -// disrupt parsing of GPG command outputs, so we force it to happen here, once, -// rather than try and handle it on each GPG call. -func (*GPG) ensureFirstRunHasOccurred() error { - _, err := BuildAndRunCommand("gpg", "--fingerprint") - if err != nil { - return trace.Wrap(err, "failed to ensure GPG has been ran once") - } - - return nil -} - -func (*GPG) ensureSecretKeyExists() error { - output, err := BuildAndRunCommand("gpg", "--list-secret-keys", "--with-colons") - if err != nil { - return trace.Wrap(err, "failed to ensure GPG secret key exists") - } - - outputLineCount := strings.Count(output, "\n") - if outputLineCount < 1 { - return trace.Errorf("failed to find a GPG secret key") - } - - return nil -} - -// Creates a detached, armored signature for the provided file using the default GPG key -func (*GPG) SignFile(filePath string) error { - // While this could be done via a Go module, the x/crypto/openpgp library has been frozen - // and deprecated for almost 18 months. Others exist, but given the security implications of - // using a less reputable Go module I've decided to just call `gpg` via shell instead. - // Additionally this works and is just _so easy_ that it's probably not worth the effort to - // use another library that reinvents the wheel. - logrus.Debugf("Signing repo metadata at %q", filePath) - - // gpg --batch --yes --detach-sign --armor - _, err := BuildAndRunCommand("gpg", "--batch", "--yes", "--detach-sign", "--armor", filePath) - if err != nil { - return trace.Wrap(err, "failed to run GPG signing command on %q", filePath) - } - - return nil -} - -// Get the armored default public GPG key, ready to be written to a file -func (*GPG) GetPublicKey() (string, error) { - // For reference here is how another company formats their key: - // https://download.docker.com/linux/rhel/gpg - logrus.Debug("Attempting to get the default public GPG key") - - key, err := BuildAndRunCommand("gpg", "--export", "--armor", "--no-version") - if err != nil { - return "", trace.Wrap(err, "failed to export the default public GPG key") - } - - return key, nil -} - -func (g *GPG) WritePublicKeyToFile(filePath string) error { - logrus.Debugf("Writing the default armored public GPG key to %q", filePath) - - key, err := g.GetPublicKey() - if err != nil { - return trace.Wrap(err, "failed to retrieve public key") - } - - err = os.WriteFile(filePath, []byte(key), 0664) - if err != nil { - return trace.Wrap(err, "failed to write key to %q", filePath) - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/main.go b/build.assets/tooling/cmd/build-os-package-repos/main.go deleted file mode 100644 index 58f89f1dd1d16..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/main.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" -) - -func main() { - err := run() - if err != nil { - logrus.Fatal(err.Error()) - } -} - -func buildSubcommandRunners() ([]Runner, error) { - ar, err := NewAptRunner() - if err != nil { - return nil, trace.Wrap(err, "failed to instantiate new APT runner") - } - - yr, err := NewYumRunner() - if err != nil { - return nil, trace.Wrap(err, "failed to instantiate new YUM runner") - } - - // These should be sorted alphabetically by `Name()` - return []Runner{ - *ar, - *yr, - }, nil -} - -func run() error { - subcommands, err := buildSubcommandRunners() - if err != nil { - return trace.Wrap(err, "failed to build subcommand runners") - } - - // 2 = program name + subcommand - if len(os.Args) < 2 { - logHelp(subcommands) - return trace.Errorf("subcommand not provided") - } - - subcommandName := strings.ToLower(os.Args[1]) - for _, subcommand := range subcommands { - if strings.ToLower(subcommandName) != subcommand.Name() { - continue - } - - // 2 = program name + subcommand, skip them and get subcommand arguments - args := os.Args[2:] - err := subcommand.Init(args) - if err != nil { - return trace.Wrap(err, "failed to initialize runner for subcommand %q", subcommandName) - } - - setupLogger(subcommand.GetLoggerConfig()) - err = subcommand.Run() - if err != nil { - return trace.Wrap(err, "failed to run subcommand %q", subcommandName) - } - - return nil - } - - if subcommandName == "-h" { - logHelp(subcommands) - return nil - } - - logHelp(subcommands) - return trace.Errorf("no subcommands found matching %q", subcommandName) -} - -func logHelp(subcommands []Runner) { - executableName := os.Args[0] - fmt.Printf("%s - OS package repo builder/updater\n", executableName) - fmt.Println() - fmt.Println("Commands:") - fmt.Println() - for _, subcommand := range subcommands { - fmt.Printf("\t%s\t%s\n", subcommand.Name(), subcommand.Info()) - } - fmt.Println() - fmt.Printf("Use \"%s -h\" for more information about a command.\n", executableName) - fmt.Println() -} - -func setupLogger(config *LoggerConfig) { - if config.logJSON { - logrus.SetFormatter(&logrus.JSONFormatter{}) - } else { - logrus.SetFormatter(&logrus.TextFormatter{}) - } - logrus.SetOutput(os.Stdout) - logrus.SetLevel(logrus.Level(config.logLevel)) - logrus.Debugf("Setup logger with config: %+v", config) -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/repo.go b/build.assets/tooling/cmd/build-os-package-repos/repo.go deleted file mode 100644 index 5060c130269df..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/repo.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/gravitational/trace" -) - -type Repo struct { - os string - osVersion string - releaseChannel string - majorVersion string - publishedSourcePath string -} - -// Returns a unique name for the repo. -func (r *Repo) Name() string { - return fmt.Sprintf("%s-%s-%s-%s", r.os, r.osVersion, r.releaseChannel, r.majorVersion) -} - -func NewRepoFromName(name string) (*Repo, error) { - splitName := strings.Split(name, "-") - if len(splitName) != 4 { - return nil, trace.Errorf("the provided repo name %q is not a valid repo name", name) - } - - return &Repo{ - os: splitName[0], - osVersion: splitName[1], - releaseChannel: splitName[2], - majorVersion: splitName[3], - }, nil -} - -// Returns the APT component to be associated with all debs in the Aptly repo. -func (r *Repo) Component() string { - return fmt.Sprintf("%s/%s", r.releaseChannel, r.majorVersion) -} - -// Returns a string that identifies the specific OS and OS version the Aptly repo targets. -func (r *Repo) OSInfo() string { - return fmt.Sprintf("%s/%s", r.os, r.osVersion) -} - -// Returns true if the repo is a recreation of a published repo, false otherewise. -// If true, publishedSourcePath is a valid existing path on the filesystem. -func (r *Repo) WasCreatedFromPublishedSource() (bool, error) { - if r.publishedSourcePath == "" { - return false, nil - } - - _, err := os.Stat(r.publishedSourcePath) - if os.IsNotExist(err) { - return false, trace.Errorf("the published source path of repo %q was not empty (%q), but does not exist on disk", r.Name(), r.publishedSourcePath) - } - - return true, nil -} - -// Returns the absolute path to the published repo on disk that this repo was created from. -func (r *Repo) PublishedRepoAbsolutePath() (string, error) { - wasCreatedFromPublishedSource, err := r.WasCreatedFromPublishedSource() - if err != nil { - return "", trace.Wrap(err, "failed to verify if the repo %q is a recreation of an existing published repo", r.Name()) - } - - if !wasCreatedFromPublishedSource { - return "", trace.Errorf("repo %q was not created from a publish source and therefor has no published source path", r.Name()) - } - - // `///dists////` - return filepath.Join(r.publishedSourcePath, r.os, "dists", r.osVersion, r.releaseChannel, r.majorVersion), nil -} - -// Helper function that calls `Name()` on all repos in the provided list. -func RepoNames(repos []*Repo) []string { - repoNames := make([]string, len(repos)) - for i, repo := range repos { - repoNames[i] = repo.Name() - } - - return repoNames -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/runners.go b/build.assets/tooling/cmd/build-os-package-repos/runners.go deleted file mode 100644 index 9ecda55eea2c8..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/runners.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - - "github.com/gravitational/trace" -) - -// Pattern from https://www.digitalocean.com/community/tutorials/how-to-use-the-flag-package-in-go -type Runner interface { - Init([]string) error - Run() error - GetLoggerConfig() *LoggerConfig - Name() string - Info() string -} - -// APT implementation -type AptRunner struct { - flags *flag.FlagSet - config *AptConfig - supportedOSs map[string][]string -} - -func NewAptRunner() (*AptRunner, error) { - runner := &AptRunner{ - supportedOSs: map[string][]string{ - "debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details - "stretch", // 9 - "buster", // 10 - "bullseye", // 11 - "bookwork", // 12 - "trixie", // 13 - }, - "ubuntu": { // See https://wiki.ubuntu.com/Releases for details - "xenial", // 16.04 LTS - "yakkety", // 16.10 (EOL) - "zesty", // 17.04 (EOL) - "artful", // 17.10 (EOL) - "bionic", // 18.04 LTS - "cosmic", // 18.10 (EOL) - "disco", // 19.04 (EOL) - "eoan", // 19.10 (EOL) - "focal", // 20.04 LTS - "groovy", // 20.10 (EOL) - "hirsuite", // 21.04 (EOL) - "impish", // 21.10 (EOL) - "jammy", // 22.04 LTS - }, - }, - } - - runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) - config, err := NewAptConfigWithFlagSet(runner.flags) - if err != nil { - return nil, trace.Wrap(err, "failed to create a new APT config instance") - } - - runner.config = config - - return runner, nil -} - -func (ar AptRunner) Init(args []string) error { - err := ar.flags.Parse(args) - if err != nil { - return trace.Wrap(err, "failed to parse arguments") - } - - err = ar.config.Check() - if err != nil { - return trace.Wrap(err, "failed to validate APT config arguments") - } - - return nil -} - -func (ar AptRunner) Run() error { - if ar.config.printHelp { - ar.flags.Usage() - return nil - } - - art, err := NewAptRepoTool(ar.config, ar.supportedOSs) - if err != nil { - return trace.Wrap(err, "failed to create a new APT repo tool instance") - } - - err = art.Run() - if err != nil { - return trace.Wrap(err, "APT runner failed") - } - - return nil -} - -func (AptRunner) Name() string { - return "apt" -} - -func (AptRunner) Info() string { - return "builds APT repos" -} - -func (ar AptRunner) GetLoggerConfig() *LoggerConfig { - return ar.config.LoggerConfig -} - -// YUM implementation -type YumRunner struct { - flags *flag.FlagSet - config *YumConfig - supportedOSs map[string][]string -} - -func NewYumRunner() (*YumRunner, error) { - runner := &YumRunner{ - supportedOSs: map[string][]string{ - "rhel": { // See https://access.redhat.com/articles/3078 for details - "7", - "8", - "9", - }, - "centos": { // See https://endoflife.date/centos for details - "7", - "8", - "9", - }, - // "$releasever" is a hot mess for Amazon Linux. No good documentation on this outside of just running - // a container or EC2 instance and manually checking $releasever values - "amzn": { - // "latest" // 1, aka 2018.03.0.20201028.0 - "2", // 2, aka 2.0.20201111.0 - // "2022.0.20220531" // 2022 (new naming scheme, preview) aka 2022.0.20220531 - }, - }, - } - - runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) - runner.config = NewYumConfigWithFlagSet(runner.flags) - - return runner, nil -} - -func (yr YumRunner) Init(args []string) error { - err := yr.flags.Parse(args) - if err != nil { - return trace.Wrap(err, "failed to parse arguments") - } - - err = yr.config.Check() - if err != nil { - return trace.Wrap(err, "failed to validate YUM config arguments") - } - - return nil -} - -func (yr YumRunner) Run() error { - yrt, err := NewYumRepoTool(yr.config, yr.supportedOSs) - if err != nil { - return trace.Wrap(err, "failed to create a new YUM repo tool instance") - } - - err = yrt.Run() - if err != nil { - return trace.Wrap(err, "YUM runner failed") - } - - return nil -} - -func (YumRunner) Name() string { - return "yum" -} - -func (YumRunner) Info() string { - return "builds YUM repos" -} - -func (yr YumRunner) GetLoggerConfig() *LoggerConfig { - return yr.config.LoggerConfig -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/s3logger.go b/build.assets/tooling/cmd/build-os-package-repos/s3logger.go deleted file mode 100644 index 9229e8859dd6c..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/s3logger.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "github.com/sirupsen/logrus" -) - -// This maps the s3sync logging functions to logrus so that they match the -// rest of the logging framework's settings. -// Their docs outline the custom logger configuration here: -// https://github.com/seqsense/s3sync#sets-the-custom-logger -type s3logger struct{} - -// Maps the s3sync log function to logrus. -func (s3logger) Log(v ...interface{}) { - logrus.Debugln(v...) -} - -// Maps the s3sync logf function to logrus. -func (s3logger) Logf(format string, v ...interface{}) { - logrus.Debugf(format, v...) -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/s3manager.go b/build.assets/tooling/cmd/build-os-package-repos/s3manager.go deleted file mode 100644 index ccdd443c19933..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/s3manager.go +++ /dev/null @@ -1,511 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "fmt" - "io" - "io/fs" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/gravitational/trace" - "github.com/inhies/go-bytesize" - "github.com/seqsense/s3sync" - "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" -) - -type S3manager struct { - syncManager *s3sync.Manager - uploader *s3manager.Uploader - downloader *s3manager.Downloader - bucketLocalPath string - bucketName string - bucketURL *url.URL - maxConcurrentSyncs int - downloadedBytes int64 -} - -func NewS3Manager(config *S3Config) (*S3manager, error) { - // Right now the AWS session is only used by this manager, but if it ends - // up being needed elsewhere then it should probably be moved to an arg - awsSession, err := session.NewSession() - if err != nil { - return nil, trace.Wrap(err, "failed to create a new AWS session") - } - - syncManagerMaxConcurrentSyncs := config.maxConcurrentSyncs - if syncManagerMaxConcurrentSyncs < 0 { - // This isn't unlimited but due to the s3sync library's parallelism implementation - // this must be limited to a "reasonable" number - syncManagerMaxConcurrentSyncs = 128 - } - - s := &S3manager{ - bucketName: config.bucketName, - bucketURL: &url.URL{ - Scheme: "s3", - Host: config.bucketName, - }, - syncManager: s3sync.New(awsSession, s3sync.WithParallel(syncManagerMaxConcurrentSyncs)), - uploader: s3manager.NewUploader(awsSession), - downloader: s3manager.NewDownloader(awsSession), - maxConcurrentSyncs: config.maxConcurrentSyncs, - } - s.ChangeLocalBucketPath(config.localBucketPath) - - s3sync.SetLogger(&s3logger{}) - - return s, nil -} - -func (s *S3manager) ChangeLocalBucketPath(newBucketPath string) error { - s.bucketLocalPath = newBucketPath - - // Ensure the local bucket path exists as it will be needed by all functions - err := os.MkdirAll(s.bucketLocalPath, 0660) - if err != nil { - return trace.Wrap(err, "failed to ensure path %q exists", s.bucketLocalPath) - } - - return nil -} - -func (s *S3manager) DownloadExistingRepo() error { - err := deleteAllFilesInDirectory(s.bucketLocalPath) - if err != nil { - return trace.Wrap(err, "failed to remove all filesystem entries in %q", s.bucketLocalPath) - } - - downloadGroup := &errgroup.Group{} - downloadGroup.SetLimit(s.maxConcurrentSyncs) - linkMap := make(map[string]string) - - var continuationToken *string - for { - listObjResponse, err := s.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: &s.bucketName, - ContinuationToken: continuationToken, - }) - if err != nil { - return trace.Wrap(err, "failed to list objects for bucket %q", s.bucketName) - } - - for _, s3object := range listObjResponse.Contents { - s.processS3ObjectDownload(s3object, downloadGroup, &linkMap) - } - - continuationToken = listObjResponse.NextContinuationToken - if continuationToken == nil { - break - } - } - - // Even if an error has occurred we should wait to exit until all running syncs have - // completed, even if not successful - logrus.Info("Waiting for download to complete...") - err = downloadGroup.Wait() - if err != nil { - return trace.Wrap(err, "failed to perform S3 sync from remote bucket %q to local bucket %q", s.bucketName, s.bucketLocalPath) - } - - // Links must be created after their target exists - err = createLinks(linkMap) - if err != nil { - return trace.Wrap(err, "failed to create filesystem links for bucket %q", s.bucketName) - } - - logrus.Infof("Downloaded %s bytes", bytesize.New(float64(s.downloadedBytes))) - return nil -} - -func (s *S3manager) processS3ObjectDownload(s3object *s3.Object, downloadGroup *errgroup.Group, linkMap *map[string]string) { - downloadGroup.Go(func() error { - objectLink, err := s.getObjectLink(s3object) - if err != nil { - return trace.Wrap(err, "failed to get object link for key %q in bucket %q", *s3object.Key, s.bucketName) - } - - // If the link does not start with a '/' then it is not a filesystem link - if objectLink != nil && len(*objectLink) > 0 && (*objectLink)[0] == '/' { - localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) - linkTarget := filepath.Join(s.bucketLocalPath, *objectLink) - (*linkMap)[localObjectPath] = linkTarget - return nil - } - - err = s.downloadFile(s3object) - if err != nil { - return trace.Wrap(err, "failed to download S3 file %q from bucket %q", *s3object.Key, s.bucketName) - } - - return nil - }) -} - -func createLinks(linkMap map[string]string) error { - for file, target := range linkMap { - logrus.Infof("Creating a symlink from %q to %q", file, target) - err := os.MkdirAll(filepath.Dir(file), 0660) - if err != nil { - return trace.Wrap(err, "failed to create directory structure for %q", file) - } - - err = os.Symlink(target, file) - if err != nil { - return trace.Wrap(err, "failed to symlink %q to %q", file, target) - } - } - - return nil -} - -// This could potentially be made more efficient by running `os.RemoveAll` in a goroutine -// as random access on storage devices performs better at a higher queue depth -func deleteAllFilesInDirectory(dir string) error { - // Note that os.ReadDir does not follow/eval links which is important here - dirEntries, err := os.ReadDir(dir) - if err != nil { - return trace.Wrap(err, "failed to list directory entries for directory %q", dir) - } - - for _, dirEntry := range dirEntries { - dirEntryPath := filepath.Join(dir, dirEntry.Name()) - err = os.RemoveAll(dirEntryPath) - if err != nil { - return trace.Wrap(err, "failed to remove directory entry %q", dirEntryPath) - } - } - - return nil -} - -func (s *S3manager) getObjectLink(s3object *s3.Object) (*string, error) { - s3HeadObjectOutput, err := s.downloader.S3.HeadObject(&s3.HeadObjectInput{ - Bucket: &s.bucketName, - Key: s3object.Key, - // Probably unnecessary but this will cause an error to be thrown if somebody is - // modifying the object while this program is running - IfMatch: s3object.ETag, - IfUnmodifiedSince: s3object.LastModified, - }) - if err != nil { - return nil, trace.Wrap(err, "failed to retrieve metadata for key %q in bucket %q", *s3object.Key, s.bucketName) - } - - return s3HeadObjectOutput.WebsiteRedirectLocation, nil -} - -// s3sync has a bug when downloading a single file so this call reimplements s3sync's download -func (s *S3manager) downloadFile(s3object *s3.Object) error { - logrus.Infof("Downloading %q...", *s3object.Key) - localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) - - err := os.MkdirAll(filepath.Dir(localObjectPath), 0660) - if err != nil { - return trace.Wrap(err, "failed to create directory structure for %q", localObjectPath) - } - - fileWriter, err := os.Create(localObjectPath) - if err != nil { - return trace.Wrap(err, "failed to open %q for writing", localObjectPath) - } - defer fileWriter.Close() - - fileDownloadByteCount, err := s.downloader.Download(fileWriter, &s3.GetObjectInput{ - Bucket: aws.String(s.bucketName), - Key: aws.String(*s3object.Key), - }) - if err != nil { - return trace.Wrap(err, "failed to download object %q from bucket %q to local path %q", *s3object.Key, s.bucketName, localObjectPath) - } - - s.downloadedBytes += fileDownloadByteCount - - err = os.Chtimes(localObjectPath, *s3object.LastModified, *s3object.LastModified) - if err != nil { - return trace.Wrap(err, "failed to update the access and modification time on file %q to %v", localObjectPath, *s3object.LastModified) - } - - logrus.Infof("Download %q complete", *s3object.Key) - return nil -} - -func (s *S3manager) UploadBuiltRepo() error { - err := s.sync(false) - if err != nil { - return trace.Wrap(err, "failed to upload bucket") - } - - return nil -} - -func (s *S3manager) UploadBuiltRepoWithRedirects(extensionToMatch, relativeRedirectDir string) error { - uploadGroup := &errgroup.Group{} - uploadGroup.SetLimit(s.maxConcurrentSyncs) - - walkErr := filepath.WalkDir(s.bucketLocalPath, func(absPath string, info fs.DirEntry, err error) error { - logrus.Debugf("Starting on %q...", absPath) - - if err != nil { - return trace.Wrap(err, "failed to walk over directory %q on path %q", s.bucketLocalPath) - } - - syncFunc, err := s.syncGenericFsObject(absPath, info) - if err != nil { - return trace.Wrap(err, "failed to get syncing function for %q", absPath) - } - - uploadGroup.Go(syncFunc) - logrus.Debugf("Upload for %q queued", absPath) - return nil - }) - - // Even if an error has occurred we should wait to exit until all running syncs have - // completed, even if not successful - logrus.Info("Waiting for sync to complete...") - syncErr := uploadGroup.Wait() - // Future work: add upload logging information once - // https://github.com/seqsense/s3sync/commit/29b3fcb259293d80634cb3916e0f28467d017087 has been released - logrus.Info("Sync has completed") - - errs := make([]error, 0, 2) - if walkErr != nil { - errs = append(errs, trace.Wrap(walkErr, "failed to walk over entries in %q", s.bucketLocalPath)) - } - - if syncErr != nil { - errs = append(errs, trace.Wrap(syncErr, "failed to perform S3 sync from local bucket %q to remote bucket %q", s.bucketLocalPath, s.bucketName)) - } - - if len(errs) > 0 { - return trace.Wrap(trace.NewAggregate(errs...), "one or more erros occurred while uploading built repo %q", s.bucketLocalPath) - } - - return nil -} - -func (s *S3manager) syncGenericFsObject(absPath string, dirEntryInfo fs.DirEntry) (func() error, error) { - // Don't do anything with non-empty directories as they will be caught later by their contents - if dirEntryInfo.IsDir() { - f, err := s.buildSyncDirFunc(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to build directory syncing function to sync %q", absPath) - } - - return f, nil - } else - // If symbolic link - if dirEntryInfo.Type()&fs.ModeSymlink != 0 { - f, err := s.buildSyncSymbolicLinkFunc(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to build symbolic link file syncing function to sync %q", absPath) - } - - return f, nil - } - - // sync a single file or directory - f, err := s.buildSyncSingleFsEntryFunc(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) - } - - return f, nil -} - -func (s *S3manager) buildSyncDirFunc(absPath string) (func() error, error) { - isDirEmpty, err := isDirectoryEmpty(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to determine if directory %q is empty", absPath) - } - - if !isDirEmpty { - logrus.Debug("Skipping non-empty directory") - return func() error { return nil }, nil - } - - // If the directory has no contents, call sync normally which will create the directory remotely if not exists - f, err := s.buildSyncSingleFsEntryFunc(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) - } - - return f, nil -} - -func (s *S3manager) buildSyncSymbolicLinkFunc(absPath string) (func() error, error) { - actualFilePath, err := filepath.EvalSymlinks(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to follow symlink for path %q", absPath) - } - - isInBucket, err := isPathChildOfAnother(s.bucketLocalPath, actualFilePath) - if err != nil { - return nil, trace.Wrap(err, "failed to determine if %q is a child of %q", actualFilePath, s.bucketLocalPath) - } - - if isInBucket { - // This will re-upload every redirect file ever created. Implementing "sync" functionality would - // require significantly more engineering effort and this cost is low so this shouldn't be a - // problem. - return func() error { - err := s.UploadRedirectFile(absPath, actualFilePath) - if err != nil { - return trace.Wrap(err, "failed to upload a redirect file to S3 for %q targeting %q", absPath, actualFilePath) - } - - return nil - }, nil - } - - // If not in bucket, call sync normally which will follow the symlink to the actual file and upload it - f, err := s.buildSyncSingleFsEntryFunc(absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) - } - - return f, nil -} - -func (s *S3manager) buildSyncSingleFsEntryFunc(absPath string) (func() error, error) { - relPath, err := filepath.Rel(s.bucketLocalPath, absPath) - if err != nil { - return nil, trace.Wrap(err, "failed to get %q relative to %q", absPath, s.bucketLocalPath) - } - - remoteURL := getURLWithPath(*s.bucketURL, relPath) - return func() error { - err := s.syncManager.Sync(absPath, remoteURL) - if err != nil { - return trace.Wrap(err, "failed to sync from %q to %q", absPath, remoteURL) - } - - return nil - }, nil -} - -func getURLWithPath(baseURL url.URL, path string) string { - // Because this function is pass-by-value it should not modify `baseUrl`, where doing this directly on the - // provided parameter would modify it - baseURL.Path = path - return baseURL.String() -} - -func isPathChildOfAnother(baseAbsPath string, testAbsPath string) (bool, error) { - // General implementation from https://stackoverflow.com/questions/28024731/check-if-given-path-is-a-subdirectory-of-another-in-golang - relPath, err := filepath.Rel(baseAbsPath, testAbsPath) - if err != nil { - return false, trace.Wrap(err, "failed to get the path of %q relative to %q", testAbsPath, baseAbsPath) - } - - return !strings.HasPrefix(relPath, fmt.Sprintf("..%c", os.PathSeparator)) && relPath != "..", nil -} - -func (s *S3manager) UploadRedirectFile(localAbsSrcPath, localAbsRemoteTargetPath string) error { - relSrcPath, err := filepath.Rel(s.bucketLocalPath, localAbsSrcPath) - if err != nil { - return trace.Wrap(err, "failed to get %q relative to %q", localAbsSrcPath, s.bucketLocalPath) - } - - relTargetPath, err := filepath.Rel(s.bucketLocalPath, localAbsRemoteTargetPath) - if err != nil { - return trace.Wrap(err, "failed to get %q relative to %q", localAbsRemoteTargetPath, s.bucketLocalPath) - } - - logrus.Infof("Creating a redirect file from %q to %q", relSrcPath, relTargetPath) - // S3 requires a prepended "/" to inform the redirect metadata that the target is another S3 object - // in the same bucket - s3TargetPath := filepath.Join("/", relTargetPath) - // Upload an empty file that when requested will redirect to the real one - _, err = s.uploader.Upload(&s3manager.UploadInput{ - Bucket: &s.bucketName, - Key: &relSrcPath, - Body: bytes.NewReader([]byte{}), - WebsiteRedirectLocation: &s3TargetPath, - }) - if err != nil { - return trace.Wrap(err, "failed to upload an empty redirect file to %q in bucket %q", relSrcPath, s.bucketName) - } - - return nil -} - -func (s *S3manager) UploadRedirectURL(remoteAbsSourcePath, targetURL string) error { - logrus.Infof("Creating redirect from %q to %q", remoteAbsSourcePath, targetURL) - - _, err := s.uploader.Upload(&s3manager.UploadInput{ - Bucket: &s.bucketName, - Key: &remoteAbsSourcePath, - Body: bytes.NewReader([]byte{}), - WebsiteRedirectLocation: &targetURL, - }) - - if err != nil { - return trace.Wrap(err, "failed to upload URL redirect file targeting %q to %q", targetURL, remoteAbsSourcePath) - } - - return nil -} - -func isDirectoryEmpty(dirPath string) (bool, error) { - // Pulled from https://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty - f, err := os.Open(dirPath) - if err != nil { - return false, trace.Wrap(err, "failed to open directory %q", dirPath) - } - defer f.Close() - - _, err = f.Readdirnames(1) - if err == io.EOF { - return true, nil - } - - if err != nil { - return false, trace.Wrap(err, "failed to read the name of directories in %q", dirPath) - } - - return false, nil -} - -func (s *S3manager) sync(download bool) error { - var src, dest string - if download { - src = s.bucketURL.String() - dest = s.bucketLocalPath - } else { - src = s.bucketLocalPath - dest = s.bucketURL.String() - } - - logrus.Infof("Performing S3 sync from %q to %q...", src, dest) - err := s.syncManager.Sync(src, dest) - if err != nil { - return trace.Wrap(err, "failed to sync %q to %q", src, dest) - } - logrus.Infoln("S3 sync complete") - - return nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh b/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh deleted file mode 100755 index 813c4e53d7a23..0000000000000 --- a/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2016,SC1004,SC2174,SC2155 - -set -xeu - -# These must be set for the script to run -: "$AWS_ACCESS_KEY_ID" -: "$AWS_SECRET_ACCESS_KEY" -: "$AWS_SESSION_TOKEN" - -ART_VERSION_TAG="8.3.15" -ARTIFACT_PATH="/go/artifacts" -CACHE_DIR="/mnt/createrepo_cache" -GNUPGHOME="/tmpfs/gnupg" -REPO_S3_BUCKET="fred-test1" -BUCKET_CACHE_PATH="/mnt/bucket" -export AWS_REGION="us-west-2" - -: ' -Run command: -docker run \ - --rm -it \ - -v "$(git rev-parse --show-toplevel)":/go/src/github.com/gravitational/teleport/ \ - -v "$HOME/.aws":"/root/.aws" \ - -e AWS_PROFILE="$AWS_PROFILE" \ - -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ - -e DEBIAN_FRONTEND="noninteractive" \ - golang:1.18.4-bullseye /go/src/github.com/gravitational/teleport/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh -' - -# Download the artifacts -apt update -apt install -y wget -mkdir -pv "$ARTIFACT_PATH" -cd "$ARTIFACT_PATH" -wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.x86_64.rpm" -wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm64.rpm" -wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.i386.rpm" -wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm.rpm" - -apt install -y createrepo-c gnupg -mkdir -pv "$CACHE_DIR" -mkdir -pv -m0700 "$GNUPGHOME" -chown -R root:root "$GNUPGHOME" -export GPG_TTY=$(tty) -gpg --batch --gen-key <" + since.Format(time.RFC3339), } - runIDs := make(RunIDSet) + allRuns := make([]*github.WorkflowRun, 0) for { runs, resp, err := actions.ListWorkflowRunsByFileName(ctx, owner, repo, path, &listOptions) @@ -81,9 +80,7 @@ func ListWorkflowRuns(ctx context.Context, actions WorkflowRuns, owner, repo, pa return nil, trace.Wrap(err, "Failed to fetch runs") } - for _, r := range runs.WorkflowRuns { - runIDs.Insert(r.GetID()) - } + allRuns = append(allRuns, runs.WorkflowRuns...) if resp.NextPage == 0 { break @@ -92,6 +89,22 @@ func ListWorkflowRuns(ctx context.Context, actions WorkflowRuns, owner, repo, pa listOptions.Page = resp.NextPage } + return allRuns, nil +} + +// ListWorkflowRunIDs returns a set of RunIDs, representing the set of all for +// workflow runs created since the supplied start time. +func ListWorkflowRunIDs(ctx context.Context, actions WorkflowRuns, owner, repo, path, branch string, since time.Time) (RunIDSet, error) { + workflowRuns, err := ListWorkflowRuns(ctx, actions, owner, repo, path, branch, since) + if err != nil { + return nil, trace.Wrap(err, "failed to get a list of workflow runs") + } + + runIDs := make(RunIDSet, len(workflowRuns)) + for _, workflowRun := range workflowRuns { + runIDs.Insert(workflowRun.GetID()) + } + return runIDs, nil } @@ -126,7 +139,7 @@ func ListWorkflowJobs(ctx context.Context, lister WorkflowJobLister, owner, repo // WaitForRun blocks until the specified workflow run completes, and returns the overall // workflow status. -func WaitForRun(ctx context.Context, actions WorkflowRuns, owner, repo, path, ref string, runID int64) (string, error) { +func WaitForRun(ctx context.Context, actions WorkflowRuns, owner, repo, path string, runID int64) (string, error) { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() diff --git a/dronegen/apt.go b/dronegen/apt.go deleted file mode 100644 index e59b8ea3706fd..0000000000000 --- a/dronegen/apt.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 Gravitational, Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import "path" - -// This function calls the build-apt-repos tool which handles the APT portion of RFD 0058. -func promoteAptPipeline() pipeline { - return getAptPipelineBuilder().buildPromoteOsPackagePipeline() -} - -func migrateAptPipeline(triggerBranch string, migrationVersions []string) pipeline { - return getAptPipelineBuilder().buildMigrateOsPackagePipeline(triggerBranch, migrationVersions) -} - -func getAptPipelineBuilder() *OsPackageToolPipelineBuilder { - optpb := NewOsPackageToolPipelineBuilder( - "drone-s3-aptrepo-pvc", - "deb", - "apt", - NewRepoBucketSecrets( - "APT_REPO_NEW_AWS_S3_BUCKET", - "APT_REPO_NEW_AWS_ACCESS_KEY_ID", - "APT_REPO_NEW_AWS_SECRET_ACCESS_KEY", - "APT_REPO_NEW_AWS_ROLE", - ), - ) - - optpb.environmentVars["APTLY_ROOT_DIR"] = value{ - raw: path.Join(optpb.pvcMountPoint, "aptly"), - } - - optpb.requiredPackages = []string{ - "aptly", - } - - optpb.extraArgs = []string{ - "-aptly-root-dir \"$APTLY_ROOT_DIR\"", - } - - return optpb -} diff --git a/dronegen/common.go b/dronegen/common.go index 2039052aa63e2..afdf8ea3fb465 100644 --- a/dronegen/common.go +++ b/dronegen/common.go @@ -19,7 +19,6 @@ import ( "fmt" "log" "os/exec" - "path" "strings" ) @@ -337,24 +336,6 @@ func cloneRepoStep(clonePath, commit string) step { } } -func verifyNotPrereleaseStep() step { - clonePath := "/tmp/repo" - commands := []string{ - "apk add git", - } - commands = append(commands, cloneRepoCommands(clonePath, "${DRONE_TAG}")...) - commands = append(commands, - fmt.Sprintf("cd %q", path.Join(clonePath, "build.assets", "tooling")), - "go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not continuing promotion for ${DRONE_TAG}' && exit 78)", - ) - - return step{ - Name: "Check if tag is prerelease", - Image: fmt.Sprintf("golang:%s-alpine", GoVersion), - Commands: commands, - } -} - func sliceSelect[T, V any](slice []T, selector func(T) V) []V { selectedValues := make([]V, len(slice)) for i, entry := range slice { diff --git a/dronegen/gha.go b/dronegen/gha.go index af9082d897e5f..f54cad472575d 100644 --- a/dronegen/gha.go +++ b/dronegen/gha.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "path" "sort" "strings" "time" @@ -23,75 +24,115 @@ import ( "golang.org/x/exp/maps" ) +type ghaWorkflow struct { + name string + stepName string + srcRefVar string + ref string + timeout time.Duration + slackOnError bool + shouldTagWorkflow bool + seriesRun bool + inputs map[string]string +} + type ghaBuildType struct { buildType trigger pipelineName string - ghaWorkflow string - srcRefVar string - workflowRef string - timeout time.Duration - slackOnError bool + checkoutPath string dependsOn []string - inputs map[string]string + workflows []ghaWorkflow +} + +func ghaBuildPipeline(ghaBuild ghaBuildType) pipeline { + return ghaMultiBuildPipeline(nil, ghaBuild) } -func ghaBuildPipeline(b ghaBuildType) pipeline { - p := newKubePipeline(b.pipelineName) - p.Trigger = b.trigger +// ghaMultiBuildPipeline returns a pipeline with multiple supported workflow call steps +func ghaMultiBuildPipeline(setupSteps []step, ghaBuild ghaBuildType) pipeline { + p := newKubePipeline(ghaBuild.pipelineName) + p.Trigger = ghaBuild.trigger p.Workspace = workspace{Path: "/go"} - p.DependsOn = append(p.DependsOn, b.dependsOn...) + p.DependsOn = append(p.DependsOn, ghaBuild.dependsOn...) + + checkoutPath := ghaBuild.checkoutPath + if checkoutPath == "" { + checkoutPath = "/go/src/github.com/gravitational/teleport" + } + + p.Steps = []step{ + { + Name: "Check out code", + Image: "docker:git", + Pull: "if-not-exists", + Environment: map[string]value{ + "GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"}, + }, + Commands: pushCheckoutCommandsWithPath(ghaBuild.buildType, checkoutPath), + }, + } + + p.Steps = append(p.Steps, setupSteps...) + + for _, workflow := range ghaBuild.workflows { + p.Steps = append(p.Steps, buildGHAWorkflowCallStep(workflow, checkoutPath)) + + if workflow.slackOnError { + p.Steps = append(p.Steps, sendErrorToSlackStep()) + } + } + + return p +} +func buildGHAWorkflowCallStep(workflow ghaWorkflow, checkoutPath string) step { var cmd strings.Builder cmd.WriteString(`go run ./cmd/gh-trigger-workflow `) cmd.WriteString(`-owner ${DRONE_REPO_OWNER} `) cmd.WriteString(`-repo teleport.e `) - cmd.WriteString(`-tag-workflow `) - fmt.Fprintf(&cmd, `-timeout %s `, b.timeout.String()) - fmt.Fprintf(&cmd, `-workflow %s `, b.ghaWorkflow) - fmt.Fprintf(&cmd, `-workflow-ref=%s `, b.workflowRef) + + if workflow.shouldTagWorkflow { + cmd.WriteString(`-tag-workflow `) + } + + if workflow.seriesRun { + cmd.WriteString(`-series-run `) + } + + fmt.Fprintf(&cmd, `-timeout %s `, workflow.timeout.String()) + fmt.Fprintf(&cmd, `-workflow %s `, workflow.name) + fmt.Fprintf(&cmd, `-workflow-ref=%s `, workflow.ref) // If we don't need to build teleport... - if b.srcRefVar != "" { + if workflow.srcRefVar != "" { cmd.WriteString(`-input oss-teleport-repo=${DRONE_REPO} `) - fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, b.srcRefVar) + fmt.Fprintf(&cmd, `-input oss-teleport-ref=${%s} `, workflow.srcRefVar) } // Sort inputs so the are output in a consistent order to avoid // spurious changes in the generated drone config. - keys := maps.Keys(b.inputs) + keys := maps.Keys(workflow.inputs) sort.Strings(keys) for _, k := range keys { - fmt.Fprintf(&cmd, `-input "%s=%s" `, k, b.inputs[k]) + fmt.Fprintf(&cmd, `-input "%s=%s" `, k, workflow.inputs[k]) } - p.Steps = []step{ - { - Name: "Check out code", - Image: "docker:git", - Pull: "if-not-exists", - Environment: map[string]value{ - "GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"}, - }, - Commands: pushCheckoutCommands(b.buildType), - }, - { - Name: "Delegate build to GitHub", - Image: fmt.Sprintf("golang:%s-alpine", GoVersion), - Pull: "if-not-exists", - Environment: map[string]value{ - "GHA_APP_KEY": {fromSecret: "GITHUB_WORKFLOW_APP_PRIVATE_KEY"}, - }, - Commands: []string{ - `cd "/go/src/github.com/gravitational/teleport/build.assets/tooling"`, - cmd.String(), - }, - }, + stepName := workflow.stepName + if stepName == "" { + stepName = "Delegate build to GitHub" } - if b.slackOnError { - p.Steps = append(p.Steps, sendErrorToSlackStep()) + return step{ + Name: stepName, + Image: fmt.Sprintf("golang:%s-alpine", GoVersion), + Pull: "if-not-exists", + Environment: map[string]value{ + "GHA_APP_KEY": {fromSecret: "GITHUB_WORKFLOW_APP_PRIVATE_KEY"}, + }, + Commands: []string{ + fmt.Sprintf(`cd %q`, path.Join(checkoutPath, "build.assets", "tooling")), + cmd.String(), + }, } - - return p } diff --git a/dronegen/mac_gha.go b/dronegen/mac_gha.go index 9c886a20491b7..2359c04a0d5d0 100644 --- a/dronegen/mac_gha.go +++ b/dronegen/mac_gha.go @@ -28,14 +28,19 @@ func darwinTagPipelineGHA() pipeline { buildType: buildType{os: "darwin", arch: "amd64"}, trigger: triggerTag, pipelineName: "build-darwin-amd64", - ghaWorkflow: "release-mac-amd64.yaml", - srcRefVar: "DRONE_TAG", - workflowRef: "${DRONE_TAG}", - timeout: 60 * time.Minute, - slackOnError: true, - inputs: map[string]string{ - "release-artifacts": "true", - "build-packages": "true", + workflows: []ghaWorkflow{ + { + name: "release-mac-amd64.yaml", + srcRefVar: "DRONE_TAG", + ref: "${DRONE_TAG}", + timeout: 60 * time.Minute, + slackOnError: true, + shouldTagWorkflow: true, + inputs: map[string]string{ + "release-artifacts": "true", + "build-packages": "true", + }, + }, }, } return ghaBuildPipeline(bt) @@ -51,14 +56,19 @@ func darwinPushPipelineGHA() pipeline { buildType: buildType{os: "darwin", arch: "amd64"}, trigger: triggerPush, pipelineName: "push-build-darwin-amd64", - ghaWorkflow: "release-mac-amd64.yaml", - srcRefVar: "DRONE_COMMIT", - workflowRef: "${DRONE_BRANCH}", - timeout: 60 * time.Minute, - slackOnError: true, - inputs: map[string]string{ - "release-artifacts": "false", - "build-packages": "false", + workflows: []ghaWorkflow{ + { + name: "release-mac-amd64.yaml", + srcRefVar: "DRONE_COMMIT", + ref: "${DRONE_BRANCH}", + timeout: 60 * time.Minute, + slackOnError: true, + shouldTagWorkflow: true, + inputs: map[string]string{ + "release-artifacts": "false", + "build-packages": "false", + }, + }, }, } return ghaBuildPipeline(bt) diff --git a/dronegen/main.go b/dronegen/main.go index 5ce4e6998ad31..875c71cf827a2 100644 --- a/dronegen/main.go +++ b/dronegen/main.go @@ -32,7 +32,6 @@ func main() { pipelines = append(pipelines, pushPipelines()...) pipelines = append(pipelines, tagPipelines()...) pipelines = append(pipelines, cronPipelines()...) - pipelines = append(pipelines, buildOsRepoPipelines()...) pipelines = append(pipelines, promoteBuildPipelines()...) pipelines = append(pipelines, updateDocsPipeline()) pipelines = append(pipelines, buildboxPipeline()) diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go index 1775b43035254..f50c5b71fdf29 100644 --- a/dronegen/os_repos.go +++ b/dronegen/os_repos.go @@ -17,442 +17,84 @@ package main import ( "fmt" "path" - "strings" + "time" ) -func buildOsRepoPipelines() []pipeline { - pipelines := promoteBuildOsRepoPipelines() - pipelines = append(pipelines, artifactMigrationPipeline()...) - - return pipelines -} - -func promoteBuildOsRepoPipelines() []pipeline { - aptPipeline := promoteAptPipeline() - yumPipeline := promoteYumPipeline() - return []pipeline{ - aptPipeline, - yumPipeline, - } -} - -// Used for one-off migrations of older versions. -// Use cases include: -// - We want to support another OS while providing backwards compatibility -// - We want to support another OS version while providing backwards compatibility -// - A customer wants to be able to install an older version via APT/YUM even if we -// no longer support it -// - RPM migrations after new YUM pipeline is done -func artifactMigrationPipeline() []pipeline { - migrationVersions := []string{ - // These versions were migrated as a part of the new `promoteAptPipeline` - // "v6.2.31", - // "v7.3.17", - // "v7.3.18", - // "v7.3.19", - // "v7.3.20", - // "v7.3.21", - // "v7.3.23", - // "v8.3.3", - // "v8.3.4", - // "v8.3.5", - // "v8.3.6", - // "v8.3.7", - // "v8.3.8", - // "v8.3.9", - // "v8.3.10", - // "v8.3.11", - // "v8.3.12", - // "v8.3.14", - // "v8.3.15", - // "v8.3.16", - // "v9.0.0", - // "v9.0.1", - // "v9.0.2", - // "v9.0.3", - // "v9.0.4", - // "v9.1.0", - // "v9.1.1", - // "v9.1.2", - // "v9.1.3", - // "v9.2.0", - // "v9.2.1", - // "v9.2.2", - // "v9.2.3", - // "v9.2.4", - // "v9.3.0", - // "v9.3.2", - // "v9.3.4", - // "v9.3.5", - // "v9.3.6", - // "v9.3.7", - // "v9.3.9", - // "v9.3.10", - // "v9.3.12", - // "v9.3.13", - // "v9.3.14", - // "v10.0.0", - // "v10.0.1", - // "v10.0.2", - // "v10.1.2", - // "v10.1.4", - } - // Pushing to this branch will trigger the listed versions to be migrated. Typically this should be - // the branch that these changes are being committed to. - migrationBranch := "" // "rfd/0058-package-distribution" - - aptPipeline := migrateAptPipeline(migrationBranch, migrationVersions) - yumPipeline := migrateYumPipeline(migrationBranch, migrationVersions) - return []pipeline{ - aptPipeline, - yumPipeline, - } -} - -type RepoBucketSecrets struct { - awsRoleSettings - bucketName value +type osPackageDeployment struct { + versionChannel string + packageNameFilter string + packageToTest string + displayName string } -func NewRepoBucketSecrets(bucketName, accessKeyID, secretAccessKey, role string) *RepoBucketSecrets { - return &RepoBucketSecrets{ - awsRoleSettings: awsRoleSettings{ - awsAccessKeyID: value{fromSecret: accessKeyID}, - awsSecretAccessKey: value{fromSecret: secretAccessKey}, - role: value{fromSecret: role}, - }, - bucketName: value{fromSecret: bucketName}, - } -} - -type OsPackageToolPipelineBuilder struct { - clameName string - packageType string - packageManagerName string - volumeName string - pipelineNameSuffix string - artifactPath string - pvcMountPoint string - bucketSecrets *RepoBucketSecrets - extraArgs []string - requiredPackages []string - setupCommands []string - environmentVars map[string]value -} - -// This function configures the build tool with it's requirements and sensible defaults. -// If additional configuration required then the returned struct should be modified prior -// to calling "build" functions on it. -func NewOsPackageToolPipelineBuilder(claimName, packageType, packageManagerName string, bucketSecrets *RepoBucketSecrets) *OsPackageToolPipelineBuilder { - optpb := &OsPackageToolPipelineBuilder{ - clameName: claimName, - packageType: packageType, - packageManagerName: packageManagerName, - bucketSecrets: bucketSecrets, - extraArgs: []string{}, - setupCommands: []string{}, - requiredPackages: []string{}, - volumeName: fmt.Sprintf("%s-persistence", packageManagerName), - pipelineNameSuffix: fmt.Sprintf("%s-new-repos", packageManagerName), - artifactPath: "/go/artifacts", - pvcMountPoint: "/mnt", - } - - optpb.environmentVars = map[string]value{ - "REPO_S3_BUCKET": optpb.bucketSecrets.bucketName, - "AWS_REGION": { - raw: "us-west-2", - }, - "BUCKET_CACHE_PATH": { - // If we need to cache the bucket on the PVC for some reason in the future - // uncomment this line - // raw: path.Join(pvcMountPoint, "bucket-cache"), - raw: "/tmp/bucket", - }, - "ARTIFACT_PATH": { - raw: optpb.artifactPath, - }, - "GNUPGHOME": { - raw: "/tmpfs/gnupg", - }, - "GPG_RPM_SIGNING_ARCHIVE": { - fromSecret: "GPG_RPM_SIGNING_ARCHIVE", - }, - "DEBIAN_FRONTEND": { - raw: "noninteractive", +func promoteBuildOsRepoPipeline() pipeline { + packageDeployments := []osPackageDeployment{ + // Normal release pipelines + { + versionChannel: "${DRONE_TAG}", + packageNameFilter: `$($DRONE_REPO_PRIVATE && echo "*ent*" || echo "")`, + packageToTest: "teleport-ent", + displayName: "Teleport", }, } - return optpb + return buildPromoteOsPackagePipelines(packageDeployments) } -func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipeline { - pipelineName := fmt.Sprintf("publish-%s", optpb.pipelineNameSuffix) - checkoutPath := "/go/src/github.com/gravitational/teleport" - commitName := "${DRONE_TAG}" +func buildPromoteOsPackagePipelines(packageDeployments []osPackageDeployment) pipeline { + releaseEnvironmentFilePath := "/go/vars/release-environment.txt" + clonePath := "/go/src/github.com/gravitational/teleport" - p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutPath, commitName) - p.Trigger = triggerPromote - p.Trigger.Repo.Include = []string{ - "gravitational/teleport", - "gravitational/teleport-private", + ghaBuild := ghaBuildType{ + trigger: triggerPromote, + pipelineName: "publish-os-package-repos", + checkoutPath: clonePath, + workflows: buildWorkflows(releaseEnvironmentFilePath, packageDeployments), } - setupSteps := []step{ - verifyTaggedStep(), - cloneRepoStep(checkoutPath, commitName), - } - - setupStepNames := make([]string, 0, len(setupSteps)) - for _, setupStep := range setupSteps { - setupStepNames = append(setupStepNames, setupStep.Name) - } - - versionSteps := optpb.getDroneTagVersionSteps(checkoutPath) - for i := range versionSteps { - versionStep := &versionSteps[i] - if versionStep.DependsOn == nil { - versionStep.DependsOn = setupStepNames - continue - } - - versionStep.DependsOn = append(versionStep.DependsOn, setupStepNames...) - } - - p.Steps = append(setupSteps, versionSteps...) - - return p -} - -func (optpb *OsPackageToolPipelineBuilder) buildMigrateOsPackagePipeline(triggerBranch string, migrationVersions []string) pipeline { - pipelineName := fmt.Sprintf("migrate-%s", optpb.pipelineNameSuffix) - checkoutPath := "/go/src/github.com/gravitational/teleport" - // DRONE_TAG is not available outside of promotion pipelines and will cause drone to fail with a - // "migrate-apt-new-repos: bad substitution" error if used here - commitName := "${DRONE_COMMIT}" - - // If migrations are not configured then don't run - if triggerBranch == "" || len(migrationVersions) == 0 { - return buildNeverTriggerPipeline(pipelineName) - } - - p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutPath, commitName) - p.Trigger = trigger{ - Repo: triggerRef{Include: []string{"gravitational/teleport"}}, - Event: triggerRef{Include: []string{"push"}}, - Branch: triggerRef{Include: []string{triggerBranch}}, - } - - for _, migrationVersion := range migrationVersions { - // Not enabling parallelism here so that multiple migrations don't run at once - p.Steps = append(p.Steps, optpb.getVersionSteps(checkoutPath, migrationVersion, false)...) - } - - setStepResourceLimits(p.Steps) - - return p -} - -// Builds a pipeline that is syntactically correct but should never trigger to create -// a placeholder pipeline -func buildNeverTriggerPipeline(pipelineName string) pipeline { - p := newKubePipeline(pipelineName) - p.Trigger = trigger{ - Event: triggerRef{Include: []string{"custom"}}, - Repo: triggerRef{Include: []string{"non-existent-repository"}}, - Branch: triggerRef{Include: []string{"non-existent-branch"}}, - } - - p.Steps = []step{ { - Name: "Placeholder", - Image: "alpine:latest", + Name: "Determine if release should go to development or production", + Image: fmt.Sprintf("golang:%s-alpine", GoVersion), Commands: []string{ - "echo \"This command, step, and pipeline never runs\"", - }, - }, - } - - return p -} - -// Functions that use this method should add at least: -// * a Trigger -// * Steps for checkout -func (optpb *OsPackageToolPipelineBuilder) buildBaseOsPackagePipeline(pipelineName, checkoutPath, commit string) pipeline { - p := newKubePipeline(pipelineName) - p.Workspace = workspace{Path: "/go"} - p.Volumes = []volume{ - { - Name: optpb.volumeName, - Claim: &volumeClaim{ - Name: optpb.clameName, + fmt.Sprintf("cd %q", path.Join(clonePath, "build.assets", "tooling")), + fmt.Sprintf("mkdir -pv %q", path.Dir(releaseEnvironmentFilePath)), + fmt.Sprintf(`(go run ./cmd/check -tag ${DRONE_TAG} -check prerelease && echo "promote" || echo "build") > %q`, releaseEnvironmentFilePath), }, }, - volumeTmpfs, - volumeAwsConfig, } - p.Steps = []step{cloneRepoStep(checkoutPath, commit)} - setStepResourceLimits(p.Steps) - - return p -} - -func setStepResourceLimits(steps []step) { - // Not currently supported - // for i := range steps { - // step := &steps[i] - // if step.Resources == nil { - // step.Resources = &containerResources{} - // } - - // if step.Resources.Requests == nil { - // step.Resources.Requests = &resourceSet{} - // } - - // step.Resources.Requests.Cpu = 100 - // step.Resources.Requests.Memory = (*resourceQuantity)(resource.NewQuantity(100*1024*1024, resource.BinarySI)) - // } -} -func (optpb *OsPackageToolPipelineBuilder) getDroneTagVersionSteps(codePath string) []step { - return optpb.getVersionSteps(codePath, "${DRONE_TAG}", true) + return ghaMultiBuildPipeline(setupSteps, ghaBuild) } -// Version should start with a 'v', i.e. v1.2.3 or v9.0.1, or should be an environment var -// i.e. ${DRONE_TAG} -func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version string, enableParallelism bool) []step { - var bucketFolder string - switch version[0:1] { - // If environment var - case "$": - // Remove the 'v' at runtime as the value isn't known at compile time - // This will change "${SOME_VAR}" to "${SOME_VAR##v}". `version` isn't actually - // an environment variable - it's a Drone substitution variable. See - // https://docs.drone.io/pipeline/environment/substitution/ for details. - bucketFolder = fmt.Sprintf("%s##v}", version[:len(version)-1]) - // If static string - case "v": - // Remove the 'v' at compile time as the value is known then - bucketFolder = version[1:] - } - - toolSetupCommands := []string{} - if len(optpb.requiredPackages) > 0 { - toolSetupCommands = []string{ - "apt update", - fmt.Sprintf("apt install -y %s", strings.Join(optpb.requiredPackages, " ")), +func buildWorkflows(releaseEnvironmentFilePath string, packageDeployments []osPackageDeployment) []ghaWorkflow { + repoTypes := []string{"apt", "yum"} + workflows := make([]ghaWorkflow, 0, len(repoTypes)*len(packageDeployments)) + for _, packageDeployment := range packageDeployments { + for _, repoType := range repoTypes { + inputs := map[string]string{ + "repo-type": repoType, + "environment": fmt.Sprintf("$(cat %q)", releaseEnvironmentFilePath), + "artifact-tag": "${DRONE_TAG}", + "release-channel": "stable", + "version-channel": packageDeployment.versionChannel, + "package-name-filter": packageDeployment.packageNameFilter, + } + + if packageDeployment.packageToTest != "" { + inputs["package-to-test"] = packageDeployment.packageToTest + } + + workflows = append(workflows, ghaWorkflow{ + stepName: fmt.Sprintf("Publish %s to stable/%s %s repo", packageDeployment.displayName, packageDeployment.versionChannel, repoType), + name: "deploy-packages.yaml", + ref: "refs/heads/master", + timeout: 12 * time.Hour, // DR takes a long time + shouldTagWorkflow: true, + seriesRun: true, + inputs: inputs, + }) } } - toolSetupCommands = append(toolSetupCommands, optpb.setupCommands...) - - assumeDownloadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{ - awsRoleSettings: awsRoleSettings{ - awsAccessKeyID: value{fromSecret: "AWS_ACCESS_KEY_ID"}, - awsSecretAccessKey: value{fromSecret: "AWS_SECRET_ACCESS_KEY"}, - role: value{fromSecret: "AWS_ROLE"}, - }, - configVolume: volumeRefAwsConfig, - name: "Assume Download AWS Role", - }) - - downloadStep := step{ - Name: fmt.Sprintf("Download artifacts for %q", version), - Image: "amazon/aws-cli", - Environment: map[string]value{ - "AWS_S3_BUCKET": { - fromSecret: "AWS_S3_BUCKET", - }, - "ARTIFACT_PATH": { - raw: optpb.artifactPath, - }, - }, - Volumes: []volumeRef{volumeRefAwsConfig}, - Commands: []string{ - "mkdir -pv \"$ARTIFACT_PATH\"", - // Clear out old versions from previous steps - "rm -rf \"$ARTIFACT_PATH\"/*", - // Conditionally match ONLY enterprise and fips binaries based off of file name, - // if running in the context of a private repo (teleport-private) - "if [ \"${DRONE_REPO_PRIVATE}\" = true ]; then ENT_FILTER=\"*ent\"; fi", - fmt.Sprintf("FILTER=\"${ENT_FILTER}*.%s*\"", optpb.packageType), - strings.Join( - []string{ - "aws s3 sync", - "--no-progress", - "--delete", - "--exclude \"*\"", - "--include \"$FILTER\"", - fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder), - "\"$ARTIFACT_PATH\"", - }, - " ", - ), - }, - } - - assumeUploadRoleStep := kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{ - awsRoleSettings: optpb.bucketSecrets.awsRoleSettings, - configVolume: volumeRefAwsConfig, - name: "Assume Upload AWS Role", - }) - - verifyNotPrereleaseStep := verifyNotPrereleaseStep() - - buildAndUploadStep := step{ - Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version), - Image: "golang:1.18.4-bullseye", - Environment: optpb.environmentVars, - Commands: append( - toolSetupCommands, - []string{ - "mkdir -pv -m0700 \"$GNUPGHOME\"", - "echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME", - "chown -R root:root \"$GNUPGHOME\"", - fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")), - fmt.Sprintf("export VERSION=%q", version), - "export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured - strings.Join( - append( - []string{ - // This just makes the (long) command a little more readable - "go run ./cmd/build-os-package-repos", - optpb.packageManagerName, - "-bucket \"$REPO_S3_BUCKET\"", - "-local-bucket-path \"$BUCKET_CACHE_PATH\"", - "-artifact-version \"$VERSION\"", - "-release-channel \"$RELEASE_CHANNEL\"", - "-artifact-path \"$ARTIFACT_PATH\"", - "-log-level 4", // Set this to 5 for debug logging - }, - optpb.extraArgs..., - ), - " ", - ), - }..., - ), - Volumes: []volumeRef{ - { - Name: optpb.volumeName, - Path: optpb.pvcMountPoint, - }, - volumeRefTmpfs, - volumeRefAwsConfig, - }, - } - if enableParallelism { - downloadStep.DependsOn = []string{assumeDownloadRoleStep.Name} - assumeUploadRoleStep.DependsOn = []string{downloadStep.Name} - verifyNotPrereleaseStep.DependsOn = []string{assumeUploadRoleStep.Name} - buildAndUploadStep.DependsOn = []string{verifyNotPrereleaseStep.Name} - } - - return []step{ - assumeDownloadRoleStep, - downloadStep, - assumeUploadRoleStep, - verifyNotPrereleaseStep, - buildAndUploadStep, - } + return workflows } diff --git a/dronegen/promote.go b/dronegen/promote.go index 34aced52ee289..d339a5e53feb4 100644 --- a/dronegen/promote.go +++ b/dronegen/promote.go @@ -16,7 +16,7 @@ package main func promoteBuildPipelines() []pipeline { promotePipelines := make([]pipeline, 0) - promotePipelines = append(promotePipelines, promoteBuildOsRepoPipelines()...) + promotePipelines = append(promotePipelines, promoteBuildOsRepoPipeline()) return promotePipelines } diff --git a/dronegen/push.go b/dronegen/push.go index c7c5ceda6c9d4..44a30bce126f4 100644 --- a/dronegen/push.go +++ b/dronegen/push.go @@ -21,10 +21,12 @@ import ( // pushCheckoutCommands builds a list of commands for Drone to check out a git commit on a push build func pushCheckoutCommands(b buildType) []string { - cloneDirectory := "/go/src/github.com/gravitational/teleport" + return pushCheckoutCommandsWithPath(b, "/go/src/github.com/gravitational/teleport") +} +func pushCheckoutCommandsWithPath(b buildType, checkoutPath string) []string { var commands []string - commands = append(commands, cloneRepoCommands(cloneDirectory, "${DRONE_COMMIT_SHA}")...) + commands = append(commands, cloneRepoCommands(checkoutPath, "${DRONE_COMMIT_SHA}")...) commands = append(commands, `mkdir -m 0700 /root/.ssh && echo "$GITHUB_PRIVATE_KEY" > /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa`, `ssh-keyscan -H github.com > /root/.ssh/known_hosts 2>/dev/null && chmod 600 /root/.ssh/known_hosts`, @@ -78,12 +80,17 @@ func pushPipelines() []pipeline { buildType: buildType{os: "linux", arch: "arm64"}, trigger: triggerPush, pipelineName: "push-build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - timeout: 60 * time.Minute, - slackOnError: true, - srcRefVar: "DRONE_COMMIT", - workflowRef: "${DRONE_BRANCH}", - inputs: map[string]string{"upload-artifacts": "false"}, + workflows: []ghaWorkflow{ + { + name: "release-linux-arm64.yml", + timeout: 60 * time.Minute, + slackOnError: true, + srcRefVar: "DRONE_COMMIT", + ref: "${DRONE_BRANCH}", + shouldTagWorkflow: true, + inputs: map[string]string{"upload-artifacts": "false"}, + }, + }, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/tag.go b/dronegen/tag.go index 383d83ac66980..1f60393bef290 100644 --- a/dronegen/tag.go +++ b/dronegen/tag.go @@ -193,12 +193,17 @@ func tagPipelines() []pipeline { buildType: buildType{os: "linux", arch: "arm64", fips: false}, trigger: triggerTag, pipelineName: "build-linux-arm64", - ghaWorkflow: "release-linux-arm64.yml", - srcRefVar: "DRONE_TAG", - workflowRef: "${DRONE_TAG}", - timeout: 60 * time.Minute, dependsOn: []string{tagCleanupPipelineName}, - inputs: map[string]string{"upload-artifacts": "true"}, + workflows: []ghaWorkflow{ + { + name: "release-linux-arm64.yml", + srcRefVar: "DRONE_TAG", + ref: "${DRONE_TAG}", + timeout: 60 * time.Minute, + shouldTagWorkflow: true, + inputs: map[string]string{"upload-artifacts": "true"}, + }, + }, })) // Only amd64 Windows is supported for now. diff --git a/dronegen/yum.go b/dronegen/yum.go deleted file mode 100644 index 2cbbb44a3fa63..0000000000000 --- a/dronegen/yum.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2021 Gravitational, Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "path" -) - -// This function calls the build-apt-repos tool which handles the APT portion of RFD 0058. -func promoteYumPipeline() pipeline { - return getYumPipelineBuilder().buildPromoteOsPackagePipeline() -} - -func migrateYumPipeline(triggerBranch string, migrationVersions []string) pipeline { - return getYumPipelineBuilder().buildMigrateOsPackagePipeline(triggerBranch, migrationVersions) -} - -func getYumPipelineBuilder() *OsPackageToolPipelineBuilder { - optpb := NewOsPackageToolPipelineBuilder( - "drone-s3-yumrepo-pvc", - "rpm", - "yum", - NewRepoBucketSecrets( - "YUM_REPO_NEW_AWS_S3_BUCKET", - "YUM_REPO_NEW_AWS_ACCESS_KEY_ID", - "YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY", - "YUM_REPO_NEW_AWS_ROLE", - ), - ) - - optpb.environmentVars["CACHE_DIR"] = value{ - raw: path.Join(optpb.pvcMountPoint, "createrepo_cache"), - } - optpb.environmentVars["BUCKET_CACHE_PATH"] = value{ - raw: path.Join(optpb.pvcMountPoint, "bucket"), - } - - optpb.requiredPackages = []string{ - "createrepo-c", - } - - optpb.setupCommands = []string{ - "mkdir -pv \"$CACHE_DIR\"", - } - - optpb.extraArgs = []string{ - "-cache-dir \"$CACHE_DIR\"", - } - - return optpb -}