diff --git a/.drone.yml b/.drone.yml index ca0a997ec6372..8e3f041b9a8c6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5398,7 +5398,7 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/misc.go:138 +# Generated at dronegen/os_repos.go:270 ################################################ kind: pipeline @@ -5426,7 +5426,7 @@ steps: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/misc.go:162 +# Generated at dronegen/os_repos.go:294 ################################################ kind: pipeline @@ -5469,6 +5469,7 @@ steps: image: amazon/aws-cli commands: - mkdir -pv "$ARTIFACT_PATH" + - rm -rf "${ARTIFACT_PATH}/*" - aws s3 sync --no-progress --delete --exclude "*" --include "*.deb*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ "$ARTIFACT_PATH" environment: @@ -5479,25 +5480,25 @@ steps: from_secret: AWS_S3_BUCKET AWS_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY + depends_on: + - Verify build is tagged + - Check out code + - Check if tag is prerelease - name: Publish debs to APT repos for "${DRONE_TAG}" - image: golang:1.18.1-bullseye + image: golang:1.18.4-bullseye commands: - - mkdir -pv -m0700 $GNUPGHOME - - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME - - chown -R root:root $GNUPGHOME - apt update - - apt install aptly tree -y + - apt install -y aptly + - mkdir -pv -m0700 "$GNUPGHOME" + - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME + - chown -R root:root "$GNUPGHOME" - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - export VERSION="${DRONE_TAG}" - export RELEASE_CHANNEL="stable" - - go run ./cmd/build-apt-repos -bucket "$APT_S3_BUCKET" -local-bucket-path "$BUCKET_CACHE_PATH" - -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" -aptly-root-dir - "$APTLY_ROOT_DIR" -artifact-path "$ARTIFACT_PATH" -log-level 4 - - rm -rf "$BUCKET_CACHE_PATH" - - df -h "$APTLY_ROOT_DIR" + - go run ./cmd/build-os-package-repos apt -bucket "$REPO_S3_BUCKET" -local-bucket-path + "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" + -artifact-path "$ARTIFACT_PATH" -log-level 4 -aptly-root-dir "$APTLY_ROOT_DIR" environment: - APT_S3_BUCKET: - from_secret: APT_REPO_NEW_AWS_S3_BUCKET APTLY_ROOT_DIR: /mnt/aptly ARTIFACT_PATH: /go/artifacts AWS_ACCESS_KEY_ID: @@ -5506,16 +5507,24 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: APT_REPO_NEW_AWS_SECRET_ACCESS_KEY BUCKET_CACHE_PATH: /tmp/bucket + DEBIAN_FRONTEND: noninteractive GNUPGHOME: /tmpfs/gnupg GPG_RPM_SIGNING_ARCHIVE: from_secret: GPG_RPM_SIGNING_ARCHIVE + REPO_S3_BUCKET: + from_secret: APT_REPO_NEW_AWS_S3_BUCKET volumes: - - name: aptrepo + - name: apt-persistence path: /mnt - name: tmpfs path: /tmpfs + depends_on: + - Download artifacts for "${DRONE_TAG}" + - Verify build is tagged + - Check out code + - Check if tag is prerelease volumes: -- name: aptrepo +- name: apt-persistence claim: name: drone-s3-aptrepo-pvc - name: tmpfs @@ -5526,12 +5535,150 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/promote.go:81 +# Generated at dronegen/os_repos.go:270 ################################################ kind: pipeline type: kubernetes -name: promote-docker-quay +name: migrate-yum-new-repos +trigger: + event: + include: + - custom + repo: + include: + - non-existent-repository + branch: + include: + - non-existent-branch +clone: + disable: true +steps: +- name: Placeholder + image: alpine:latest + commands: + - echo "This command, step, and pipeline never runs" + +--- +################################################ +# Generated using dronegen, do not edit by hand! +# Use 'make dronegen' to update. +# Generated at dronegen/os_repos.go:294 +################################################ + +kind: pipeline +type: kubernetes +name: publish-yum-new-repos +trigger: + event: + include: + - promote + target: + include: + - production + repo: + include: + - gravitational/teleport +workspace: + path: /go +clone: + disable: true +steps: +- name: Verify build is tagged + image: alpine:latest + commands: + - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' + && exit 1)' +- name: Check out code + image: alpine/git:latest + commands: + - mkdir -p "/go/src/github.com/gravitational/teleport" + - cd "/go/src/github.com/gravitational/teleport" + - git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git . + - git checkout "${DRONE_TAG}" +- name: Check if tag is prerelease + image: golang:1.17-alpine + commands: + - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" + - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is + a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78) +- name: Download artifacts for "${DRONE_TAG}" + image: amazon/aws-cli + commands: + - mkdir -pv "$ARTIFACT_PATH" + - rm -rf "${ARTIFACT_PATH}/*" + - aws s3 sync --no-progress --delete --exclude "*" --include "*.rpm*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ + "$ARTIFACT_PATH" + environment: + ARTIFACT_PATH: /go/artifacts + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + AWS_S3_BUCKET: + from_secret: AWS_S3_BUCKET + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + depends_on: + - Verify build is tagged + - Check out code + - Check if tag is prerelease +- name: Publish rpms to YUM repos for "${DRONE_TAG}" + image: golang:1.18.4-bullseye + commands: + - apt update + - apt install -y createrepo-c + - mkdir -pv "$CACHE_DIR" + - mkdir -pv -m0700 "$GNUPGHOME" + - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME + - chown -R root:root "$GNUPGHOME" + - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" + - export VERSION="${DRONE_TAG}" + - export RELEASE_CHANNEL="stable" + - go run ./cmd/build-os-package-repos yum -bucket "$REPO_S3_BUCKET" -local-bucket-path + "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" + -artifact-path "$ARTIFACT_PATH" -log-level 4 -cache-dir "$CACHE_DIR" + environment: + ARTIFACT_PATH: /go/artifacts + AWS_ACCESS_KEY_ID: + from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID + AWS_REGION: us-west-2 + AWS_SECRET_ACCESS_KEY: + from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY + BUCKET_CACHE_PATH: /mnt/bucket + CACHE_DIR: /mnt/createrepo_cache + DEBIAN_FRONTEND: noninteractive + GNUPGHOME: /tmpfs/gnupg + GPG_RPM_SIGNING_ARCHIVE: + from_secret: GPG_RPM_SIGNING_ARCHIVE + REPO_S3_BUCKET: + from_secret: YUM_REPO_NEW_AWS_S3_BUCKET + volumes: + - name: yum-persistence + path: /mnt + - name: tmpfs + path: /tmpfs + depends_on: + - Download artifacts for "${DRONE_TAG}" + - Verify build is tagged + - Check out code + - Check if tag is prerelease +volumes: +- name: yum-persistence + claim: + name: drone-s3-yumrepo-pvc +- name: tmpfs + temp: + medium: memory + +--- +################################################ +# Generated using dronegen, do not edit by hand! +# Use 'make dronegen' to update. +# Generated at dronegen/promote.go:28 +################################################ + +kind: pipeline +type: kubernetes +name: promote-docker-ecr trigger: event: include: @@ -5540,7 +5687,7 @@ trigger: include: - production - promote-docker - - promote-docker-quay + - promote-docker-ecr repo: include: - gravitational/* @@ -5566,43 +5713,31 @@ steps: commands: - apk add --no-cache aws-cli - export VERSION=${DRONE_TAG##v} - - docker login -u="$STAGING_QUAY_USERNAME" -p="$STAGING_QUAY_PASSWORD" quay.io - aws ecr get-login-password --region=us-west-2 | docker login -u="AWS" --password-stdin 146628656107.dkr.ecr.us-west-2.amazonaws.com - echo "---> Pulling images for $${VERSION}" - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport:$${VERSION} - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION} - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION}-fips - - docker pull quay.io/gravitational/teleport-operator-ci:$${VERSION} - echo "---> Tagging images for $${VERSION}" - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport:$${VERSION} - quay.io/gravitational/teleport:$${VERSION} + public.ecr.aws/gravitational/teleport:$${VERSION} - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION} - quay.io/gravitational/teleport-ent:$${VERSION} + public.ecr.aws/gravitational/teleport-ent:$${VERSION} - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION}-fips - quay.io/gravitational/teleport-ent:$${VERSION}-fips - - docker tag quay.io/gravitational/teleport-operator-ci:$${VERSION} quay.io/gravitational/teleport-operator:$${VERSION} - - docker logout quay.io + public.ecr.aws/gravitational/teleport-ent:$${VERSION}-fips - docker logout 146628656107.dkr.ecr.us-west-2.amazonaws.com - - docker login -u="$QUAY_USERNAME" -p="$QUAY_PASSWORD" quay.io + - aws ecr-public get-login-password --region=us-east-1 | docker login -u="AWS" --password-stdin + public.ecr.aws - echo "---> Pushing images for $${VERSION}" - - docker push quay.io/gravitational/teleport:$${VERSION} - - docker push quay.io/gravitational/teleport-ent:$${VERSION} - - docker push quay.io/gravitational/teleport-ent:$${VERSION}-fips - - docker push quay.io/gravitational/teleport-operator:$${VERSION} + - docker push public.ecr.aws/gravitational/teleport:$${VERSION} + - docker push public.ecr.aws/gravitational/teleport-ent:$${VERSION} + - docker push public.ecr.aws/gravitational/teleport-ent:$${VERSION}-fips environment: AWS_ACCESS_KEY_ID: - from_secret: STAGING_TELEPORT_DRONE_USER_ECR_KEY + from_secret: PRODUCTION_TELEPORT_DRONE_USER_ECR_KEY AWS_SECRET_ACCESS_KEY: - from_secret: STAGING_TELEPORT_DRONE_USER_ECR_SECRET - QUAY_PASSWORD: - from_secret: PRODUCTION_QUAYIO_DOCKER_PASSWORD - QUAY_USERNAME: - from_secret: PRODUCTION_QUAYIO_DOCKER_USERNAME - STAGING_QUAY_PASSWORD: - from_secret: QUAYIO_DOCKER_PASSWORD - STAGING_QUAY_USERNAME: - from_secret: QUAYIO_DOCKER_USERNAME + from_secret: PRODUCTION_TELEPORT_DRONE_USER_ECR_SECRET volumes: - name: dockersock path: /var/run @@ -5621,12 +5756,12 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/promote.go:27 +# Generated at dronegen/promote.go:82 ################################################ kind: pipeline type: kubernetes -name: promote-docker-ecr +name: promote-docker-quay trigger: event: include: @@ -5635,7 +5770,7 @@ trigger: include: - production - promote-docker - - promote-docker-ecr + - promote-docker-quay repo: include: - gravitational/* @@ -5661,31 +5796,43 @@ steps: commands: - apk add --no-cache aws-cli - export VERSION=${DRONE_TAG##v} + - docker login -u="$STAGING_QUAY_USERNAME" -p="$STAGING_QUAY_PASSWORD" quay.io - aws ecr get-login-password --region=us-west-2 | docker login -u="AWS" --password-stdin 146628656107.dkr.ecr.us-west-2.amazonaws.com - echo "---> Pulling images for $${VERSION}" - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport:$${VERSION} - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION} - docker pull 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION}-fips + - docker pull quay.io/gravitational/teleport-operator-ci:$${VERSION} - echo "---> Tagging images for $${VERSION}" - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport:$${VERSION} - public.ecr.aws/gravitational/teleport:$${VERSION} + quay.io/gravitational/teleport:$${VERSION} - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION} - public.ecr.aws/gravitational/teleport-ent:$${VERSION} + quay.io/gravitational/teleport-ent:$${VERSION} - docker tag 146628656107.dkr.ecr.us-west-2.amazonaws.com/gravitational/teleport-ent:$${VERSION}-fips - public.ecr.aws/gravitational/teleport-ent:$${VERSION}-fips + quay.io/gravitational/teleport-ent:$${VERSION}-fips + - docker tag quay.io/gravitational/teleport-operator-ci:$${VERSION} quay.io/gravitational/teleport-operator:$${VERSION} + - docker logout quay.io - docker logout 146628656107.dkr.ecr.us-west-2.amazonaws.com - - aws ecr-public get-login-password --region=us-east-1 | docker login -u="AWS" --password-stdin - public.ecr.aws + - docker login -u="$QUAY_USERNAME" -p="$QUAY_PASSWORD" quay.io - echo "---> Pushing images for $${VERSION}" - - docker push public.ecr.aws/gravitational/teleport:$${VERSION} - - docker push public.ecr.aws/gravitational/teleport-ent:$${VERSION} - - docker push public.ecr.aws/gravitational/teleport-ent:$${VERSION}-fips + - docker push quay.io/gravitational/teleport:$${VERSION} + - docker push quay.io/gravitational/teleport-ent:$${VERSION} + - docker push quay.io/gravitational/teleport-ent:$${VERSION}-fips + - ERSION} environment: AWS_ACCESS_KEY_ID: - from_secret: PRODUCTION_TELEPORT_DRONE_USER_ECR_KEY + from_secret: STAGING_TELEPORT_DRONE_USER_ECR_KEY AWS_SECRET_ACCESS_KEY: - from_secret: PRODUCTION_TELEPORT_DRONE_USER_ECR_SECRET + from_secret: STAGING_TELEPORT_DRONE_USER_ECR_SECRET + QUAY_PASSWORD: + from_secret: PRODUCTION_QUAYIO_DOCKER_PASSWORD + QUAY_USERNAME: + from_secret: PRODUCTION_QUAYIO_DOCKER_USERNAME + STAGING_QUAY_PASSWORD: + from_secret: QUAYIO_DOCKER_PASSWORD + STAGING_QUAY_USERNAME: + from_secret: QUAYIO_DOCKER_USERNAME volumes: - name: dockersock path: /var/run @@ -6091,6 +6238,6 @@ volumes: name: drone-s3-debrepo-pvc --- kind: signature -hmac: cfe9263c545d10b26f27dc10b8b5e5b833d0fca4333860550dae28113cef681a +hmac: 886b4ee9d5440155e696730354e0d8f773def8d703105d58d7017bf997e0c687 ... diff --git a/build.assets/tooling/cmd/build-apt-repos/config.go b/build.assets/tooling/cmd/build-apt-repos/config.go deleted file mode 100644 index e4ea43cfc7835..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/config.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" -) - -const StableChannelFlagValue string = "stable" - -type Config struct { - artifactPath string - artifactVersion string - bucketName string - localBucketPath string - releaseChannel string - aptlyPath string - logLevel uint - logJSON bool -} - -// Parses and validates the provided flags, returning the parsed arguments in a struct. -func ParseFlags() (*Config, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, trace.Wrap(err, "failed to get user's home directory path") - } - - config := &Config{} - flag.StringVar(&config.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb files to add to the APT repos") - flag.StringVar(&config.artifactVersion, "artifact-version", "", "The version of the artifacts that will be added to the APT repos") - flag.StringVar(&config.releaseChannel, "release-channel", "", "The release channel of the APT repos that the artifacts should be added to") - flag.StringVar(&config.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from") - flag.StringVar(&config.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to") - flag.StringVar(&config.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)") - flag.UintVar(&config.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose") - flag.BoolVar(&config.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging") - - flag.Parse() - if err := Check(config); err != nil { - return nil, trace.Wrap(err, "failed to validate flags") - } - - return config, nil -} - -func Check(config *Config) error { - if err := validateArtifactPath(config.artifactPath); err != nil { - return trace.Wrap(err, "failed to validate the artifact path flag") - } - if err := validateBucketName(config.bucketName); err != nil { - return trace.Wrap(err, "failed to validate the bucket name flag") - } - if err := validateLocalBucketPath(config.localBucketPath); err != nil { - return trace.Wrap(err, "failed to validate the local bucket path flag") - } - if err := validateArtifactVersion(config.artifactVersion); err != nil { - return trace.Wrap(err, "failed to validate the artifact version flag") - } - if err := validateReleaseChannel(config.releaseChannel); err != nil { - return trace.Wrap(err, "failed to validate the release channel flag") - } - if err := validateLogLevel(config.logLevel); err != nil { - return trace.Wrap(err, "failed to validate the log level flag") - } - - return nil -} - -func validateArtifactPath(value string) error { - if value == "" { - return trace.BadParameter("the artifact-path flag should not be empty") - } - - if stat, err := os.Stat(value); os.IsNotExist(err) { - return trace.BadParameter("the artifact-path %q does not exist", value) - } else if !stat.IsDir() { - return trace.BadParameter("the artifact-path %q is not a directory", value) - } - - return nil -} - -func validateBucketName(value string) error { - if value == "" { - return trace.BadParameter("the bucket flag should not be empty") - } - - return nil -} - -func validateLocalBucketPath(value string) error { - if value == "" { - return trace.BadParameter("the local-bucket-path flag should not be empty") - } - - if stat, err := os.Stat(value); err == nil && !stat.IsDir() { - return trace.BadParameter("the local bucket path points to a file instead of a directory") - } - - return nil -} - -func validateArtifactVersion(value string) error { - if value == "" { - return trace.BadParameter("the artifact-version flag should not be empty") - } - - if !semver.IsValid(value) { - return trace.BadParameter("the artifact-version flag does not contain a valid semver version string") - } - - return nil -} - -func validateReleaseChannel(value string) error { - if value == "" { - return trace.BadParameter("the release-channel flag should not be empty") - } - - // Not sure what other channels we'd want to support, but they should be listed here - validReleaseChannels := []string{StableChannelFlagValue} - - for _, validReleaseChannel := range validReleaseChannels { - if value == validReleaseChannel { - return nil - } - } - - return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ",")) -} - -func validateLogLevel(value uint) error { - if value > 6 { - return trace.BadParameter("the log-level flag should be between 0 and 6") - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-apt-repos/main.go b/build.assets/tooling/cmd/build-apt-repos/main.go deleted file mode 100644 index c4c08d03ef41d..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/main.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - log "github.com/sirupsen/logrus" -) - -func main() { - supportedOSs := map[string][]string{ - "debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details - "stretch", // 9 - "buster", // 10 - "bullseye", // 11 - "bookwork", // 12 - "trixie", // 13 - }, - "ubuntu": { // See https://wiki.ubuntu.com/Releases for details - "xenial", // 16.04 LTS - "yakkety", // 16.10 (EOL) - "zesty", // 17.04 (EOL) - "artful", // 17.10 (EOL) - "bionic", // 18.04 LTS - "cosmic", // 18.10 (EOL) - "disco", // 19.04 (EOL) - "eoan", // 19.10 (EOL) - "focal", // 20.04 LTS - "groovy", // 20.10 (EOL) - "hirsuite", // 21.04 (EOL) - "impish", // 21.10 (EOL on 7/14/22) - "jammy", // 22.04 LTS - }, - } - - config, err := ParseFlags() - if err != nil { - log.Fatal(err.Error()) - } - - setupLogger(config) - log.Debugf("Starting tool with config: %v", config) - - art, err := NewAptRepoTool(config, supportedOSs) - if err != nil { - log.Fatal(err.Error()) - } - - err = art.Run() - if err != nil { - log.Fatal(err.Error()) - } -} - -func setupLogger(config *Config) { - if config.logJSON { - log.SetFormatter(&log.JSONFormatter{}) - } else { - log.SetFormatter(&log.TextFormatter{}) - } - log.SetOutput(os.Stdout) - log.SetLevel(log.Level(config.logLevel)) -} diff --git a/build.assets/tooling/cmd/build-apt-repos/s3manager.go b/build.assets/tooling/cmd/build-apt-repos/s3manager.go deleted file mode 100644 index 3a060eb05e5dd..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/s3manager.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "github.com/aws/aws-sdk-go/aws/session" - "github.com/gravitational/trace" - "github.com/seqsense/s3sync" - "github.com/sirupsen/logrus" -) - -type S3manager struct { - syncManager *s3sync.Manager - bucketName string - bucketPath string -} - -func NewS3Manager(bucketName string) *S3manager { - // Right now the AWS session is only used by this manager, but if it ends - // up being needed elsewhere then it should probably be moved to an arg - awsSession := session.Must(session.NewSession()) - - manager := &S3manager{ - syncManager: s3sync.New(awsSession), - bucketName: bucketName, - bucketPath: fmt.Sprintf("s3://%s", bucketName), - } - - s3sync.SetLogger(&s3logger{}) - - return manager -} - -func (s *S3manager) DownloadExistingRepo(localPath string) error { - err := ensureDirectoryExists(localPath) - if err != nil { - return trace.Wrap(err, "failed to ensure path %q exists", localPath) - } - - err = s.sync(localPath, true) - if err != nil { - return trace.Wrap(err, "failed to download bucket") - } - - return nil -} - -func (s *S3manager) UploadBuiltRepo(localPath string) error { - err := s.sync(localPath, false) - - if err != nil { - return trace.Wrap(err, "failed to upload bucket") - } - - return nil -} - -func (s *S3manager) sync(localPath string, download bool) error { - var src, dest string - if download { - src = s.bucketPath - dest = localPath - } else { - src = localPath - dest = s.bucketPath - } - - logrus.Infof("Performing S3 sync from %q to %q...", src, dest) - err := s.syncManager.Sync(src, dest) - if err != nil { - return trace.Wrap(err, "failed to sync %q to %q", src, dest) - } - logrus.Infoln("S3 sync complete") - - return nil -} - -func ensureDirectoryExists(path string) error { - err := os.MkdirAll(path, 0660) - if err != nil { - return trace.Wrap(err, "failed to create directory %q", path) - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go b/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go similarity index 82% rename from build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go rename to build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go index 27ae11f1bd428..66e0e60ed9545 100644 --- a/build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go +++ b/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go @@ -22,40 +22,51 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" "github.com/gravitational/trace" "github.com/sirupsen/logrus" "golang.org/x/mod/semver" ) type AptRepoTool struct { - config *Config + config *AptConfig aptly *Aptly + gpg *GPG s3Manager *S3manager supportedOSs map[string][]string } // Instantiates a new apt repo tool instance and performs any required setup/config. -func NewAptRepoTool(config *Config, supportedOSs map[string][]string) (*AptRepoTool, error) { - art := &AptRepoTool{ - config: config, - s3Manager: NewS3Manager(config.bucketName), - supportedOSs: supportedOSs, - } - +func NewAptRepoTool(config *AptConfig, supportedOSs map[string][]string) (*AptRepoTool, error) { aptly, err := NewAptly(config.aptlyPath) if err != nil { return nil, trace.Wrap(err, "failed to create a new aptly instance") } - art.aptly = aptly + gpg, err := NewGPG() + if err != nil { + return nil, trace.Wrap(err, "failed to create a new GPG instance") + } + + s3Manager, err := NewS3Manager(config.S3Config) + if err != nil { + return nil, trace.Wrap(err, "failed to create a new s3manager instance") + } - return art, nil + return &AptRepoTool{ + aptly: aptly, + config: config, + gpg: gpg, + s3Manager: s3Manager, + supportedOSs: supportedOSs, + }, nil } // Runs the tool, creating and updating APT repos based upon the current configuration. func (art *AptRepoTool) Run() error { start := time.Now() logrus.Infoln("Starting APT repo build process...") + logrus.Debugf("Using config: %+v", spew.Sdump(art.config)) isFirstRun, err := art.aptly.IsFirstRun() if err != nil { @@ -65,7 +76,7 @@ func (art *AptRepoTool) Run() error { if isFirstRun { logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from APT repository...") - err = art.s3Manager.DownloadExistingRepo(art.config.localBucketPath) + err = art.s3Manager.DownloadExistingRepo() if err != nil { return trace.Wrap(err, "failed to sync existing repo from S3 bucket") } @@ -74,6 +85,8 @@ func (art *AptRepoTool) Run() error { if err != nil { return trace.Wrap(err, "failed to recreate existing repos") } + } else { + logrus.Debugf("Not first run of tool, skipping Aptly repository rebuild process") } // Note: this logic will only push the artifact into the `art.supportedOSs` repos. @@ -94,11 +107,24 @@ func (art *AptRepoTool) Run() error { return trace.Wrap(err, "failed to publish repos") } - err = art.s3Manager.UploadBuiltRepo(filepath.Join(art.aptly.rootDir, "public")) + // Both Hashicorp and Docker publish their key to this path + err = art.gpg.WritePublicKeyToFile(filepath.Join(art.aptly.rootDir, "public", "gpg")) + if err != nil { + return trace.Wrap(err, "failed to write GPG public key") + } + + art.s3Manager.ChangeLocalBucketPath(filepath.Join(art.aptly.rootDir, "public")) + err = art.s3Manager.UploadBuiltRepo() if err != nil { return trace.Wrap(err, "failed to sync changes to S3 bucket") } + // Future work: add literals to config? + err = art.s3Manager.UploadRedirectURL("index.html", "https://goteleport.com/docs/installation/#linux") + if err != nil { + return trace.Wrap(err, "failed to redirect index page to Teleport docs") + } + logrus.Infof("APT repo build process completed in %s", time.Since(start).Round(time.Millisecond)) return nil } diff --git a/build.assets/tooling/cmd/build-apt-repos/aptly.go b/build.assets/tooling/cmd/build-os-package-repos/aptly.go similarity index 94% rename from build.assets/tooling/cmd/build-apt-repos/aptly.go rename to build.assets/tooling/cmd/build-os-package-repos/aptly.go index 26dcb5e9f7410..32255d105740d 100644 --- a/build.assets/tooling/cmd/build-apt-repos/aptly.go +++ b/build.assets/tooling/cmd/build-os-package-repos/aptly.go @@ -22,9 +22,7 @@ import ( "errors" "fmt" "io/fs" - "log" "os" - "os/exec" "path" "path/filepath" "regexp" @@ -66,7 +64,7 @@ func (*Aptly) ensureDefaultConfigExists() error { // ran, which messes up the output. // Note: it is important to not use any repo-related commands here as they have a side effect of // also creating the Aptly rootDir structure which is usually undesirable here - _, err := buildAndRunCommand("aptly", "config", "show") + _, err := BuildAndRunCommand("aptly", "config", "show") if err != nil { return trace.Wrap(err, "failed to create default Aptly config") } @@ -86,7 +84,7 @@ func (a *Aptly) updateConfiguration() error { logrus.Debugf("Built Aptly config: %v", aptlyConfigMap) saveAptlyConfigMap(aptlyConfigMap) - configOutput, err := buildAndRunCommand("aptly", "config", "show") + configOutput, err := BuildAndRunCommand("aptly", "config", "show") if err != nil { return trace.Wrap(err, "failed to check Aptly config") } @@ -192,7 +190,7 @@ func (a *Aptly) CreateRepoIfNotExists(r *Repo) (bool, error) { distributionArg := fmt.Sprintf("-distribution=%s", r.osVersion) componentArg := fmt.Sprintf("-component=%s/%s", r.releaseChannel, r.majorVersion) - _, err = buildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name()) + _, err = BuildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name()) if err != nil { return false, trace.Wrap(err, "failed to create repo %q", r.Name()) } @@ -222,7 +220,7 @@ func (a *Aptly) GetExistingRepoNames() ([]string, error) { // ... // // ``` - output, err := buildAndRunCommand("aptly", "repo", "list", "-raw") + output, err := BuildAndRunCommand("aptly", "repo", "list", "-raw") if err != nil { return nil, trace.Wrap(err, "failed to get a list of existing repos") } @@ -248,7 +246,7 @@ func (a *Aptly) GetExistingRepoNames() ([]string, error) { func (a *Aptly) ImportDeb(repoName string, debPath string) error { logrus.Infof("Importing deb(s) from %q into repo %q...", debPath, repoName) - _, err := buildAndRunCommand("aptly", "repo", "add", repoName, debPath) + _, err := BuildAndRunCommand("aptly", "repo", "add", repoName, debPath) if err != nil { return trace.Wrap(err, "failed to add %q to repo %q", debPath, repoName) } @@ -320,7 +318,7 @@ func parsePackagesFile(packagesPath string) ([]string, error) { logrus.Debugf("Parsing packages file %q", packagesPath) file, err := os.Open(packagesPath) if err != nil { - log.Fatal(err) + logrus.Fatal(err) } defer file.Close() @@ -394,7 +392,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) // If all repos have been published if areSomePublished && !areSomeUnpublished { // Update rather than republish - _, err := buildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS) + _, err := BuildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS) if err != nil { return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) } @@ -406,7 +404,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) // This will occur if there is a new major release, a OS version is supported, or a new release channel is added if areSomePublished && areSomeUnpublished { // Drop the currently published APT repo so that it can be rebuilt from scratch - _, err := buildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS) + _, err := BuildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS) if err != nil { return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) } @@ -423,7 +421,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) args = append(args, repoOS) // Full command is `aptly publish repo -component=<, repeating len(repos) - 1 times> ` - _, err = buildAndRunCommand("aptly", args...) + _, err = BuildAndRunCommand("aptly", args...) if err != nil { return trace.Wrap(err, "failed to publish repos") } @@ -501,7 +499,7 @@ func (a *Aptly) GetPublishedRepoNames() ([]string, error) { // No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`. // ``` // Note that the `-raw` argument is not used here as it does not provide sufficient information - output, err := buildAndRunCommand("aptly", "publish", "list") + output, err := BuildAndRunCommand("aptly", "publish", "list") if err != nil { return nil, trace.Wrap(err, "failed to get a list of published repos") } @@ -675,26 +673,3 @@ func getSubdirectories(basePath string) ([]string, error) { return subdirectories, nil } - -func buildAndRunCommand(command string, args ...string) (string, error) { - cmd := exec.Command(command, args...) - logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '")) - output, err := cmd.CombinedOutput() - - if output != nil { - logrus.Debugf("Command output: %s", string(output)) - } - - if err != nil { - if exitError, ok := err.(*exec.ExitError); ok { - exitCode := exitError.ExitCode() - logrus.Debugf("Command exited with exit code %d", exitCode) - } else { - logrus.Debugln("Command failed without an exit code") - } - return "", trace.Wrap(err, "Command failed, see debug output for additional details") - } - - logrus.Debugln("Command exited successfully") - return string(output), nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/command_executor.go b/build.assets/tooling/cmd/build-os-package-repos/command_executor.go new file mode 100644 index 0000000000000..78a9bb8b7342d --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/command_executor.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "os/exec" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +// Builds an runs a command with the provided arguments. Extensively logs command +// details to the debug log. Returns stdout and stderr combined, along with an +// error iff one occurred. +func BuildAndRunCommand(command string, args ...string) (string, error) { + cmd := exec.Command(command, args...) + logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '")) + output, err := cmd.CombinedOutput() + + if output != nil { + logrus.Debugf("Command output: %s", string(output)) + } + + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + exitCode := exitError.ExitCode() + logrus.Debugf("Command exited with exit code %d", exitCode) + } else { + logrus.Debugln("Command failed without an exit code") + } + return "", trace.Wrap(err, "Command failed, see debug output for additional details") + } + + logrus.Debugln("Command exited successfully") + return string(output), nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/config.go b/build.assets/tooling/cmd/build-os-package-repos/config.go new file mode 100644 index 0000000000000..df3357f2dee57 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/config.go @@ -0,0 +1,282 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" +) + +const StableChannelFlagValue string = "stable" + +type LoggerConfig struct { + logLevel uint + logJSON bool +} + +func NewLoggerConfigWithFlagset(fs *flag.FlagSet) *LoggerConfig { + lc := &LoggerConfig{} + fs.UintVar(&lc.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose") + fs.BoolVar(&lc.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging") + + return lc +} + +func (lc *LoggerConfig) Check() error { + if err := lc.validateLogLevel(); err != nil { + return trace.Wrap(err, "failed to validate the log level flag") + } + + return nil +} + +func (lc *LoggerConfig) validateLogLevel() error { + if lc.logLevel > 6 { + return trace.BadParameter("the log-level flag should be between 0 and 6") + } + + return nil +} + +type S3Config struct { + bucketName string + localBucketPath string + maxConcurrentSyncs int +} + +func NewS3ConfigWithFlagset(fs *flag.FlagSet) *S3Config { + s3c := &S3Config{} + fs.StringVar(&s3c.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from") + fs.StringVar(&s3c.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to") + fs.IntVar(&s3c.maxConcurrentSyncs, "max-concurrent-syncs", 16, "The maximum number of S3 bucket syncs that may run in parallel (-1 for unlimited, 16 default)") + + return s3c +} + +func (s3c *S3Config) Check() error { + if err := s3c.validateBucketName(); err != nil { + return trace.Wrap(err, "failed to validate the bucket name flag") + } + if err := s3c.validateLocalBucketPath(); err != nil { + return trace.Wrap(err, "failed to validate the local bucket path flag") + } + if err := s3c.validateMaxConcurrentSyncs(); err != nil { + return trace.Wrap(err, "failed to validate the max concurrent syncs flag") + } + + return nil +} + +func (s3c *S3Config) validateBucketName() error { + if s3c.bucketName == "" { + return trace.BadParameter("the bucket flag should not be empty") + } + + return nil +} + +func (s3c *S3Config) validateLocalBucketPath() error { + if s3c.localBucketPath == "" { + return trace.BadParameter("the local-bucket-path flag should not be empty") + } + + if stat, err := os.Stat(s3c.localBucketPath); err == nil && !stat.IsDir() { + return trace.BadParameter("the local bucket path points to a file instead of a directory") + } + + return nil +} + +func (s3c *S3Config) validateMaxConcurrentSyncs() error { + if s3c.maxConcurrentSyncs < -1 { + return trace.BadParameter("the max-concurrent-syncs flag must be greater than -1") + } + + return nil +} + +// This type is common to all other config types +type Config struct { + *LoggerConfig + *S3Config + artifactPath string + artifactVersion string + printHelp bool + releaseChannel string +} + +func NewConfigWithFlagset(fs *flag.FlagSet) *Config { + c := &Config{} + c.LoggerConfig = NewLoggerConfigWithFlagset(fs) + c.S3Config = NewS3ConfigWithFlagset(fs) + + fs.StringVar(&c.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb or *.rpm files to add to the repos") + fs.StringVar(&c.artifactVersion, "artifact-version", "", "The version of the artifacts that will be added to the repos") + fs.Visit(func(f *flag.Flag) { + if f.Name == "-h" || f.Name == "--help" { + c.printHelp = true + } + }) + fs.StringVar(&c.releaseChannel, "release-channel", "", "The release channel of the repos that the artifacts should be added to") + + return c +} + +func (c *Config) Check() error { + if err := c.LoggerConfig.Check(); err != nil { + return trace.Wrap(err, "failed to validate logger config") + } + + if err := c.S3Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate S3 config") + } + + if err := c.validateArtifactPath(); err != nil { + return trace.Wrap(err, "failed to validate the artifact path flag") + } + if err := c.validateArtifactVersion(); err != nil { + return trace.Wrap(err, "failed to validate the artifact version flag") + } + if err := c.validateReleaseChannel(); err != nil { + return trace.Wrap(err, "failed to validate the release channel flag") + } + + return nil +} + +func (c *Config) validateArtifactPath() error { + if c.artifactPath == "" { + return trace.BadParameter("the artifact-path flag should not be empty") + } + + if stat, err := os.Stat(c.artifactPath); os.IsNotExist(err) { + return trace.BadParameter("the artifact-path %q does not exist", c.artifactPath) + } else if !stat.IsDir() { + return trace.BadParameter("the artifact-path %q is not a directory", c.artifactPath) + } + + return nil +} + +func (c *Config) validateArtifactVersion() error { + if c.artifactVersion == "" { + return trace.BadParameter("the artifact-version flag should not be empty") + } + + if !semver.IsValid(c.artifactVersion) { + return trace.BadParameter("the artifact-version flag does not contain a valid semver version string") + } + + return nil +} + +func (c *Config) validateReleaseChannel() error { + if c.releaseChannel == "" { + return trace.BadParameter("the release-channel flag should not be empty") + } + + // Not sure what other channels we'd want to support, but they should be listed here + validReleaseChannels := []string{StableChannelFlagValue} + + for _, validReleaseChannel := range validReleaseChannels { + if c.releaseChannel == validReleaseChannel { + return nil + } + } + + return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ",")) +} + +// APT-specific config +type AptConfig struct { + *Config + aptlyPath string +} + +func NewAptConfigWithFlagSet(fs *flag.FlagSet) (*AptConfig, error) { + ac := &AptConfig{} + ac.Config = NewConfigWithFlagset(fs) + + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, trace.Wrap(err, "failed to get user's home directory path") + } + + fs.StringVar(&ac.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)") + + return ac, nil +} + +func (ac *AptConfig) validateAptlyPath() error { + if ac.aptlyPath == "" { + return trace.BadParameter("the aptly-root-dir flag should not be empty") + } + + return nil +} + +func (ac *AptConfig) Check() error { + if err := ac.Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate common config") + } + + if err := ac.validateAptlyPath(); err != nil { + return trace.Wrap(err, "failed to validate the aptly-root-dir path flag") + } + + return nil +} + +// YUM-specific config +type YumConfig struct { + *Config + cacheDir string +} + +func NewYumConfigWithFlagSet(fs *flag.FlagSet) *YumConfig { + yc := &YumConfig{} + yc.Config = NewConfigWithFlagset(fs) + + fs.StringVar(&yc.cacheDir, "cache-dir", "/tmp/createrepo/cache", "The createrepo checksum caching directory (see https://linux.die.net/man/8/createrepo for details") + + return yc +} + +func (yc *YumConfig) validateCacheDir() error { + if yc.cacheDir == "" { + return trace.BadParameter("the cache-dir flag should not be empty") + } + + return nil +} + +func (yc *YumConfig) Check() error { + if err := yc.Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate common config") + } + + if err := yc.validateCacheDir(); err != nil { + return trace.Wrap(err, "failed to validate the cache-dir path flag") + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/createrepo.go b/build.assets/tooling/cmd/build-os-package-repos/createrepo.go new file mode 100644 index 0000000000000..5c3a208926891 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/createrepo.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "os/exec" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +type CreateRepo struct { + cacheDir string + binaryName string +} + +// Instantiates createrepo, ensuring all system requirements for performing createrepo operations +// have been met +func NewCreateRepo(cacheDir string) (*CreateRepo, error) { + cr := &CreateRepo{ + cacheDir: cacheDir, + // `createrepo_c` is the "new" (as in 9 years old) replacement for `createrepo` + // This can be replace with "createrepo" in the unlikely chance that there is + // a problem + binaryName: "createrepo_c", + } + + err := cr.ensureBinaryExists() + if err != nil { + return nil, trace.Wrap(err, "failed to ensure CreateRepo binary exists") + } + + // Ensure the cache dir exists + err = os.MkdirAll(cr.cacheDir, 0660) + if err != nil { + return nil, trace.Wrap(err, "failed to ensure %q exists", cr.cacheDir) + } + + return cr, nil +} + +func (cr *CreateRepo) ensureBinaryExists() error { + _, err := exec.LookPath(cr.binaryName) + if err != nil { + return trace.Wrap(err, "failed to verify that %q binary exists", cr.binaryName) + } + + return nil +} + +func (cr *CreateRepo) CreateOrUpdateRepo(repoPath string) error { + // --cachedir --update + logrus.Debugf("Updating repo metadata for repo at %q", repoPath) + + args := []string{ + "--cachedir", + cr.cacheDir, + "--update", + repoPath, + } + + _, err := BuildAndRunCommand(cr.binaryName, args...) + if err != nil { + return trace.Wrap(err, "createrepo create/update command failed on path %q with cache directory %q", repoPath, cr.cacheDir) + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/gpg.go b/build.assets/tooling/cmd/build-os-package-repos/gpg.go new file mode 100644 index 0000000000000..531649d4d45fc --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/gpg.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +type GPG struct{} + +// Instantiates GPG, ensuring all system requirements for using GPG are fulfilled +func NewGPG() (*GPG, error) { + g := &GPG{} + + err := g.ensureFirstRunHasOccurred() + if err != nil { + return nil, trace.Wrap(err, "failed to setup GPG") + } + + err = g.ensureSecretKeyExists() + if err != nil { + return nil, trace.Wrap(err, "failed to ensure a secret key exists") + } + + return g, nil +} + +// The first time GPG is run for a user with any "meaningful" arguments it will +// generate several files and log what it created to stdout. These logs can +// disrupt parsing of GPG command outputs, so we force it to happen here, once, +// rather than try and handle it on each GPG call. +func (*GPG) ensureFirstRunHasOccurred() error { + _, err := BuildAndRunCommand("gpg", "--fingerprint") + if err != nil { + return trace.Wrap(err, "failed to ensure GPG has been ran once") + } + + return nil +} + +func (*GPG) ensureSecretKeyExists() error { + output, err := BuildAndRunCommand("gpg", "--list-secret-keys", "--with-colons") + if err != nil { + return trace.Wrap(err, "failed to ensure GPG secret key exists") + } + + outputLineCount := strings.Count(output, "\n") + if outputLineCount < 1 { + return trace.Errorf("failed to find a GPG secret key") + } + + return nil +} + +// Creates a detached, armored signature for the provided file using the default GPG key +func (*GPG) SignFile(filePath string) error { + // While this could be done via a Go module, the x/crypto/openpgp library has been frozen + // and deprecated for almost 18 months. Others exist, but given the security implications of + // using a less reputable Go module I've decided to just call `gpg` via shell instead. + // Additionally this works and is just _so easy_ that it's probably not worth the effort to + // use another library that reinvents the wheel. + logrus.Debugf("Signing repo metadata at %q", filePath) + + // gpg --batch --yes --detach-sign --armor + _, err := BuildAndRunCommand("gpg", "--batch", "--yes", "--detach-sign", "--armor", filePath) + if err != nil { + return trace.Wrap(err, "failed to run GPG signing command on %q", filePath) + } + + return nil +} + +// Get the armored default public GPG key, ready to be written to a file +func (*GPG) GetPublicKey() (string, error) { + // For reference here is how another company formats their key: + // https://download.docker.com/linux/rhel/gpg + logrus.Debug("Attempting to get the default public GPG key") + + key, err := BuildAndRunCommand("gpg", "--export", "--armor", "--no-version") + if err != nil { + return "", trace.Wrap(err, "failed to export the default public GPG key") + } + + return key, nil +} + +func (g *GPG) WritePublicKeyToFile(filePath string) error { + logrus.Debugf("Writing the default armored public GPG key to %q", filePath) + + key, err := g.GetPublicKey() + if err != nil { + return trace.Wrap(err, "failed to retrieve public key") + } + + err = os.WriteFile(filePath, []byte(key), 0664) + if err != nil { + return trace.Wrap(err, "failed to write key to %q", filePath) + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/main.go b/build.assets/tooling/cmd/build-os-package-repos/main.go new file mode 100644 index 0000000000000..58f89f1dd1d16 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/main.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +func main() { + err := run() + if err != nil { + logrus.Fatal(err.Error()) + } +} + +func buildSubcommandRunners() ([]Runner, error) { + ar, err := NewAptRunner() + if err != nil { + return nil, trace.Wrap(err, "failed to instantiate new APT runner") + } + + yr, err := NewYumRunner() + if err != nil { + return nil, trace.Wrap(err, "failed to instantiate new YUM runner") + } + + // These should be sorted alphabetically by `Name()` + return []Runner{ + *ar, + *yr, + }, nil +} + +func run() error { + subcommands, err := buildSubcommandRunners() + if err != nil { + return trace.Wrap(err, "failed to build subcommand runners") + } + + // 2 = program name + subcommand + if len(os.Args) < 2 { + logHelp(subcommands) + return trace.Errorf("subcommand not provided") + } + + subcommandName := strings.ToLower(os.Args[1]) + for _, subcommand := range subcommands { + if strings.ToLower(subcommandName) != subcommand.Name() { + continue + } + + // 2 = program name + subcommand, skip them and get subcommand arguments + args := os.Args[2:] + err := subcommand.Init(args) + if err != nil { + return trace.Wrap(err, "failed to initialize runner for subcommand %q", subcommandName) + } + + setupLogger(subcommand.GetLoggerConfig()) + err = subcommand.Run() + if err != nil { + return trace.Wrap(err, "failed to run subcommand %q", subcommandName) + } + + return nil + } + + if subcommandName == "-h" { + logHelp(subcommands) + return nil + } + + logHelp(subcommands) + return trace.Errorf("no subcommands found matching %q", subcommandName) +} + +func logHelp(subcommands []Runner) { + executableName := os.Args[0] + fmt.Printf("%s - OS package repo builder/updater\n", executableName) + fmt.Println() + fmt.Println("Commands:") + fmt.Println() + for _, subcommand := range subcommands { + fmt.Printf("\t%s\t%s\n", subcommand.Name(), subcommand.Info()) + } + fmt.Println() + fmt.Printf("Use \"%s -h\" for more information about a command.\n", executableName) + fmt.Println() +} + +func setupLogger(config *LoggerConfig) { + if config.logJSON { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } else { + logrus.SetFormatter(&logrus.TextFormatter{}) + } + logrus.SetOutput(os.Stdout) + logrus.SetLevel(logrus.Level(config.logLevel)) + logrus.Debugf("Setup logger with config: %+v", config) +} diff --git a/build.assets/tooling/cmd/build-apt-repos/repo.go b/build.assets/tooling/cmd/build-os-package-repos/repo.go similarity index 100% rename from build.assets/tooling/cmd/build-apt-repos/repo.go rename to build.assets/tooling/cmd/build-os-package-repos/repo.go diff --git a/build.assets/tooling/cmd/build-os-package-repos/runners.go b/build.assets/tooling/cmd/build-os-package-repos/runners.go new file mode 100644 index 0000000000000..9ecda55eea2c8 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/runners.go @@ -0,0 +1,199 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + + "github.com/gravitational/trace" +) + +// Pattern from https://www.digitalocean.com/community/tutorials/how-to-use-the-flag-package-in-go +type Runner interface { + Init([]string) error + Run() error + GetLoggerConfig() *LoggerConfig + Name() string + Info() string +} + +// APT implementation +type AptRunner struct { + flags *flag.FlagSet + config *AptConfig + supportedOSs map[string][]string +} + +func NewAptRunner() (*AptRunner, error) { + runner := &AptRunner{ + supportedOSs: map[string][]string{ + "debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details + "stretch", // 9 + "buster", // 10 + "bullseye", // 11 + "bookwork", // 12 + "trixie", // 13 + }, + "ubuntu": { // See https://wiki.ubuntu.com/Releases for details + "xenial", // 16.04 LTS + "yakkety", // 16.10 (EOL) + "zesty", // 17.04 (EOL) + "artful", // 17.10 (EOL) + "bionic", // 18.04 LTS + "cosmic", // 18.10 (EOL) + "disco", // 19.04 (EOL) + "eoan", // 19.10 (EOL) + "focal", // 20.04 LTS + "groovy", // 20.10 (EOL) + "hirsuite", // 21.04 (EOL) + "impish", // 21.10 (EOL) + "jammy", // 22.04 LTS + }, + }, + } + + runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) + config, err := NewAptConfigWithFlagSet(runner.flags) + if err != nil { + return nil, trace.Wrap(err, "failed to create a new APT config instance") + } + + runner.config = config + + return runner, nil +} + +func (ar AptRunner) Init(args []string) error { + err := ar.flags.Parse(args) + if err != nil { + return trace.Wrap(err, "failed to parse arguments") + } + + err = ar.config.Check() + if err != nil { + return trace.Wrap(err, "failed to validate APT config arguments") + } + + return nil +} + +func (ar AptRunner) Run() error { + if ar.config.printHelp { + ar.flags.Usage() + return nil + } + + art, err := NewAptRepoTool(ar.config, ar.supportedOSs) + if err != nil { + return trace.Wrap(err, "failed to create a new APT repo tool instance") + } + + err = art.Run() + if err != nil { + return trace.Wrap(err, "APT runner failed") + } + + return nil +} + +func (AptRunner) Name() string { + return "apt" +} + +func (AptRunner) Info() string { + return "builds APT repos" +} + +func (ar AptRunner) GetLoggerConfig() *LoggerConfig { + return ar.config.LoggerConfig +} + +// YUM implementation +type YumRunner struct { + flags *flag.FlagSet + config *YumConfig + supportedOSs map[string][]string +} + +func NewYumRunner() (*YumRunner, error) { + runner := &YumRunner{ + supportedOSs: map[string][]string{ + "rhel": { // See https://access.redhat.com/articles/3078 for details + "7", + "8", + "9", + }, + "centos": { // See https://endoflife.date/centos for details + "7", + "8", + "9", + }, + // "$releasever" is a hot mess for Amazon Linux. No good documentation on this outside of just running + // a container or EC2 instance and manually checking $releasever values + "amzn": { + // "latest" // 1, aka 2018.03.0.20201028.0 + "2", // 2, aka 2.0.20201111.0 + // "2022.0.20220531" // 2022 (new naming scheme, preview) aka 2022.0.20220531 + }, + }, + } + + runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) + runner.config = NewYumConfigWithFlagSet(runner.flags) + + return runner, nil +} + +func (yr YumRunner) Init(args []string) error { + err := yr.flags.Parse(args) + if err != nil { + return trace.Wrap(err, "failed to parse arguments") + } + + err = yr.config.Check() + if err != nil { + return trace.Wrap(err, "failed to validate YUM config arguments") + } + + return nil +} + +func (yr YumRunner) Run() error { + yrt, err := NewYumRepoTool(yr.config, yr.supportedOSs) + if err != nil { + return trace.Wrap(err, "failed to create a new YUM repo tool instance") + } + + err = yrt.Run() + if err != nil { + return trace.Wrap(err, "YUM runner failed") + } + + return nil +} + +func (YumRunner) Name() string { + return "yum" +} + +func (YumRunner) Info() string { + return "builds YUM repos" +} + +func (yr YumRunner) GetLoggerConfig() *LoggerConfig { + return yr.config.LoggerConfig +} diff --git a/build.assets/tooling/cmd/build-apt-repos/s3logger.go b/build.assets/tooling/cmd/build-os-package-repos/s3logger.go similarity index 100% rename from build.assets/tooling/cmd/build-apt-repos/s3logger.go rename to build.assets/tooling/cmd/build-os-package-repos/s3logger.go diff --git a/build.assets/tooling/cmd/build-os-package-repos/s3manager.go b/build.assets/tooling/cmd/build-os-package-repos/s3manager.go new file mode 100644 index 0000000000000..00405efca9656 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/s3manager.go @@ -0,0 +1,511 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/gravitational/trace" + "github.com/inhies/go-bytesize" + "github.com/seqsense/s3sync" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +type S3manager struct { + syncManager *s3sync.Manager + uploader *s3manager.Uploader + downloader *s3manager.Downloader + bucketLocalPath string + bucketName string + bucketURL *url.URL + maxConcurrentSyncs int + downloadedBytes int64 +} + +func NewS3Manager(config *S3Config) (*S3manager, error) { + // Right now the AWS session is only used by this manager, but if it ends + // up being needed elsewhere then it should probably be moved to an arg + awsSession, err := session.NewSession() + if err != nil { + return nil, trace.Wrap(err, "failed to create a new AWS session") + } + + syncManagerMaxConcurrentSyncs := config.maxConcurrentSyncs + if syncManagerMaxConcurrentSyncs < 0 { + // This isn't unlimited but due to the s3sync library's parallelism implementation + // this must be limited to a "reasonable" number + syncManagerMaxConcurrentSyncs = 128 + } + + s := &S3manager{ + bucketName: config.bucketName, + bucketURL: &url.URL{ + Scheme: "s3", + Host: config.bucketName, + }, + syncManager: s3sync.New(awsSession, s3sync.WithParallel(syncManagerMaxConcurrentSyncs)), + uploader: s3manager.NewUploader(awsSession), + downloader: s3manager.NewDownloader(awsSession), + maxConcurrentSyncs: config.maxConcurrentSyncs, + } + s.ChangeLocalBucketPath(config.localBucketPath) + + s3sync.SetLogger(&s3logger{}) + + return s, nil +} + +func (s *S3manager) ChangeLocalBucketPath(newBucketPath string) error { + s.bucketLocalPath = newBucketPath + + // Ensure the local bucket path exists as it will be needed by all functions + err := os.MkdirAll(s.bucketLocalPath, 0660) + if err != nil { + return trace.Wrap(err, "failed to ensure path %q exists", s.bucketLocalPath) + } + + return nil +} + +func (s *S3manager) DownloadExistingRepo() error { + err := deleteAllFilesInDirectory(s.bucketLocalPath) + if err != nil { + return trace.Wrap(err, "failed to remove all filesystem entries in %q", s.bucketLocalPath) + } + + downloadGroup := &errgroup.Group{} + downloadGroup.SetLimit(s.maxConcurrentSyncs) + linkMap := make(map[string]string) + + var continuationToken *string + for { + listObjResponse, err := s.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &s.bucketName, + ContinuationToken: continuationToken, + }) + if err != nil { + return trace.Wrap(err, "failed to list objects for bucket %q", s.bucketName) + } + + for _, s3object := range listObjResponse.Contents { + s.processS3ObjectDownload(s3object, downloadGroup, &linkMap) + } + + continuationToken = listObjResponse.NextContinuationToken + if continuationToken == nil { + break + } + } + + // Even if an error has occurred we should wait to exit until all running syncs have + // completed, even if not successful + logrus.Info("Waiting for download to complete...") + err = downloadGroup.Wait() + if err != nil { + return trace.Wrap(err, "failed to perform S3 sync from remote bucket %q to local bucket %q", s.bucketName, s.bucketLocalPath) + } + + // Links must be created after their target exists + err = createLinks(linkMap) + if err != nil { + return trace.Wrap(err, "failed to create filesystem links for bucket %q", s.bucketName) + } + + logrus.Infof("Downloaded %s bytes", bytesize.New(float64(s.downloadedBytes))) + return nil +} + +func (s *S3manager) processS3ObjectDownload(s3object *s3.Object, downloadGroup *errgroup.Group, linkMap *map[string]string) { + downloadGroup.Go(func() error { + objectLink, err := s.getObjectLink(s3object) + if err != nil { + return trace.Wrap(err, "failed to get object link for key %q in bucket %q", *s3object.Key, s.bucketName) + } + + // If the link does not start with a '/' then it is not a filesystem link + if objectLink != nil && len(*objectLink) > 0 && (*objectLink)[0] == '/' { + localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) + linkTarget := filepath.Join(s.bucketLocalPath, *objectLink) + (*linkMap)[localObjectPath] = linkTarget + return nil + } + + err = s.downloadFile(s3object) + if err != nil { + return trace.Wrap(err, "failed to download S3 file %q from bucket %q", *s3object.Key, s.bucketName) + } + + return nil + }) +} + +func createLinks(linkMap map[string]string) error { + for file, target := range linkMap { + logrus.Infof("Creating a symlink from %q to %q", file, target) + err := os.MkdirAll(filepath.Dir(file), 0660) + if err != nil { + return trace.Wrap(err, "failed to create directory structure for %q", file) + } + + err = os.Symlink(target, file) + if err != nil { + return trace.Wrap(err, "failed to symlink %q to %q", file, target) + } + } + + return nil +} + +// This could potentially be made more efficient by running `os.RemoveAll` in a goroutine +// as random access on storage devices performs better at a higher queue depth +func deleteAllFilesInDirectory(dir string) error { + // Note that os.ReadDir does not follow/eval links which is important here + dirEntries, err := os.ReadDir(dir) + if err != nil { + return trace.Wrap(err, "failed to list directory entries for directory %q", dir) + } + + for _, dirEntry := range dirEntries { + dirEntryPath := filepath.Join(dir, dirEntry.Name()) + err = os.RemoveAll(dirEntryPath) + if err != nil { + return trace.Wrap(err, "failed to remove directory entry %q", dirEntryPath) + } + } + + return nil +} + +func (s *S3manager) getObjectLink(s3object *s3.Object) (*string, error) { + s3HeadObjectOutput, err := s.downloader.S3.HeadObject(&s3.HeadObjectInput{ + Bucket: &s.bucketName, + Key: s3object.Key, + // Probably unnecessary but this will cause an error to be thrown if somebody is + // modifying the object while this program is running + IfMatch: s3object.ETag, + IfUnmodifiedSince: s3object.LastModified, + }) + if err != nil { + return nil, trace.Wrap(err, "failed to retrieve metadata for key %q in bucket %q", *s3object.Key, s.bucketName) + } + + return s3HeadObjectOutput.WebsiteRedirectLocation, nil +} + +// s3sync has a bug when downloading a single file so this call reimplements s3sync's download +func (s *S3manager) downloadFile(s3object *s3.Object) error { + logrus.Infof("Downloading %q...", *s3object.Key) + localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) + + err := os.MkdirAll(filepath.Dir(localObjectPath), 0660) + if err != nil { + return trace.Wrap(err, "failed to create directory structure for %q", localObjectPath) + } + + fileWriter, err := os.Create(localObjectPath) + if err != nil { + return trace.Wrap(err, "failed to open %q for writing", localObjectPath) + } + defer fileWriter.Close() + + fileDownloadByteCount, err := s.downloader.Download(fileWriter, &s3.GetObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(*s3object.Key), + }) + if err != nil { + return trace.Wrap(err, "failed to download object %q from bucket %q to local path %q", *s3object.Key, s.bucketName, localObjectPath) + } + + s.downloadedBytes += fileDownloadByteCount + + err = os.Chtimes(localObjectPath, *s3object.LastModified, *s3object.LastModified) + if err != nil { + return trace.Wrap(err, "failed to update the access and modification time on file %q to %v", localObjectPath, *s3object.LastModified) + } + + logrus.Infof("Download %q complete", *s3object.Key) + return nil +} + +func (s *S3manager) UploadBuiltRepo() error { + err := s.sync(false) + if err != nil { + return trace.Wrap(err, "failed to upload bucket") + } + + return nil +} + +func (s *S3manager) UploadBuiltRepoWithRedirects(extensionToMatch, relativeRedirectDir string) error { + uploadGroup := &errgroup.Group{} + uploadGroup.SetLimit(s.maxConcurrentSyncs) + + walkErr := filepath.WalkDir(s.bucketLocalPath, func(absPath string, info fs.DirEntry, err error) error { + logrus.Debugf("Starting on %q...", absPath) + + if err != nil { + return trace.Wrap(err, "failed to walk over directory %q on path %q", s.bucketLocalPath) + } + + syncFunc, err := s.syncGenericFsObject(absPath, info) + if err != nil { + return trace.Wrap(err, "failed to get syncing function for %q", absPath) + } + + uploadGroup.Go(syncFunc) + logrus.Debugf("Upload for %q queued", absPath) + return nil + }) + + // Even if an error has occurred we should wait to exit until all running syncs have + // completed, even if not successful + logrus.Info("Waiting for sync to complete...") + syncErr := uploadGroup.Wait() + // Future work: add upload logging information once + // https://github.com/seqsense/s3sync/commit/29b3fcb259293d80634cb3916e0f28467d017087 has been released + logrus.Info("Sync has completed") + + errs := make([]error, 0, 2) + if walkErr != nil { + errs = append(errs, trace.Wrap(walkErr, "failed to walk over entries in %q", s.bucketLocalPath)) + } + + if syncErr != nil { + errs = append(errs, trace.Wrap(syncErr, "failed to perform S3 sync from local bucket %q to remote bucket %q", s.bucketLocalPath, s.bucketName)) + } + + if len(errs) > 0 { + return trace.Wrap(trace.NewAggregate(errs...), "one or more erros occurred while uploading built repo %q", s.bucketLocalPath) + } + + return nil +} + +func (s *S3manager) syncGenericFsObject(absPath string, dirEntryInfo fs.DirEntry) (func() error, error) { + // Don't do anything with non-empty directories as they will be caught later by their contents + if dirEntryInfo.IsDir() { + f, err := s.buildSyncDirFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build directory syncing function to sync %q", absPath) + } + + return f, nil + } else + // If symbolic link + if dirEntryInfo.Type()&fs.ModeSymlink != 0 { + f, err := s.buildSyncSymbolicLinkFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build symbolic link file syncing function to sync %q", absPath) + } + + return f, nil + } + + // sync a single file or directory + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncDirFunc(absPath string) (func() error, error) { + isDirEmpty, err := isDirectoryEmpty(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to determine if directory %q is empty", absPath) + } + + if !isDirEmpty { + logrus.Debug("Skipping non-empty directory") + return func() error { return nil }, nil + } + + // If the directory has no contents, call sync normally which will create the directory remotely if not exists + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncSymbolicLinkFunc(absPath string) (func() error, error) { + actualFilePath, err := filepath.EvalSymlinks(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to follow symlink for path %q", absPath) + } + + isInBucket, err := isPathChildOfAnother(s.bucketLocalPath, actualFilePath) + if err != nil { + return nil, trace.Wrap(err, "failed to determine if %q is a child of %q", actualFilePath, s.bucketLocalPath) + } + + if isInBucket { + // This will re-upload every redirect file ever created. Implementing "sync" functionality would + // require significantly more engineering effort and this cost is low so this shouldn't be a + // problem. + return func() error { + err := s.UploadRedirectFile(absPath, actualFilePath) + if err != nil { + return trace.Wrap(err, "failed to upload a redirect file to S3 for %q targeting %q", absPath, actualFilePath) + } + + return nil + }, nil + } + + // If not in bucket, call sync normally which will follow the symlink to the actual file and upload it + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncSingleFsEntryFunc(absPath string) (func() error, error) { + relPath, err := filepath.Rel(s.bucketLocalPath, absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to get %q relative to %q", absPath, s.bucketLocalPath) + } + + remoteURL := getURLWithPath(*s.bucketURL, relPath) + return func() error { + err := s.syncManager.Sync(absPath, remoteURL) + if err != nil { + return trace.Wrap(err, "failed to sync from %q to %q", absPath, remoteURL) + } + + return nil + }, nil +} + +func getURLWithPath(baseURL url.URL, path string) string { + // Because this function is pass-by-value it should not modify `baseUrl`, where doing this directly on the + // provided parameter would modify it + baseURL.Path = path + return baseURL.String() +} + +func isPathChildOfAnother(baseAbsPath string, testAbsPath string) (bool, error) { + // General implementation from https://stackoverflow.com/questions/28024731/check-if-given-path-is-a-subdirectory-of-another-in-golang + relPath, err := filepath.Rel(baseAbsPath, testAbsPath) + if err != nil { + return false, trace.Wrap(err, "failed to get the path of %q relative to %q", testAbsPath, baseAbsPath) + } + + return !strings.HasPrefix(relPath, fmt.Sprintf("..%c", os.PathSeparator)) && relPath != "..", nil +} + +func (s *S3manager) UploadRedirectFile(localAbsSrcPath, localAbsRemoteTargetPath string) error { + relSrcPath, err := filepath.Rel(s.bucketLocalPath, localAbsSrcPath) + if err != nil { + return trace.Wrap(err, "failed to get %q relative to %q", localAbsSrcPath, s.bucketLocalPath) + } + + relTargetPath, err := filepath.Rel(s.bucketLocalPath, localAbsRemoteTargetPath) + if err != nil { + return trace.Wrap(err, "failed to get %q relative to %q", localAbsRemoteTargetPath, s.bucketLocalPath) + } + + logrus.Infof("Creating a redirect file from %q to %q", relSrcPath, relTargetPath) + // S3 requires a prepended "/" to inform the redirect metadata that the target is another S3 object + // in the same bucket + s3TargetPath := filepath.Join("/", relTargetPath) + // Upload an empty file that when requested will redirect to the real one + _, err = s.uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.bucketName, + Key: &relSrcPath, + Body: bytes.NewReader([]byte{}), + WebsiteRedirectLocation: &s3TargetPath, + }) + if err != nil { + return trace.Wrap(err, "failed to upload an empty redirect file to %q in bucket %q", relSrcPath, s.bucketName) + } + + return nil +} + +func (s *S3manager) UploadRedirectURL(remoteAbsSourcePath, targetURL string) error { + logrus.Infof("Creating redirect from %q to %q", remoteAbsSourcePath, targetURL) + + _, err := s.uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.bucketName, + Key: &remoteAbsSourcePath, + Body: bytes.NewReader([]byte{}), + WebsiteRedirectLocation: &targetURL, + }) + + if err != nil { + return trace.Wrap(err, "failed to upload URL redirect file targeting %q to %q", targetURL, remoteAbsSourcePath) + } + + return nil +} + +func isDirectoryEmpty(dirPath string) (bool, error) { + // Pulled from https://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty + f, err := os.Open(dirPath) + if err != nil { + return false, trace.Wrap(err, "failed to open directory %q", dirPath) + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + + if err != nil { + return false, trace.Wrap(err, "failed to read the name of directories in %q", dirPath) + } + + return false, nil +} + +func (s *S3manager) sync(download bool) error { + var src, dest string + if download { + src = s.bucketURL.String() + dest = s.bucketLocalPath + } else { + src = s.bucketLocalPath + dest = s.bucketURL.String() + } + + logrus.Infof("Performing S3 sync from %q to %q...", src, dest) + err := s.syncManager.Sync(src, dest) + if err != nil { + return trace.Wrap(err, "failed to sync %q to %q", src, dest) + } + logrus.Infoln("S3 sync complete") + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh b/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh new file mode 100755 index 0000000000000..813c4e53d7a23 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# shellcheck disable=SC2016,SC1004,SC2174,SC2155 + +set -xeu + +# These must be set for the script to run +: "$AWS_ACCESS_KEY_ID" +: "$AWS_SECRET_ACCESS_KEY" +: "$AWS_SESSION_TOKEN" + +ART_VERSION_TAG="8.3.15" +ARTIFACT_PATH="/go/artifacts" +CACHE_DIR="/mnt/createrepo_cache" +GNUPGHOME="/tmpfs/gnupg" +REPO_S3_BUCKET="fred-test1" +BUCKET_CACHE_PATH="/mnt/bucket" +export AWS_REGION="us-west-2" + +: ' +Run command: +docker run \ + --rm -it \ + -v "$(git rev-parse --show-toplevel)":/go/src/github.com/gravitational/teleport/ \ + -v "$HOME/.aws":"/root/.aws" \ + -e AWS_PROFILE="$AWS_PROFILE" \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ + -e DEBIAN_FRONTEND="noninteractive" \ + golang:1.18.4-bullseye /go/src/github.com/gravitational/teleport/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh +' + +# Download the artifacts +apt update +apt install -y wget +mkdir -pv "$ARTIFACT_PATH" +cd "$ARTIFACT_PATH" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.x86_64.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm64.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.i386.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm.rpm" + +apt install -y createrepo-c gnupg +mkdir -pv "$CACHE_DIR" +mkdir -pv -m0700 "$GNUPGHOME" +chown -R root:root "$GNUPGHOME" +export GPG_TTY=$(tty) +gpg --batch --gen-key < This is a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)", - }, - }, - ) - steps = append(steps, getDroneTagVersionSteps(checkoutPath, aptVolumeName)...) - p.Steps = steps - - return p -} - -func migrateAptPipeline(triggerBranch string, migrationVersions []string) pipeline { - aptVolumeName := "aptrepo" - pipelineName := "migrate-apt-new-repos" - // DRONE_TAG is not available outside of promotion pipelines and will cause drone to fail with a - // "migrate-apt-new-repos: bad substitution" error if used here - checkoutPath := "/go/src/github.com/gravitational/teleport" - commitName := "${DRONE_COMMIT}" - - // If migrations are not configured then don't run - if triggerBranch == "" || len(migrationVersions) == 0 { - return buildNeverTriggerPipeline(pipelineName) - } - - p := buildBaseAptPipeline(pipelineName, aptVolumeName, checkoutPath, commitName) - p.Trigger = trigger{ - Repo: triggerRef{Include: []string{"gravitational/teleport"}}, - Event: triggerRef{Include: []string{"push"}}, - Branch: triggerRef{Include: []string{triggerBranch}}, - } - - for _, migrationVersion := range migrationVersions { - p.Steps = append(p.Steps, getVersionSteps(checkoutPath, migrationVersion, aptVolumeName)...) - } - - return p -} - -// Builds a pipeline that is syntactically correct but should never trigger to create -// a placeholder pipeline -func buildNeverTriggerPipeline(pipelineName string) pipeline { - p := newKubePipeline(pipelineName) - p.Trigger = trigger{ - Event: triggerRef{Include: []string{"custom"}}, - Repo: triggerRef{Include: []string{"non-existent-repository"}}, - Branch: triggerRef{Include: []string{"non-existent-branch"}}, - } - - p.Steps = []step{ - { - Name: "Placeholder", - Image: "alpine:latest", - Commands: []string{ - "echo \"This command, step, and pipeline never runs\"", - }, - }, - } - - return p -} - -// Functions that use this method should add at least: -// * a Trigger -// * Steps for checkout -func buildBaseAptPipeline(pipelineName, aptVolumeName, commit, checkoutPath string) pipeline { - p := newKubePipeline(pipelineName) - p.Workspace = workspace{Path: "/go"} - p.Volumes = []volume{ - { - Name: aptVolumeName, - Claim: &volumeClaim{ - Name: "drone-s3-aptrepo-pvc", - }, - }, - volumeTmpfs, - } - p.Steps = []step{ - { - Name: "Check out code", - Image: "alpine/git:latest", - Commands: aptToolCheckoutCommands(checkoutPath, commit), - }, - } - - return p -} - -func getDroneTagVersionSteps(codePath, aptVolumeName string) []step { - return getVersionSteps(codePath, "${DRONE_TAG}", aptVolumeName) -} - -// Version should start with a 'v', i.e. v1.2.3 or v9.0.1, or should be an environment var -// i.e. ${DRONE_TAG} -func getVersionSteps(codePath, version, aptVolumeName string) []step { - artifactPath := "/go/artifacts" - pvcMountPoint := "/mnt" - - var bucketFolder string - switch version[0:1] { - // If environment var - case "$": - // Remove the 'v' at runtime as the value isn't known at compile time - // This will change "${SOME_VAR}" to "${SOME_VAR##v}". `version` isn't actually - // an environment variable - it's a Drone substitution variable. See - // https://docs.drone.io/pipeline/environment/substitution/ for details. - bucketFolder = fmt.Sprintf("%s##v}", version[:len(version)-1]) - // If static string - case "v": - // Remove the 'v' at compile time as the value is known then - bucketFolder = version[1:] - } - - return []step{ - { - Name: fmt.Sprintf("Download artifacts for %q", version), - Image: "amazon/aws-cli", - Environment: map[string]value{ - "AWS_S3_BUCKET": { - fromSecret: "AWS_S3_BUCKET", - }, - "AWS_ACCESS_KEY_ID": { - fromSecret: "AWS_ACCESS_KEY_ID", - }, - "AWS_SECRET_ACCESS_KEY": { - fromSecret: "AWS_SECRET_ACCESS_KEY", - }, - "ARTIFACT_PATH": { - raw: artifactPath, - }, - }, - Commands: []string{ - "mkdir -pv \"$ARTIFACT_PATH\"", - strings.Join( - []string{ - "aws s3 sync", - "--no-progress", - "--delete", - "--exclude \"*\"", - "--include \"*.deb*\"", - fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder), - "\"$ARTIFACT_PATH\"", - }, - " ", - ), - }, - }, - { - Name: fmt.Sprintf("Publish debs to APT repos for %q", version), - // TODO set this if drongen `step` supports https://docs.drone.io/pipeline/ssh/syntax/parallelism/ in the future - // DependsOn: []string { - // "Check out code", - // "Download artifacts", - // }, - Image: "golang:1.18.1-bullseye", - Environment: map[string]value{ - "APT_S3_BUCKET": { - fromSecret: "APT_REPO_NEW_AWS_S3_BUCKET", - }, - "BUCKET_CACHE_PATH": { - // If we need to cache the bucket on the PVC for some reason in the future - // uncomment this line - // raw: path.Join(pvcMountPoint, "bucket-cache"), - raw: "/tmp/bucket", - }, - "AWS_REGION": { - raw: "us-west-2", - }, - "AWS_ACCESS_KEY_ID": { - fromSecret: "APT_REPO_NEW_AWS_ACCESS_KEY_ID", - }, - "AWS_SECRET_ACCESS_KEY": { - fromSecret: "APT_REPO_NEW_AWS_SECRET_ACCESS_KEY", - }, - "ARTIFACT_PATH": { - raw: artifactPath, - }, - "APTLY_ROOT_DIR": { - raw: path.Join(pvcMountPoint, "aptly"), - }, - "GNUPGHOME": { - raw: "/tmpfs/gnupg", - }, - "GPG_RPM_SIGNING_ARCHIVE": { - fromSecret: "GPG_RPM_SIGNING_ARCHIVE", - }, - }, - Commands: []string{ - "mkdir -pv -m0700 $GNUPGHOME", - "echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME", - "chown -R root:root $GNUPGHOME", - "apt update", - "apt install aptly tree -y", - fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")), - fmt.Sprintf("export VERSION=%q", version), - "export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured - // "rm -rf \"$APTLY_ROOT_DIR\"", // Uncomment this to completely dump the Aptly database and force a rebuild - strings.Join( - []string{ - // This just makes the (long) command a little more readable - "go run ./cmd/build-apt-repos", - "-bucket \"$APT_S3_BUCKET\"", - "-local-bucket-path \"$BUCKET_CACHE_PATH\"", - "-artifact-version \"$VERSION\"", - "-release-channel \"$RELEASE_CHANNEL\"", - "-aptly-root-dir \"$APTLY_ROOT_DIR\"", - "-artifact-path \"$ARTIFACT_PATH\"", - "-log-level 4", // Set this to 5 for debug logging - }, - " ", - ), - "rm -rf \"$BUCKET_CACHE_PATH\"", - "df -h \"$APTLY_ROOT_DIR\"", - }, - Volumes: []volumeRef{ - { - Name: aptVolumeName, - Path: pvcMountPoint, - }, - volumeRefTmpfs, - }, - }, - } -} - -// Note that tags are also valid here as a tag refers to a specific commit -func aptToolCheckoutCommands(commit, checkoutPath string) []string { - commands := []string{ - fmt.Sprintf("mkdir -p %q", checkoutPath), - fmt.Sprintf("cd %q", checkoutPath), - `git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git .`, - fmt.Sprintf("git checkout %q", commit), - } - return commands -} - func updateDocsPipeline() pipeline { // TODO: migrate return pipeline{} diff --git a/dronegen/os_repos.go b/dronegen/os_repos.go new file mode 100644 index 0000000000000..8df081ba37b59 --- /dev/null +++ b/dronegen/os_repos.go @@ -0,0 +1,460 @@ +// Copyright 2021 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "path" + "strings" +) + +func promoteBuildOsRepoPipelines() []pipeline { + aptPipeline := promoteAptPipeline() + yumPipeline := promoteYumPipeline() + return []pipeline{ + aptPipeline, + yumPipeline, + } +} + +// Used for one-off migrations of older versions. +// Use cases include: +// * We want to support another OS while providing backwards compatibility +// * We want to support another OS version while providing backwards compatibility +// * A customer wants to be able to install an older version via APT/YUM even if we +// no longer support it +// * RPM migrations after new YUM pipeline is done +func artifactMigrationPipeline() []pipeline { + migrationVersions := []string{ + // These versions were migrated as a part of the new `promoteAptPipeline` + // "v6.2.31", + // "v7.3.17", + // "v7.3.18", + // "v7.3.19", + // "v7.3.20", + // "v7.3.21", + // "v7.3.23", + // "v8.3.3", + // "v8.3.4", + // "v8.3.5", + // "v8.3.6", + // "v8.3.7", + // "v8.3.8", + // "v8.3.9", + // "v8.3.10", + // "v8.3.11", + // "v8.3.12", + // "v8.3.14", + // "v8.3.15", + // "v8.3.16", + // "v9.0.0", + // "v9.0.1", + // "v9.0.2", + // "v9.0.3", + // "v9.0.4", + // "v9.1.0", + // "v9.1.1", + // "v9.1.2", + // "v9.1.3", + // "v9.2.0", + // "v9.2.1", + // "v9.2.2", + // "v9.2.3", + // "v9.2.4", + // "v9.3.0", + // "v9.3.2", + // "v9.3.4", + // "v9.3.5", + // "v9.3.6", + // "v9.3.7", + // "v9.3.9", + // "v9.3.10", + // "v9.3.12", + // "v9.3.13", + // "v10.0.0", + // "v10.0.1", + // "v10.0.2", + } + // Pushing to this branch will trigger the listed versions to be migrated. Typically this should be + // the branch that these changes are being committed to. + migrationBranch := "" // "rfd/0058-package-distribution" + + aptPipeline := migrateAptPipeline(migrationBranch, migrationVersions) + yumPipeline := migrateYumPipeline(migrationBranch, migrationVersions) + return []pipeline{ + aptPipeline, + yumPipeline, + } +} + +type RepoBucketSecretNames struct { + bucketName string + accessKeyID string + secretAccessKey string +} + +func NewRepoBucketSecretNames(bucketName, accessKeyID, secretAccessKey string) *RepoBucketSecretNames { + return &RepoBucketSecretNames{ + bucketName: bucketName, + accessKeyID: accessKeyID, + secretAccessKey: secretAccessKey, + } +} + +type OsPackageToolPipelineBuilder struct { + clameName string + packageType string + packageManagerName string + volumeName string + pipelineNameSuffix string + artifactPath string + pvcMountPoint string + bucketSecrets *RepoBucketSecretNames + extraArgs []string + requiredPackages []string + setupCommands []string + environmentVars map[string]value +} + +// This function configures the build tool with it's requirements and sensible defaults. +// If additional configuration required then the returned struct should be modified prior +// to calling "build" functions on it. +func NewOsPackageToolPipelineBuilder(claimName, packageType, packageManagerName string, bucketSecrets *RepoBucketSecretNames) *OsPackageToolPipelineBuilder { + optpb := &OsPackageToolPipelineBuilder{ + clameName: claimName, + packageType: packageType, + packageManagerName: packageManagerName, + bucketSecrets: bucketSecrets, + extraArgs: []string{}, + setupCommands: []string{}, + requiredPackages: []string{}, + volumeName: fmt.Sprintf("%s-persistence", packageManagerName), + pipelineNameSuffix: fmt.Sprintf("%s-new-repos", packageManagerName), + artifactPath: "/go/artifacts", + pvcMountPoint: "/mnt", + } + + optpb.environmentVars = map[string]value{ + "REPO_S3_BUCKET": { + fromSecret: optpb.bucketSecrets.bucketName, + }, + "AWS_ACCESS_KEY_ID": { + fromSecret: optpb.bucketSecrets.accessKeyID, + }, + "AWS_SECRET_ACCESS_KEY": { + fromSecret: optpb.bucketSecrets.secretAccessKey, + }, + "AWS_REGION": { + raw: "us-west-2", + }, + "BUCKET_CACHE_PATH": { + // If we need to cache the bucket on the PVC for some reason in the future + // uncomment this line + // raw: path.Join(pvcMountPoint, "bucket-cache"), + raw: "/tmp/bucket", + }, + "ARTIFACT_PATH": { + raw: optpb.artifactPath, + }, + "GNUPGHOME": { + raw: "/tmpfs/gnupg", + }, + "GPG_RPM_SIGNING_ARCHIVE": { + fromSecret: "GPG_RPM_SIGNING_ARCHIVE", + }, + "DEBIAN_FRONTEND": { + raw: "noninteractive", + }, + } + + return optpb +} + +func (optpb *OsPackageToolPipelineBuilder) buildPromoteOsPackagePipeline() pipeline { + pipelineName := fmt.Sprintf("publish-%s", optpb.pipelineNameSuffix) + checkoutPath := "/go/src/github.com/gravitational/teleport" + commitName := "${DRONE_TAG}" + checkoutStepName := "Check out code" + + p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutStepName, checkoutPath, commitName) + p.Trigger = triggerPromote + p.Trigger.Repo.Include = []string{"gravitational/teleport"} + + setupSteps := []step{ + { + Name: "Verify build is tagged", + Image: "alpine:latest", + Commands: []string{ + "[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)", + }, + }, + } + setupSteps = append(setupSteps, p.Steps...) + setupSteps = append(setupSteps, + step{ + Name: "Check if tag is prerelease", + Image: "golang:1.17-alpine", + Commands: []string{ + fmt.Sprintf("cd %q", path.Join(checkoutPath, "build.assets", "tooling")), + "go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78)", + }, + }, + ) + + setupStepNames := make([]string, 0, len(setupSteps)) + for _, setupStep := range setupSteps { + setupStepNames = append(setupStepNames, setupStep.Name) + } + + versionSteps := optpb.getDroneTagVersionSteps(checkoutPath) + for i := range versionSteps { + versionStep := &versionSteps[i] + if versionStep.DependsOn == nil { + versionStep.DependsOn = setupStepNames + continue + } + + versionStep.DependsOn = append(versionStep.DependsOn, setupStepNames...) + } + + p.Steps = append(setupSteps, versionSteps...) + + return p +} + +func (optpb *OsPackageToolPipelineBuilder) buildMigrateOsPackagePipeline(triggerBranch string, migrationVersions []string) pipeline { + pipelineName := fmt.Sprintf("migrate-%s", optpb.pipelineNameSuffix) + checkoutPath := "/go/src/github.com/gravitational/teleport" + // DRONE_TAG is not available outside of promotion pipelines and will cause drone to fail with a + // "migrate-apt-new-repos: bad substitution" error if used here + commitName := "${DRONE_COMMIT}" + checkoutStepName := "Check out code" + + // If migrations are not configured then don't run + if triggerBranch == "" || len(migrationVersions) == 0 { + return buildNeverTriggerPipeline(pipelineName) + } + + p := optpb.buildBaseOsPackagePipeline(pipelineName, checkoutStepName, checkoutPath, commitName) + p.Trigger = trigger{ + Repo: triggerRef{Include: []string{"gravitational/teleport"}}, + Event: triggerRef{Include: []string{"push"}}, + Branch: triggerRef{Include: []string{triggerBranch}}, + } + + for _, migrationVersion := range migrationVersions { + // Not enabling parallelism here so that multiple migrations don't run at once + p.Steps = append(p.Steps, optpb.getVersionSteps(checkoutPath, migrationVersion, false)...) + } + + setStepResourceLimits(p.Steps) + + return p +} + +// Builds a pipeline that is syntactically correct but should never trigger to create +// a placeholder pipeline +func buildNeverTriggerPipeline(pipelineName string) pipeline { + p := newKubePipeline(pipelineName) + p.Trigger = trigger{ + Event: triggerRef{Include: []string{"custom"}}, + Repo: triggerRef{Include: []string{"non-existent-repository"}}, + Branch: triggerRef{Include: []string{"non-existent-branch"}}, + } + + p.Steps = []step{ + { + Name: "Placeholder", + Image: "alpine:latest", + Commands: []string{ + "echo \"This command, step, and pipeline never runs\"", + }, + }, + } + + return p +} + +// Functions that use this method should add at least: +// * a Trigger +// * Steps for checkout +func (optpb *OsPackageToolPipelineBuilder) buildBaseOsPackagePipeline(pipelineName, checkoutStepName, checkoutPath, commit string) pipeline { + p := newKubePipeline(pipelineName) + p.Workspace = workspace{Path: "/go"} + p.Volumes = []volume{ + { + Name: optpb.volumeName, + Claim: &volumeClaim{ + Name: optpb.clameName, + }, + }, + volumeTmpfs, + } + p.Steps = []step{ + { + Name: checkoutStepName, + Image: "alpine/git:latest", + Commands: toolCheckoutCommands(checkoutPath, commit), + }, + } + setStepResourceLimits(p.Steps) + + return p +} + +func setStepResourceLimits(steps []step) { + // Not currently supported + // for i := range steps { + // step := &steps[i] + // if step.Resources == nil { + // step.Resources = &containerResources{} + // } + + // if step.Resources.Requests == nil { + // step.Resources.Requests = &resourceSet{} + // } + + // step.Resources.Requests.Cpu = 100 + // step.Resources.Requests.Memory = (*resourceQuantity)(resource.NewQuantity(100*1024*1024, resource.BinarySI)) + // } +} + +// Note that tags are also valid here as a tag refers to a specific commit +func toolCheckoutCommands(checkoutPath, commit string) []string { + commands := []string{ + fmt.Sprintf("mkdir -p %q", checkoutPath), + fmt.Sprintf("cd %q", checkoutPath), + `git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git .`, + fmt.Sprintf("git checkout %q", commit), + } + return commands +} + +func (optpb *OsPackageToolPipelineBuilder) getDroneTagVersionSteps(codePath string) []step { + return optpb.getVersionSteps(codePath, "${DRONE_TAG}", true) +} + +// Version should start with a 'v', i.e. v1.2.3 or v9.0.1, or should be an environment var +// i.e. ${DRONE_TAG} +func (optpb *OsPackageToolPipelineBuilder) getVersionSteps(codePath, version string, enableParallelism bool) []step { + var bucketFolder string + switch version[0:1] { + // If environment var + case "$": + // Remove the 'v' at runtime as the value isn't known at compile time + // This will change "${SOME_VAR}" to "${SOME_VAR##v}". `version` isn't actually + // an environment variable - it's a Drone substitution variable. See + // https://docs.drone.io/pipeline/environment/substitution/ for details. + bucketFolder = fmt.Sprintf("%s##v}", version[:len(version)-1]) + // If static string + case "v": + // Remove the 'v' at compile time as the value is known then + bucketFolder = version[1:] + } + + toolSetupCommands := []string{} + if len(optpb.requiredPackages) > 0 { + toolSetupCommands = []string{ + "apt update", + fmt.Sprintf("apt install -y %s", strings.Join(optpb.requiredPackages, " ")), + } + } + toolSetupCommands = append(toolSetupCommands, optpb.setupCommands...) + + downloadStepName := fmt.Sprintf("Download artifacts for %q", version) + buildStepDependencies := []string{} + if enableParallelism { + buildStepDependencies = append(buildStepDependencies, downloadStepName) + } + + return []step{ + { + Name: downloadStepName, + Image: "amazon/aws-cli", + Environment: map[string]value{ + "AWS_S3_BUCKET": { + fromSecret: "AWS_S3_BUCKET", + }, + "AWS_ACCESS_KEY_ID": { + fromSecret: "AWS_ACCESS_KEY_ID", + }, + "AWS_SECRET_ACCESS_KEY": { + fromSecret: "AWS_SECRET_ACCESS_KEY", + }, + "ARTIFACT_PATH": { + raw: optpb.artifactPath, + }, + }, + Commands: []string{ + "mkdir -pv \"$ARTIFACT_PATH\"", + // Clear out old versions from previous steps + "rm -rf \"${ARTIFACT_PATH}/*\"", + strings.Join( + []string{ + "aws s3 sync", + "--no-progress", + "--delete", + "--exclude \"*\"", + fmt.Sprintf("--include \"*.%s*\"", optpb.packageType), + fmt.Sprintf("s3://$AWS_S3_BUCKET/teleport/tag/%s/", bucketFolder), + "\"$ARTIFACT_PATH\"", + }, + " ", + ), + }, + }, + { + Name: fmt.Sprintf("Publish %ss to %s repos for %q", optpb.packageType, strings.ToUpper(optpb.packageManagerName), version), + Image: "golang:1.18.4-bullseye", + Environment: optpb.environmentVars, + Commands: append( + toolSetupCommands, + []string{ + "mkdir -pv -m0700 \"$GNUPGHOME\"", + "echo \"$GPG_RPM_SIGNING_ARCHIVE\" | base64 -d | tar -xzf - -C $GNUPGHOME", + "chown -R root:root \"$GNUPGHOME\"", + fmt.Sprintf("cd %q", path.Join(codePath, "build.assets", "tooling")), + fmt.Sprintf("export VERSION=%q", version), + "export RELEASE_CHANNEL=\"stable\"", // The tool supports several release channels but I'm not sure where this should be configured + strings.Join( + append( + []string{ + // This just makes the (long) command a little more readable + "go run ./cmd/build-os-package-repos", + optpb.packageManagerName, + "-bucket \"$REPO_S3_BUCKET\"", + "-local-bucket-path \"$BUCKET_CACHE_PATH\"", + "-artifact-version \"$VERSION\"", + "-release-channel \"$RELEASE_CHANNEL\"", + "-artifact-path \"$ARTIFACT_PATH\"", + "-log-level 4", // Set this to 5 for debug logging + }, + optpb.extraArgs..., + ), + " ", + ), + }..., + ), + Volumes: []volumeRef{ + { + Name: optpb.volumeName, + Path: optpb.pvcMountPoint, + }, + volumeRefTmpfs, + }, + DependsOn: buildStepDependencies, + }, + } +} diff --git a/dronegen/promote.go b/dronegen/promote.go index fe5db5b9abeaa..a4193dd871ea1 100644 --- a/dronegen/promote.go +++ b/dronegen/promote.go @@ -17,10 +17,11 @@ package main import "fmt" func promoteBuildPipelines() []pipeline { - aptPipeline := promoteAptPipeline() - dockerPipelineECR := buildDockerPromotionPipelineECR() - dockerPipelineQuay := buildDockerPromotionPipelineQuay() - return []pipeline{aptPipeline, dockerPipelineECR, dockerPipelineQuay} + promotePipelines := make([]pipeline, 0) + promotePipelines = append(promotePipelines, promoteBuildOsRepoPipelines()...) + promotePipelines = append(promotePipelines, buildDockerPromotionPipelineECR(), buildDockerPromotionPipelineQuay()) + + return promotePipelines } func buildDockerPromotionPipelineECR() pipeline { diff --git a/dronegen/types.go b/dronegen/types.go index 65d2816632f5a..f86596547555e 100644 --- a/dronegen/types.go +++ b/dronegen/types.go @@ -19,6 +19,8 @@ import ( "fmt" "runtime" "strings" + + "k8s.io/apimachinery/pkg/api/resource" ) // Types to mirror the YAML fields of the drone config. @@ -159,15 +161,17 @@ type volumeRef struct { } type step struct { - Name string `yaml:"name"` - Image string `yaml:"image,omitempty"` - Commands []string `yaml:"commands,omitempty"` - Environment map[string]value `yaml:"environment,omitempty"` - Volumes []volumeRef `yaml:"volumes,omitempty"` - Settings map[string]value `yaml:"settings,omitempty"` - Template []string `yaml:"template,omitempty"` - When *condition `yaml:"when,omitempty"` - Failure string `yaml:"failure,omitempty"` + Name string `yaml:"name"` + Image string `yaml:"image,omitempty"` + Commands []string `yaml:"commands,omitempty"` + Environment map[string]value `yaml:"environment,omitempty"` + Volumes []volumeRef `yaml:"volumes,omitempty"` + Settings map[string]value `yaml:"settings,omitempty"` + Template []string `yaml:"template,omitempty"` + When *condition `yaml:"when,omitempty"` + Failure string `yaml:"failure,omitempty"` + Resources *containerResources `yaml:"resources,omitempty"` + DependsOn []string `yaml:"depends_on,omitempty"` } type condition struct { @@ -210,3 +214,43 @@ func (v *value) UnmarshalYAML(unmarshal func(interface{}) error) error { } return errors.New("can't unmarshal the value as either string or from_secret reference") } + +type containerResources struct { + Limits *resourceSet `yaml:"limits,omitempty"` + // Not currently supported + // Requests *resourceSet `yaml:"requests,omitempty"` +} + +type resourceSet struct { + // Drone does not strictly follow the k8s CRD format for resources here + // See link for details: + // https://docs.drone.io/pipeline/kubernetes/syntax/steps/#resources + // CPU *resourceQuantity `yaml:"cpu,omitempty"` + + CPU float64 `yaml:"cpu,omitempty"` + Memory *resourceQuantity `yaml:"memory,omitempty"` +} + +// This is a workaround to get resource.Quantity to unmarshal correctly +type resourceQuantity resource.Quantity + +func (rq *resourceQuantity) MarshalYAML() (interface{}, error) { + return ((*resource.Quantity)(rq)).String(), nil +} + +func (rq *resourceQuantity) UnmarshalYAML(unmarshal func(interface{}) error) error { + var value string + if err := unmarshal(&value); err != nil { + return errors.New("failed to unmarshal the value into a string") + } + + parsedValue, err := resource.ParseQuantity(value) + if err != nil { + return fmt.Errorf("failed to unmarshal string %q into resource quantity", value) + } + + q := ((*resource.Quantity)(rq)) + q.Add(parsedValue) + + return nil +} diff --git a/dronegen/yum.go b/dronegen/yum.go new file mode 100644 index 0000000000000..b65078f9c7475 --- /dev/null +++ b/dronegen/yum.go @@ -0,0 +1,62 @@ +// Copyright 2021 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "path" +) + +// This function calls the build-apt-repos tool which handles the APT portion of RFD 0058. +func promoteYumPipeline() pipeline { + return getYumPipelineBuilder().buildPromoteOsPackagePipeline() +} + +func migrateYumPipeline(triggerBranch string, migrationVersions []string) pipeline { + return getYumPipelineBuilder().buildMigrateOsPackagePipeline(triggerBranch, migrationVersions) +} + +func getYumPipelineBuilder() *OsPackageToolPipelineBuilder { + optpb := NewOsPackageToolPipelineBuilder( + "drone-s3-yumrepo-pvc", + "rpm", + "yum", + NewRepoBucketSecretNames( + "YUM_REPO_NEW_AWS_S3_BUCKET", + "YUM_REPO_NEW_AWS_ACCESS_KEY_ID", + "YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY", + ), + ) + + optpb.environmentVars["CACHE_DIR"] = value{ + raw: path.Join(optpb.pvcMountPoint, "createrepo_cache"), + } + optpb.environmentVars["BUCKET_CACHE_PATH"] = value{ + raw: path.Join(optpb.pvcMountPoint, "bucket"), + } + + optpb.requiredPackages = []string{ + "createrepo-c", + } + + optpb.setupCommands = []string{ + "mkdir -pv \"$CACHE_DIR\"", + } + + optpb.extraArgs = []string{ + "-cache-dir \"$CACHE_DIR\"", + } + + return optpb +}