From baed862f7e57c6eec13d4f4a2d2f3aa3e235f661 Mon Sep 17 00:00:00 2001 From: Fred Heinecke Date: Tue, 9 Aug 2022 11:38:21 -0500 Subject: [PATCH] Added YUM implementation of OS package build tool (#14203) * Added YUM implementation of OS package build tool * Addressed PR comments * Added YUM migrations * Added curl to YUM dependencies * Changed pipelines to use golang:1.18.4-bullseye for Go * Implemented proper repo downloading logic * Fixed other merge conflicts * Added artifacts cleanup * Removed delete on s3 sync * Added RPM migrations * v8 migrations * Partial v8 migration * Migration remainder * Reduced requested resources * Updated resource limits per step * Added k8s stage resource limits to drone * Fixed format issue * Removed resource requests * Added `depends_on` support to dronegen * v8.3 migrations * Fixed parallelism * Removed migration parallelism * Fixed RPM base arch lookup * v6 and v7 YUM migration * Fixed missing ISA * Updated repo file path * Added logging * Removed vars from repo file * v8.3 migration first batch * v8.3 migration second batch * v9.0 migration * v9.1 migration * v9.2 migration * v9.3 first migration * v9.3 second migration * v10.0 migration * Removed migrations * Disabled shell linting non-issues * Fixed linter problem * More linter fixes --- .drone.yml | 194 ++++++- .../tooling/cmd/build-apt-repos/config.go | 159 ------ .../tooling/cmd/build-apt-repos/main.go | 78 --- .../tooling/cmd/build-apt-repos/s3manager.go | 102 ---- .../apt_repo_tool.go | 50 +- .../aptly.go | 45 +- .../command_executor.go | 53 ++ .../cmd/build-os-package-repos/config.go | 282 ++++++++++ .../cmd/build-os-package-repos/createrepo.go | 83 +++ .../tooling/cmd/build-os-package-repos/gpg.go | 119 ++++ .../cmd/build-os-package-repos/main.go | 119 ++++ .../repo.go | 0 .../cmd/build-os-package-repos/runners.go | 199 +++++++ .../s3logger.go | 0 .../cmd/build-os-package-repos/s3manager.go | 511 +++++++++++++++++ .../cmd/build-os-package-repos/test-rpm.sh | 63 +++ .../build-os-package-repos/yum_repo_tool.go | 512 ++++++++++++++++++ build.assets/tooling/go.mod | 15 +- build.assets/tooling/go.sum | 31 +- 19 files changed, 2186 insertions(+), 429 deletions(-) delete mode 100644 build.assets/tooling/cmd/build-apt-repos/config.go delete mode 100644 build.assets/tooling/cmd/build-apt-repos/main.go delete mode 100644 build.assets/tooling/cmd/build-apt-repos/s3manager.go rename build.assets/tooling/cmd/{build-apt-repos => build-os-package-repos}/apt_repo_tool.go (82%) rename build.assets/tooling/cmd/{build-apt-repos => build-os-package-repos}/aptly.go (94%) create mode 100644 build.assets/tooling/cmd/build-os-package-repos/command_executor.go create mode 100644 build.assets/tooling/cmd/build-os-package-repos/config.go create mode 100644 build.assets/tooling/cmd/build-os-package-repos/createrepo.go create mode 100644 build.assets/tooling/cmd/build-os-package-repos/gpg.go create mode 100644 build.assets/tooling/cmd/build-os-package-repos/main.go rename build.assets/tooling/cmd/{build-apt-repos => build-os-package-repos}/repo.go (100%) create mode 100644 build.assets/tooling/cmd/build-os-package-repos/runners.go rename build.assets/tooling/cmd/{build-apt-repos => build-os-package-repos}/s3logger.go (100%) create mode 100644 build.assets/tooling/cmd/build-os-package-repos/s3manager.go create mode 100755 build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh create mode 100644 build.assets/tooling/cmd/build-os-package-repos/yum_repo_tool.go diff --git a/.drone.yml b/.drone.yml index 619fd8f82f289..cecfd10527a57 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5452,11 +5452,9 @@ volumes: --- ################################################ -# This was originally generated using dronegen, -# but it looks like dronegen was not backported -# to v8. -# Generated at dronegen/misc.go:149 on commit -# f4b0ae4a2abe9c17622aee99b20be1220d3a8414 +# Generated using dronegen, do not edit by hand! +# Use 'make dronegen' to update. +# Generated at dronegen/os_repos.go:270 ################################################ kind: pipeline @@ -5482,11 +5480,9 @@ steps: --- ################################################ -# This was originally generated using dronegen, -# but it looks like dronegen was not backported -# to v8. -# Generated at dronegen/misc.go:169 on commit -# f4b0ae4a2abe9c17622aee99b20be1220d3a8414 +# Generated using dronegen, do not edit by hand! +# Use 'make dronegen' to update. +# Generated at dronegen/os_repos.go:294 ################################################ kind: pipeline @@ -5529,6 +5525,7 @@ steps: image: amazon/aws-cli commands: - mkdir -pv "$ARTIFACT_PATH" + - rm -rf "${ARTIFACT_PATH}/*" - aws s3 sync --no-progress --delete --exclude "*" --include "*.deb*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ "$ARTIFACT_PATH" environment: @@ -5539,25 +5536,25 @@ steps: from_secret: AWS_S3_BUCKET AWS_SECRET_ACCESS_KEY: from_secret: AWS_SECRET_ACCESS_KEY + depends_on: + - Verify build is tagged + - Check out code + - Check if tag is prerelease - name: Publish debs to APT repos for "${DRONE_TAG}" - image: golang:1.18.1-bullseye + image: golang:1.18.4-bullseye commands: - - mkdir -pv -m0700 $GNUPGHOME - - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME - - chown -R root:root $GNUPGHOME - apt update - - apt install aptly tree -y + - apt install -y aptly + - mkdir -pv -m0700 "$GNUPGHOME" + - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME + - chown -R root:root "$GNUPGHOME" - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" - export VERSION="${DRONE_TAG}" - export RELEASE_CHANNEL="stable" - - go run ./cmd/build-apt-repos -bucket "$APT_S3_BUCKET" -local-bucket-path "$BUCKET_CACHE_PATH" - -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" -aptly-root-dir - "$APTLY_ROOT_DIR" -artifact-path "$ARTIFACT_PATH" -log-level 4 - - rm -rf "$BUCKET_CACHE_PATH" - - df -h "$APTLY_ROOT_DIR" + - go run ./cmd/build-os-package-repos apt -bucket "$REPO_S3_BUCKET" -local-bucket-path + "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" + -artifact-path "$ARTIFACT_PATH" -log-level 4 -aptly-root-dir "$APTLY_ROOT_DIR" environment: - APT_S3_BUCKET: - from_secret: APT_REPO_NEW_AWS_S3_BUCKET APTLY_ROOT_DIR: /mnt/aptly ARTIFACT_PATH: /go/artifacts AWS_ACCESS_KEY_ID: @@ -5566,16 +5563,24 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: APT_REPO_NEW_AWS_SECRET_ACCESS_KEY BUCKET_CACHE_PATH: /tmp/bucket + DEBIAN_FRONTEND: noninteractive GNUPGHOME: /tmpfs/gnupg GPG_RPM_SIGNING_ARCHIVE: from_secret: GPG_RPM_SIGNING_ARCHIVE + REPO_S3_BUCKET: + from_secret: APT_REPO_NEW_AWS_S3_BUCKET volumes: - - name: aptrepo + - name: apt-persistence path: /mnt - name: tmpfs path: /tmpfs + depends_on: + - Download artifacts for "${DRONE_TAG}" + - Verify build is tagged + - Check out code + - Check if tag is prerelease volumes: -- name: aptrepo +- name: apt-persistence claim: name: drone-s3-aptrepo-pvc - name: tmpfs @@ -5586,7 +5591,142 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/promote.go:81 +# Generated at dronegen/os_repos.go:270 +################################################ + +kind: pipeline +type: kubernetes +name: migrate-yum-new-repos +trigger: + event: + include: + - custom + repo: + include: + - non-existent-repository + branch: + include: + - non-existent-branch +clone: + disable: true +steps: +- name: Placeholder + image: alpine:latest + commands: + - echo "This command, step, and pipeline never runs" + +--- +################################################ +# Generated using dronegen, do not edit by hand! +# Use 'make dronegen' to update. +# Generated at dronegen/os_repos.go:294 +################################################ + +kind: pipeline +type: kubernetes +name: publish-yum-new-repos +trigger: + event: + include: + - promote + target: + include: + - production + repo: + include: + - gravitational/teleport +workspace: + path: /go +clone: + disable: true +steps: +- name: Verify build is tagged + image: alpine:latest + commands: + - '[ -n ${DRONE_TAG} ] || (echo ''DRONE_TAG is not set. Is the commit tagged?'' + && exit 1)' +- name: Check out code + image: alpine/git:latest + commands: + - mkdir -p "/go/src/github.com/gravitational/teleport" + - cd "/go/src/github.com/gravitational/teleport" + - git clone https://github.com/gravitational/${DRONE_REPO_NAME}.git . + - git checkout "${DRONE_TAG}" +- name: Check if tag is prerelease + image: golang:1.17-alpine + commands: + - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" + - go run ./cmd/check -tag ${DRONE_TAG} -check prerelease || (echo '---> This is + a prerelease, not publishing ${DRONE_TAG} packages to APT repos' && exit 78) +- name: Download artifacts for "${DRONE_TAG}" + image: amazon/aws-cli + commands: + - mkdir -pv "$ARTIFACT_PATH" + - rm -rf "${ARTIFACT_PATH}/*" + - aws s3 sync --no-progress --delete --exclude "*" --include "*.rpm*" s3://$AWS_S3_BUCKET/teleport/tag/${DRONE_TAG##v}/ + "$ARTIFACT_PATH" + environment: + ARTIFACT_PATH: /go/artifacts + AWS_ACCESS_KEY_ID: + from_secret: AWS_ACCESS_KEY_ID + AWS_S3_BUCKET: + from_secret: AWS_S3_BUCKET + AWS_SECRET_ACCESS_KEY: + from_secret: AWS_SECRET_ACCESS_KEY + depends_on: + - Verify build is tagged + - Check out code + - Check if tag is prerelease +- name: Publish rpms to YUM repos for "${DRONE_TAG}" + image: golang:1.18.4-bullseye + commands: + - apt update + - apt install -y createrepo-c + - mkdir -pv "$CACHE_DIR" + - mkdir -pv -m0700 "$GNUPGHOME" + - echo "$GPG_RPM_SIGNING_ARCHIVE" | base64 -d | tar -xzf - -C $GNUPGHOME + - chown -R root:root "$GNUPGHOME" + - cd "/go/src/github.com/gravitational/teleport/build.assets/tooling" + - export VERSION="${DRONE_TAG}" + - export RELEASE_CHANNEL="stable" + - go run ./cmd/build-os-package-repos yum -bucket "$REPO_S3_BUCKET" -local-bucket-path + "$BUCKET_CACHE_PATH" -artifact-version "$VERSION" -release-channel "$RELEASE_CHANNEL" + -artifact-path "$ARTIFACT_PATH" -log-level 4 -cache-dir "$CACHE_DIR" + environment: + ARTIFACT_PATH: /go/artifacts + AWS_ACCESS_KEY_ID: + from_secret: YUM_REPO_NEW_AWS_ACCESS_KEY_ID + AWS_REGION: us-west-2 + AWS_SECRET_ACCESS_KEY: + from_secret: YUM_REPO_NEW_AWS_SECRET_ACCESS_KEY + BUCKET_CACHE_PATH: /mnt/bucket + CACHE_DIR: /mnt/createrepo_cache + DEBIAN_FRONTEND: noninteractive + GNUPGHOME: /tmpfs/gnupg + GPG_RPM_SIGNING_ARCHIVE: + from_secret: GPG_RPM_SIGNING_ARCHIVE + REPO_S3_BUCKET: + from_secret: YUM_REPO_NEW_AWS_S3_BUCKET + volumes: + - name: yum-persistence + path: /mnt + - name: tmpfs + path: /tmpfs + depends_on: + - Download artifacts for "${DRONE_TAG}" + - Verify build is tagged + - Check out code + - Check if tag is prerelease +volumes: +- name: yum-persistence + claim: + name: drone-s3-yumrepo-pvc +- name: tmpfs + temp: + medium: memory + +--- + ################################################ kind: pipeline @@ -5672,7 +5812,7 @@ volumes: ################################################ # Generated using dronegen, do not edit by hand! # Use 'make dronegen' to update. -# Generated at dronegen/promote.go:27 +# Generated at dronegen/promote.go:82 ################################################ kind: pipeline @@ -6013,6 +6153,6 @@ volumes: name: drone-s3-debrepo-pvc --- kind: signature -hmac: 6b48fad3cafc583fd0d4b0848ef612bfd0d2a1acd225642f74369c1c8815eb9c +hmac: 21f4465cef6462826ea13e7fa3b5699e5f3bb52f58bfe03b09dd5d8234229537 ... diff --git a/build.assets/tooling/cmd/build-apt-repos/config.go b/build.assets/tooling/cmd/build-apt-repos/config.go deleted file mode 100644 index e4ea43cfc7835..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/config.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - "strings" - - "github.com/gravitational/trace" - "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" -) - -const StableChannelFlagValue string = "stable" - -type Config struct { - artifactPath string - artifactVersion string - bucketName string - localBucketPath string - releaseChannel string - aptlyPath string - logLevel uint - logJSON bool -} - -// Parses and validates the provided flags, returning the parsed arguments in a struct. -func ParseFlags() (*Config, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, trace.Wrap(err, "failed to get user's home directory path") - } - - config := &Config{} - flag.StringVar(&config.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb files to add to the APT repos") - flag.StringVar(&config.artifactVersion, "artifact-version", "", "The version of the artifacts that will be added to the APT repos") - flag.StringVar(&config.releaseChannel, "release-channel", "", "The release channel of the APT repos that the artifacts should be added to") - flag.StringVar(&config.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from") - flag.StringVar(&config.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to") - flag.StringVar(&config.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)") - flag.UintVar(&config.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose") - flag.BoolVar(&config.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging") - - flag.Parse() - if err := Check(config); err != nil { - return nil, trace.Wrap(err, "failed to validate flags") - } - - return config, nil -} - -func Check(config *Config) error { - if err := validateArtifactPath(config.artifactPath); err != nil { - return trace.Wrap(err, "failed to validate the artifact path flag") - } - if err := validateBucketName(config.bucketName); err != nil { - return trace.Wrap(err, "failed to validate the bucket name flag") - } - if err := validateLocalBucketPath(config.localBucketPath); err != nil { - return trace.Wrap(err, "failed to validate the local bucket path flag") - } - if err := validateArtifactVersion(config.artifactVersion); err != nil { - return trace.Wrap(err, "failed to validate the artifact version flag") - } - if err := validateReleaseChannel(config.releaseChannel); err != nil { - return trace.Wrap(err, "failed to validate the release channel flag") - } - if err := validateLogLevel(config.logLevel); err != nil { - return trace.Wrap(err, "failed to validate the log level flag") - } - - return nil -} - -func validateArtifactPath(value string) error { - if value == "" { - return trace.BadParameter("the artifact-path flag should not be empty") - } - - if stat, err := os.Stat(value); os.IsNotExist(err) { - return trace.BadParameter("the artifact-path %q does not exist", value) - } else if !stat.IsDir() { - return trace.BadParameter("the artifact-path %q is not a directory", value) - } - - return nil -} - -func validateBucketName(value string) error { - if value == "" { - return trace.BadParameter("the bucket flag should not be empty") - } - - return nil -} - -func validateLocalBucketPath(value string) error { - if value == "" { - return trace.BadParameter("the local-bucket-path flag should not be empty") - } - - if stat, err := os.Stat(value); err == nil && !stat.IsDir() { - return trace.BadParameter("the local bucket path points to a file instead of a directory") - } - - return nil -} - -func validateArtifactVersion(value string) error { - if value == "" { - return trace.BadParameter("the artifact-version flag should not be empty") - } - - if !semver.IsValid(value) { - return trace.BadParameter("the artifact-version flag does not contain a valid semver version string") - } - - return nil -} - -func validateReleaseChannel(value string) error { - if value == "" { - return trace.BadParameter("the release-channel flag should not be empty") - } - - // Not sure what other channels we'd want to support, but they should be listed here - validReleaseChannels := []string{StableChannelFlagValue} - - for _, validReleaseChannel := range validReleaseChannels { - if value == validReleaseChannel { - return nil - } - } - - return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ",")) -} - -func validateLogLevel(value uint) error { - if value > 6 { - return trace.BadParameter("the log-level flag should be between 0 and 6") - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-apt-repos/main.go b/build.assets/tooling/cmd/build-apt-repos/main.go deleted file mode 100644 index c4c08d03ef41d..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/main.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - log "github.com/sirupsen/logrus" -) - -func main() { - supportedOSs := map[string][]string{ - "debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details - "stretch", // 9 - "buster", // 10 - "bullseye", // 11 - "bookwork", // 12 - "trixie", // 13 - }, - "ubuntu": { // See https://wiki.ubuntu.com/Releases for details - "xenial", // 16.04 LTS - "yakkety", // 16.10 (EOL) - "zesty", // 17.04 (EOL) - "artful", // 17.10 (EOL) - "bionic", // 18.04 LTS - "cosmic", // 18.10 (EOL) - "disco", // 19.04 (EOL) - "eoan", // 19.10 (EOL) - "focal", // 20.04 LTS - "groovy", // 20.10 (EOL) - "hirsuite", // 21.04 (EOL) - "impish", // 21.10 (EOL on 7/14/22) - "jammy", // 22.04 LTS - }, - } - - config, err := ParseFlags() - if err != nil { - log.Fatal(err.Error()) - } - - setupLogger(config) - log.Debugf("Starting tool with config: %v", config) - - art, err := NewAptRepoTool(config, supportedOSs) - if err != nil { - log.Fatal(err.Error()) - } - - err = art.Run() - if err != nil { - log.Fatal(err.Error()) - } -} - -func setupLogger(config *Config) { - if config.logJSON { - log.SetFormatter(&log.JSONFormatter{}) - } else { - log.SetFormatter(&log.TextFormatter{}) - } - log.SetOutput(os.Stdout) - log.SetLevel(log.Level(config.logLevel)) -} diff --git a/build.assets/tooling/cmd/build-apt-repos/s3manager.go b/build.assets/tooling/cmd/build-apt-repos/s3manager.go deleted file mode 100644 index 3a060eb05e5dd..0000000000000 --- a/build.assets/tooling/cmd/build-apt-repos/s3manager.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "github.com/aws/aws-sdk-go/aws/session" - "github.com/gravitational/trace" - "github.com/seqsense/s3sync" - "github.com/sirupsen/logrus" -) - -type S3manager struct { - syncManager *s3sync.Manager - bucketName string - bucketPath string -} - -func NewS3Manager(bucketName string) *S3manager { - // Right now the AWS session is only used by this manager, but if it ends - // up being needed elsewhere then it should probably be moved to an arg - awsSession := session.Must(session.NewSession()) - - manager := &S3manager{ - syncManager: s3sync.New(awsSession), - bucketName: bucketName, - bucketPath: fmt.Sprintf("s3://%s", bucketName), - } - - s3sync.SetLogger(&s3logger{}) - - return manager -} - -func (s *S3manager) DownloadExistingRepo(localPath string) error { - err := ensureDirectoryExists(localPath) - if err != nil { - return trace.Wrap(err, "failed to ensure path %q exists", localPath) - } - - err = s.sync(localPath, true) - if err != nil { - return trace.Wrap(err, "failed to download bucket") - } - - return nil -} - -func (s *S3manager) UploadBuiltRepo(localPath string) error { - err := s.sync(localPath, false) - - if err != nil { - return trace.Wrap(err, "failed to upload bucket") - } - - return nil -} - -func (s *S3manager) sync(localPath string, download bool) error { - var src, dest string - if download { - src = s.bucketPath - dest = localPath - } else { - src = localPath - dest = s.bucketPath - } - - logrus.Infof("Performing S3 sync from %q to %q...", src, dest) - err := s.syncManager.Sync(src, dest) - if err != nil { - return trace.Wrap(err, "failed to sync %q to %q", src, dest) - } - logrus.Infoln("S3 sync complete") - - return nil -} - -func ensureDirectoryExists(path string) error { - err := os.MkdirAll(path, 0660) - if err != nil { - return trace.Wrap(err, "failed to create directory %q", path) - } - - return nil -} diff --git a/build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go b/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go similarity index 82% rename from build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go rename to build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go index 27ae11f1bd428..66e0e60ed9545 100644 --- a/build.assets/tooling/cmd/build-apt-repos/apt_repo_tool.go +++ b/build.assets/tooling/cmd/build-os-package-repos/apt_repo_tool.go @@ -22,40 +22,51 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" "github.com/gravitational/trace" "github.com/sirupsen/logrus" "golang.org/x/mod/semver" ) type AptRepoTool struct { - config *Config + config *AptConfig aptly *Aptly + gpg *GPG s3Manager *S3manager supportedOSs map[string][]string } // Instantiates a new apt repo tool instance and performs any required setup/config. -func NewAptRepoTool(config *Config, supportedOSs map[string][]string) (*AptRepoTool, error) { - art := &AptRepoTool{ - config: config, - s3Manager: NewS3Manager(config.bucketName), - supportedOSs: supportedOSs, - } - +func NewAptRepoTool(config *AptConfig, supportedOSs map[string][]string) (*AptRepoTool, error) { aptly, err := NewAptly(config.aptlyPath) if err != nil { return nil, trace.Wrap(err, "failed to create a new aptly instance") } - art.aptly = aptly + gpg, err := NewGPG() + if err != nil { + return nil, trace.Wrap(err, "failed to create a new GPG instance") + } + + s3Manager, err := NewS3Manager(config.S3Config) + if err != nil { + return nil, trace.Wrap(err, "failed to create a new s3manager instance") + } - return art, nil + return &AptRepoTool{ + aptly: aptly, + config: config, + gpg: gpg, + s3Manager: s3Manager, + supportedOSs: supportedOSs, + }, nil } // Runs the tool, creating and updating APT repos based upon the current configuration. func (art *AptRepoTool) Run() error { start := time.Now() logrus.Infoln("Starting APT repo build process...") + logrus.Debugf("Using config: %+v", spew.Sdump(art.config)) isFirstRun, err := art.aptly.IsFirstRun() if err != nil { @@ -65,7 +76,7 @@ func (art *AptRepoTool) Run() error { if isFirstRun { logrus.Warningln("First run or disaster recovery detected, attempting to rebuild existing repos from APT repository...") - err = art.s3Manager.DownloadExistingRepo(art.config.localBucketPath) + err = art.s3Manager.DownloadExistingRepo() if err != nil { return trace.Wrap(err, "failed to sync existing repo from S3 bucket") } @@ -74,6 +85,8 @@ func (art *AptRepoTool) Run() error { if err != nil { return trace.Wrap(err, "failed to recreate existing repos") } + } else { + logrus.Debugf("Not first run of tool, skipping Aptly repository rebuild process") } // Note: this logic will only push the artifact into the `art.supportedOSs` repos. @@ -94,11 +107,24 @@ func (art *AptRepoTool) Run() error { return trace.Wrap(err, "failed to publish repos") } - err = art.s3Manager.UploadBuiltRepo(filepath.Join(art.aptly.rootDir, "public")) + // Both Hashicorp and Docker publish their key to this path + err = art.gpg.WritePublicKeyToFile(filepath.Join(art.aptly.rootDir, "public", "gpg")) + if err != nil { + return trace.Wrap(err, "failed to write GPG public key") + } + + art.s3Manager.ChangeLocalBucketPath(filepath.Join(art.aptly.rootDir, "public")) + err = art.s3Manager.UploadBuiltRepo() if err != nil { return trace.Wrap(err, "failed to sync changes to S3 bucket") } + // Future work: add literals to config? + err = art.s3Manager.UploadRedirectURL("index.html", "https://goteleport.com/docs/installation/#linux") + if err != nil { + return trace.Wrap(err, "failed to redirect index page to Teleport docs") + } + logrus.Infof("APT repo build process completed in %s", time.Since(start).Round(time.Millisecond)) return nil } diff --git a/build.assets/tooling/cmd/build-apt-repos/aptly.go b/build.assets/tooling/cmd/build-os-package-repos/aptly.go similarity index 94% rename from build.assets/tooling/cmd/build-apt-repos/aptly.go rename to build.assets/tooling/cmd/build-os-package-repos/aptly.go index 26dcb5e9f7410..32255d105740d 100644 --- a/build.assets/tooling/cmd/build-apt-repos/aptly.go +++ b/build.assets/tooling/cmd/build-os-package-repos/aptly.go @@ -22,9 +22,7 @@ import ( "errors" "fmt" "io/fs" - "log" "os" - "os/exec" "path" "path/filepath" "regexp" @@ -66,7 +64,7 @@ func (*Aptly) ensureDefaultConfigExists() error { // ran, which messes up the output. // Note: it is important to not use any repo-related commands here as they have a side effect of // also creating the Aptly rootDir structure which is usually undesirable here - _, err := buildAndRunCommand("aptly", "config", "show") + _, err := BuildAndRunCommand("aptly", "config", "show") if err != nil { return trace.Wrap(err, "failed to create default Aptly config") } @@ -86,7 +84,7 @@ func (a *Aptly) updateConfiguration() error { logrus.Debugf("Built Aptly config: %v", aptlyConfigMap) saveAptlyConfigMap(aptlyConfigMap) - configOutput, err := buildAndRunCommand("aptly", "config", "show") + configOutput, err := BuildAndRunCommand("aptly", "config", "show") if err != nil { return trace.Wrap(err, "failed to check Aptly config") } @@ -192,7 +190,7 @@ func (a *Aptly) CreateRepoIfNotExists(r *Repo) (bool, error) { distributionArg := fmt.Sprintf("-distribution=%s", r.osVersion) componentArg := fmt.Sprintf("-component=%s/%s", r.releaseChannel, r.majorVersion) - _, err = buildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name()) + _, err = BuildAndRunCommand("aptly", "repo", "create", distributionArg, componentArg, r.Name()) if err != nil { return false, trace.Wrap(err, "failed to create repo %q", r.Name()) } @@ -222,7 +220,7 @@ func (a *Aptly) GetExistingRepoNames() ([]string, error) { // ... // // ``` - output, err := buildAndRunCommand("aptly", "repo", "list", "-raw") + output, err := BuildAndRunCommand("aptly", "repo", "list", "-raw") if err != nil { return nil, trace.Wrap(err, "failed to get a list of existing repos") } @@ -248,7 +246,7 @@ func (a *Aptly) GetExistingRepoNames() ([]string, error) { func (a *Aptly) ImportDeb(repoName string, debPath string) error { logrus.Infof("Importing deb(s) from %q into repo %q...", debPath, repoName) - _, err := buildAndRunCommand("aptly", "repo", "add", repoName, debPath) + _, err := BuildAndRunCommand("aptly", "repo", "add", repoName, debPath) if err != nil { return trace.Wrap(err, "failed to add %q to repo %q", debPath, repoName) } @@ -320,7 +318,7 @@ func parsePackagesFile(packagesPath string) ([]string, error) { logrus.Debugf("Parsing packages file %q", packagesPath) file, err := os.Open(packagesPath) if err != nil { - log.Fatal(err) + logrus.Fatal(err) } defer file.Close() @@ -394,7 +392,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) // If all repos have been published if areSomePublished && !areSomeUnpublished { // Update rather than republish - _, err := buildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS) + _, err := BuildAndRunCommand("aptly", "publish", "update", repoOSVersion, repoOS) if err != nil { return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) } @@ -406,7 +404,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) // This will occur if there is a new major release, a OS version is supported, or a new release channel is added if areSomePublished && areSomeUnpublished { // Drop the currently published APT repo so that it can be rebuilt from scratch - _, err := buildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS) + _, err := BuildAndRunCommand("aptly", "publish", "drop", repoOSVersion, repoOS) if err != nil { return trace.Wrap(err, "failed to update publish repos with OS %q and OS version %q", repoOS, repoOSVersion) } @@ -423,7 +421,7 @@ func (a *Aptly) PublishRepos(repos []*Repo, repoOS string, repoOSVersion string) args = append(args, repoOS) // Full command is `aptly publish repo -component=<, repeating len(repos) - 1 times> ` - _, err = buildAndRunCommand("aptly", args...) + _, err = BuildAndRunCommand("aptly", args...) if err != nil { return trace.Wrap(err, "failed to publish repos") } @@ -501,7 +499,7 @@ func (a *Aptly) GetPublishedRepoNames() ([]string, error) { // No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`. // ``` // Note that the `-raw` argument is not used here as it does not provide sufficient information - output, err := buildAndRunCommand("aptly", "publish", "list") + output, err := BuildAndRunCommand("aptly", "publish", "list") if err != nil { return nil, trace.Wrap(err, "failed to get a list of published repos") } @@ -675,26 +673,3 @@ func getSubdirectories(basePath string) ([]string, error) { return subdirectories, nil } - -func buildAndRunCommand(command string, args ...string) (string, error) { - cmd := exec.Command(command, args...) - logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '")) - output, err := cmd.CombinedOutput() - - if output != nil { - logrus.Debugf("Command output: %s", string(output)) - } - - if err != nil { - if exitError, ok := err.(*exec.ExitError); ok { - exitCode := exitError.ExitCode() - logrus.Debugf("Command exited with exit code %d", exitCode) - } else { - logrus.Debugln("Command failed without an exit code") - } - return "", trace.Wrap(err, "Command failed, see debug output for additional details") - } - - logrus.Debugln("Command exited successfully") - return string(output), nil -} diff --git a/build.assets/tooling/cmd/build-os-package-repos/command_executor.go b/build.assets/tooling/cmd/build-os-package-repos/command_executor.go new file mode 100644 index 0000000000000..78a9bb8b7342d --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/command_executor.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "os/exec" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +// Builds an runs a command with the provided arguments. Extensively logs command +// details to the debug log. Returns stdout and stderr combined, along with an +// error iff one occurred. +func BuildAndRunCommand(command string, args ...string) (string, error) { + cmd := exec.Command(command, args...) + logrus.Debugf("Running command \"%s '%s'\"", command, strings.Join(args, "' '")) + output, err := cmd.CombinedOutput() + + if output != nil { + logrus.Debugf("Command output: %s", string(output)) + } + + if err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + exitCode := exitError.ExitCode() + logrus.Debugf("Command exited with exit code %d", exitCode) + } else { + logrus.Debugln("Command failed without an exit code") + } + return "", trace.Wrap(err, "Command failed, see debug output for additional details") + } + + logrus.Debugln("Command exited successfully") + return string(output), nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/config.go b/build.assets/tooling/cmd/build-os-package-repos/config.go new file mode 100644 index 0000000000000..df3357f2dee57 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/config.go @@ -0,0 +1,282 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" +) + +const StableChannelFlagValue string = "stable" + +type LoggerConfig struct { + logLevel uint + logJSON bool +} + +func NewLoggerConfigWithFlagset(fs *flag.FlagSet) *LoggerConfig { + lc := &LoggerConfig{} + fs.UintVar(&lc.logLevel, "log-level", uint(logrus.InfoLevel), "Log level from 0 to 6, 6 being the most verbose") + fs.BoolVar(&lc.logJSON, "log-json", false, "True if the log entries should use JSON format, false for text logging") + + return lc +} + +func (lc *LoggerConfig) Check() error { + if err := lc.validateLogLevel(); err != nil { + return trace.Wrap(err, "failed to validate the log level flag") + } + + return nil +} + +func (lc *LoggerConfig) validateLogLevel() error { + if lc.logLevel > 6 { + return trace.BadParameter("the log-level flag should be between 0 and 6") + } + + return nil +} + +type S3Config struct { + bucketName string + localBucketPath string + maxConcurrentSyncs int +} + +func NewS3ConfigWithFlagset(fs *flag.FlagSet) *S3Config { + s3c := &S3Config{} + fs.StringVar(&s3c.bucketName, "bucket", "", "The name of the S3 bucket where the repo should be synced to/from") + fs.StringVar(&s3c.localBucketPath, "local-bucket-path", "/bucket", "The local path where the bucket should be synced to") + fs.IntVar(&s3c.maxConcurrentSyncs, "max-concurrent-syncs", 16, "The maximum number of S3 bucket syncs that may run in parallel (-1 for unlimited, 16 default)") + + return s3c +} + +func (s3c *S3Config) Check() error { + if err := s3c.validateBucketName(); err != nil { + return trace.Wrap(err, "failed to validate the bucket name flag") + } + if err := s3c.validateLocalBucketPath(); err != nil { + return trace.Wrap(err, "failed to validate the local bucket path flag") + } + if err := s3c.validateMaxConcurrentSyncs(); err != nil { + return trace.Wrap(err, "failed to validate the max concurrent syncs flag") + } + + return nil +} + +func (s3c *S3Config) validateBucketName() error { + if s3c.bucketName == "" { + return trace.BadParameter("the bucket flag should not be empty") + } + + return nil +} + +func (s3c *S3Config) validateLocalBucketPath() error { + if s3c.localBucketPath == "" { + return trace.BadParameter("the local-bucket-path flag should not be empty") + } + + if stat, err := os.Stat(s3c.localBucketPath); err == nil && !stat.IsDir() { + return trace.BadParameter("the local bucket path points to a file instead of a directory") + } + + return nil +} + +func (s3c *S3Config) validateMaxConcurrentSyncs() error { + if s3c.maxConcurrentSyncs < -1 { + return trace.BadParameter("the max-concurrent-syncs flag must be greater than -1") + } + + return nil +} + +// This type is common to all other config types +type Config struct { + *LoggerConfig + *S3Config + artifactPath string + artifactVersion string + printHelp bool + releaseChannel string +} + +func NewConfigWithFlagset(fs *flag.FlagSet) *Config { + c := &Config{} + c.LoggerConfig = NewLoggerConfigWithFlagset(fs) + c.S3Config = NewS3ConfigWithFlagset(fs) + + fs.StringVar(&c.artifactPath, "artifact-path", "/artifacts", "Path to the filesystem tree containing the *.deb or *.rpm files to add to the repos") + fs.StringVar(&c.artifactVersion, "artifact-version", "", "The version of the artifacts that will be added to the repos") + fs.Visit(func(f *flag.Flag) { + if f.Name == "-h" || f.Name == "--help" { + c.printHelp = true + } + }) + fs.StringVar(&c.releaseChannel, "release-channel", "", "The release channel of the repos that the artifacts should be added to") + + return c +} + +func (c *Config) Check() error { + if err := c.LoggerConfig.Check(); err != nil { + return trace.Wrap(err, "failed to validate logger config") + } + + if err := c.S3Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate S3 config") + } + + if err := c.validateArtifactPath(); err != nil { + return trace.Wrap(err, "failed to validate the artifact path flag") + } + if err := c.validateArtifactVersion(); err != nil { + return trace.Wrap(err, "failed to validate the artifact version flag") + } + if err := c.validateReleaseChannel(); err != nil { + return trace.Wrap(err, "failed to validate the release channel flag") + } + + return nil +} + +func (c *Config) validateArtifactPath() error { + if c.artifactPath == "" { + return trace.BadParameter("the artifact-path flag should not be empty") + } + + if stat, err := os.Stat(c.artifactPath); os.IsNotExist(err) { + return trace.BadParameter("the artifact-path %q does not exist", c.artifactPath) + } else if !stat.IsDir() { + return trace.BadParameter("the artifact-path %q is not a directory", c.artifactPath) + } + + return nil +} + +func (c *Config) validateArtifactVersion() error { + if c.artifactVersion == "" { + return trace.BadParameter("the artifact-version flag should not be empty") + } + + if !semver.IsValid(c.artifactVersion) { + return trace.BadParameter("the artifact-version flag does not contain a valid semver version string") + } + + return nil +} + +func (c *Config) validateReleaseChannel() error { + if c.releaseChannel == "" { + return trace.BadParameter("the release-channel flag should not be empty") + } + + // Not sure what other channels we'd want to support, but they should be listed here + validReleaseChannels := []string{StableChannelFlagValue} + + for _, validReleaseChannel := range validReleaseChannels { + if c.releaseChannel == validReleaseChannel { + return nil + } + } + + return trace.BadParameter("the release channel contains an invalid value. Valid values are: %s", strings.Join(validReleaseChannels, ",")) +} + +// APT-specific config +type AptConfig struct { + *Config + aptlyPath string +} + +func NewAptConfigWithFlagSet(fs *flag.FlagSet) (*AptConfig, error) { + ac := &AptConfig{} + ac.Config = NewConfigWithFlagset(fs) + + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, trace.Wrap(err, "failed to get user's home directory path") + } + + fs.StringVar(&ac.aptlyPath, "aptly-root-dir", homeDir, "The Aptly \"rootDir\" (see https://www.aptly.info/doc/configuration/ for details)") + + return ac, nil +} + +func (ac *AptConfig) validateAptlyPath() error { + if ac.aptlyPath == "" { + return trace.BadParameter("the aptly-root-dir flag should not be empty") + } + + return nil +} + +func (ac *AptConfig) Check() error { + if err := ac.Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate common config") + } + + if err := ac.validateAptlyPath(); err != nil { + return trace.Wrap(err, "failed to validate the aptly-root-dir path flag") + } + + return nil +} + +// YUM-specific config +type YumConfig struct { + *Config + cacheDir string +} + +func NewYumConfigWithFlagSet(fs *flag.FlagSet) *YumConfig { + yc := &YumConfig{} + yc.Config = NewConfigWithFlagset(fs) + + fs.StringVar(&yc.cacheDir, "cache-dir", "/tmp/createrepo/cache", "The createrepo checksum caching directory (see https://linux.die.net/man/8/createrepo for details") + + return yc +} + +func (yc *YumConfig) validateCacheDir() error { + if yc.cacheDir == "" { + return trace.BadParameter("the cache-dir flag should not be empty") + } + + return nil +} + +func (yc *YumConfig) Check() error { + if err := yc.Config.Check(); err != nil { + return trace.Wrap(err, "failed to validate common config") + } + + if err := yc.validateCacheDir(); err != nil { + return trace.Wrap(err, "failed to validate the cache-dir path flag") + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/createrepo.go b/build.assets/tooling/cmd/build-os-package-repos/createrepo.go new file mode 100644 index 0000000000000..5c3a208926891 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/createrepo.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "os/exec" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +type CreateRepo struct { + cacheDir string + binaryName string +} + +// Instantiates createrepo, ensuring all system requirements for performing createrepo operations +// have been met +func NewCreateRepo(cacheDir string) (*CreateRepo, error) { + cr := &CreateRepo{ + cacheDir: cacheDir, + // `createrepo_c` is the "new" (as in 9 years old) replacement for `createrepo` + // This can be replace with "createrepo" in the unlikely chance that there is + // a problem + binaryName: "createrepo_c", + } + + err := cr.ensureBinaryExists() + if err != nil { + return nil, trace.Wrap(err, "failed to ensure CreateRepo binary exists") + } + + // Ensure the cache dir exists + err = os.MkdirAll(cr.cacheDir, 0660) + if err != nil { + return nil, trace.Wrap(err, "failed to ensure %q exists", cr.cacheDir) + } + + return cr, nil +} + +func (cr *CreateRepo) ensureBinaryExists() error { + _, err := exec.LookPath(cr.binaryName) + if err != nil { + return trace.Wrap(err, "failed to verify that %q binary exists", cr.binaryName) + } + + return nil +} + +func (cr *CreateRepo) CreateOrUpdateRepo(repoPath string) error { + // --cachedir --update + logrus.Debugf("Updating repo metadata for repo at %q", repoPath) + + args := []string{ + "--cachedir", + cr.cacheDir, + "--update", + repoPath, + } + + _, err := BuildAndRunCommand(cr.binaryName, args...) + if err != nil { + return trace.Wrap(err, "createrepo create/update command failed on path %q with cache directory %q", repoPath, cr.cacheDir) + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/gpg.go b/build.assets/tooling/cmd/build-os-package-repos/gpg.go new file mode 100644 index 0000000000000..531649d4d45fc --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/gpg.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +type GPG struct{} + +// Instantiates GPG, ensuring all system requirements for using GPG are fulfilled +func NewGPG() (*GPG, error) { + g := &GPG{} + + err := g.ensureFirstRunHasOccurred() + if err != nil { + return nil, trace.Wrap(err, "failed to setup GPG") + } + + err = g.ensureSecretKeyExists() + if err != nil { + return nil, trace.Wrap(err, "failed to ensure a secret key exists") + } + + return g, nil +} + +// The first time GPG is run for a user with any "meaningful" arguments it will +// generate several files and log what it created to stdout. These logs can +// disrupt parsing of GPG command outputs, so we force it to happen here, once, +// rather than try and handle it on each GPG call. +func (*GPG) ensureFirstRunHasOccurred() error { + _, err := BuildAndRunCommand("gpg", "--fingerprint") + if err != nil { + return trace.Wrap(err, "failed to ensure GPG has been ran once") + } + + return nil +} + +func (*GPG) ensureSecretKeyExists() error { + output, err := BuildAndRunCommand("gpg", "--list-secret-keys", "--with-colons") + if err != nil { + return trace.Wrap(err, "failed to ensure GPG secret key exists") + } + + outputLineCount := strings.Count(output, "\n") + if outputLineCount < 1 { + return trace.Errorf("failed to find a GPG secret key") + } + + return nil +} + +// Creates a detached, armored signature for the provided file using the default GPG key +func (*GPG) SignFile(filePath string) error { + // While this could be done via a Go module, the x/crypto/openpgp library has been frozen + // and deprecated for almost 18 months. Others exist, but given the security implications of + // using a less reputable Go module I've decided to just call `gpg` via shell instead. + // Additionally this works and is just _so easy_ that it's probably not worth the effort to + // use another library that reinvents the wheel. + logrus.Debugf("Signing repo metadata at %q", filePath) + + // gpg --batch --yes --detach-sign --armor + _, err := BuildAndRunCommand("gpg", "--batch", "--yes", "--detach-sign", "--armor", filePath) + if err != nil { + return trace.Wrap(err, "failed to run GPG signing command on %q", filePath) + } + + return nil +} + +// Get the armored default public GPG key, ready to be written to a file +func (*GPG) GetPublicKey() (string, error) { + // For reference here is how another company formats their key: + // https://download.docker.com/linux/rhel/gpg + logrus.Debug("Attempting to get the default public GPG key") + + key, err := BuildAndRunCommand("gpg", "--export", "--armor", "--no-version") + if err != nil { + return "", trace.Wrap(err, "failed to export the default public GPG key") + } + + return key, nil +} + +func (g *GPG) WritePublicKeyToFile(filePath string) error { + logrus.Debugf("Writing the default armored public GPG key to %q", filePath) + + key, err := g.GetPublicKey() + if err != nil { + return trace.Wrap(err, "failed to retrieve public key") + } + + err = os.WriteFile(filePath, []byte(key), 0664) + if err != nil { + return trace.Wrap(err, "failed to write key to %q", filePath) + } + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/main.go b/build.assets/tooling/cmd/build-os-package-repos/main.go new file mode 100644 index 0000000000000..58f89f1dd1d16 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/main.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" +) + +func main() { + err := run() + if err != nil { + logrus.Fatal(err.Error()) + } +} + +func buildSubcommandRunners() ([]Runner, error) { + ar, err := NewAptRunner() + if err != nil { + return nil, trace.Wrap(err, "failed to instantiate new APT runner") + } + + yr, err := NewYumRunner() + if err != nil { + return nil, trace.Wrap(err, "failed to instantiate new YUM runner") + } + + // These should be sorted alphabetically by `Name()` + return []Runner{ + *ar, + *yr, + }, nil +} + +func run() error { + subcommands, err := buildSubcommandRunners() + if err != nil { + return trace.Wrap(err, "failed to build subcommand runners") + } + + // 2 = program name + subcommand + if len(os.Args) < 2 { + logHelp(subcommands) + return trace.Errorf("subcommand not provided") + } + + subcommandName := strings.ToLower(os.Args[1]) + for _, subcommand := range subcommands { + if strings.ToLower(subcommandName) != subcommand.Name() { + continue + } + + // 2 = program name + subcommand, skip them and get subcommand arguments + args := os.Args[2:] + err := subcommand.Init(args) + if err != nil { + return trace.Wrap(err, "failed to initialize runner for subcommand %q", subcommandName) + } + + setupLogger(subcommand.GetLoggerConfig()) + err = subcommand.Run() + if err != nil { + return trace.Wrap(err, "failed to run subcommand %q", subcommandName) + } + + return nil + } + + if subcommandName == "-h" { + logHelp(subcommands) + return nil + } + + logHelp(subcommands) + return trace.Errorf("no subcommands found matching %q", subcommandName) +} + +func logHelp(subcommands []Runner) { + executableName := os.Args[0] + fmt.Printf("%s - OS package repo builder/updater\n", executableName) + fmt.Println() + fmt.Println("Commands:") + fmt.Println() + for _, subcommand := range subcommands { + fmt.Printf("\t%s\t%s\n", subcommand.Name(), subcommand.Info()) + } + fmt.Println() + fmt.Printf("Use \"%s -h\" for more information about a command.\n", executableName) + fmt.Println() +} + +func setupLogger(config *LoggerConfig) { + if config.logJSON { + logrus.SetFormatter(&logrus.JSONFormatter{}) + } else { + logrus.SetFormatter(&logrus.TextFormatter{}) + } + logrus.SetOutput(os.Stdout) + logrus.SetLevel(logrus.Level(config.logLevel)) + logrus.Debugf("Setup logger with config: %+v", config) +} diff --git a/build.assets/tooling/cmd/build-apt-repos/repo.go b/build.assets/tooling/cmd/build-os-package-repos/repo.go similarity index 100% rename from build.assets/tooling/cmd/build-apt-repos/repo.go rename to build.assets/tooling/cmd/build-os-package-repos/repo.go diff --git a/build.assets/tooling/cmd/build-os-package-repos/runners.go b/build.assets/tooling/cmd/build-os-package-repos/runners.go new file mode 100644 index 0000000000000..9ecda55eea2c8 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/runners.go @@ -0,0 +1,199 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + + "github.com/gravitational/trace" +) + +// Pattern from https://www.digitalocean.com/community/tutorials/how-to-use-the-flag-package-in-go +type Runner interface { + Init([]string) error + Run() error + GetLoggerConfig() *LoggerConfig + Name() string + Info() string +} + +// APT implementation +type AptRunner struct { + flags *flag.FlagSet + config *AptConfig + supportedOSs map[string][]string +} + +func NewAptRunner() (*AptRunner, error) { + runner := &AptRunner{ + supportedOSs: map[string][]string{ + "debian": { // See https://wiki.debian.org/DebianReleases#Production_Releases for details + "stretch", // 9 + "buster", // 10 + "bullseye", // 11 + "bookwork", // 12 + "trixie", // 13 + }, + "ubuntu": { // See https://wiki.ubuntu.com/Releases for details + "xenial", // 16.04 LTS + "yakkety", // 16.10 (EOL) + "zesty", // 17.04 (EOL) + "artful", // 17.10 (EOL) + "bionic", // 18.04 LTS + "cosmic", // 18.10 (EOL) + "disco", // 19.04 (EOL) + "eoan", // 19.10 (EOL) + "focal", // 20.04 LTS + "groovy", // 20.10 (EOL) + "hirsuite", // 21.04 (EOL) + "impish", // 21.10 (EOL) + "jammy", // 22.04 LTS + }, + }, + } + + runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) + config, err := NewAptConfigWithFlagSet(runner.flags) + if err != nil { + return nil, trace.Wrap(err, "failed to create a new APT config instance") + } + + runner.config = config + + return runner, nil +} + +func (ar AptRunner) Init(args []string) error { + err := ar.flags.Parse(args) + if err != nil { + return trace.Wrap(err, "failed to parse arguments") + } + + err = ar.config.Check() + if err != nil { + return trace.Wrap(err, "failed to validate APT config arguments") + } + + return nil +} + +func (ar AptRunner) Run() error { + if ar.config.printHelp { + ar.flags.Usage() + return nil + } + + art, err := NewAptRepoTool(ar.config, ar.supportedOSs) + if err != nil { + return trace.Wrap(err, "failed to create a new APT repo tool instance") + } + + err = art.Run() + if err != nil { + return trace.Wrap(err, "APT runner failed") + } + + return nil +} + +func (AptRunner) Name() string { + return "apt" +} + +func (AptRunner) Info() string { + return "builds APT repos" +} + +func (ar AptRunner) GetLoggerConfig() *LoggerConfig { + return ar.config.LoggerConfig +} + +// YUM implementation +type YumRunner struct { + flags *flag.FlagSet + config *YumConfig + supportedOSs map[string][]string +} + +func NewYumRunner() (*YumRunner, error) { + runner := &YumRunner{ + supportedOSs: map[string][]string{ + "rhel": { // See https://access.redhat.com/articles/3078 for details + "7", + "8", + "9", + }, + "centos": { // See https://endoflife.date/centos for details + "7", + "8", + "9", + }, + // "$releasever" is a hot mess for Amazon Linux. No good documentation on this outside of just running + // a container or EC2 instance and manually checking $releasever values + "amzn": { + // "latest" // 1, aka 2018.03.0.20201028.0 + "2", // 2, aka 2.0.20201111.0 + // "2022.0.20220531" // 2022 (new naming scheme, preview) aka 2022.0.20220531 + }, + }, + } + + runner.flags = flag.NewFlagSet(runner.Name(), flag.ExitOnError) + runner.config = NewYumConfigWithFlagSet(runner.flags) + + return runner, nil +} + +func (yr YumRunner) Init(args []string) error { + err := yr.flags.Parse(args) + if err != nil { + return trace.Wrap(err, "failed to parse arguments") + } + + err = yr.config.Check() + if err != nil { + return trace.Wrap(err, "failed to validate YUM config arguments") + } + + return nil +} + +func (yr YumRunner) Run() error { + yrt, err := NewYumRepoTool(yr.config, yr.supportedOSs) + if err != nil { + return trace.Wrap(err, "failed to create a new YUM repo tool instance") + } + + err = yrt.Run() + if err != nil { + return trace.Wrap(err, "YUM runner failed") + } + + return nil +} + +func (YumRunner) Name() string { + return "yum" +} + +func (YumRunner) Info() string { + return "builds YUM repos" +} + +func (yr YumRunner) GetLoggerConfig() *LoggerConfig { + return yr.config.LoggerConfig +} diff --git a/build.assets/tooling/cmd/build-apt-repos/s3logger.go b/build.assets/tooling/cmd/build-os-package-repos/s3logger.go similarity index 100% rename from build.assets/tooling/cmd/build-apt-repos/s3logger.go rename to build.assets/tooling/cmd/build-os-package-repos/s3logger.go diff --git a/build.assets/tooling/cmd/build-os-package-repos/s3manager.go b/build.assets/tooling/cmd/build-os-package-repos/s3manager.go new file mode 100644 index 0000000000000..00405efca9656 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/s3manager.go @@ -0,0 +1,511 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/gravitational/trace" + "github.com/inhies/go-bytesize" + "github.com/seqsense/s3sync" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +type S3manager struct { + syncManager *s3sync.Manager + uploader *s3manager.Uploader + downloader *s3manager.Downloader + bucketLocalPath string + bucketName string + bucketURL *url.URL + maxConcurrentSyncs int + downloadedBytes int64 +} + +func NewS3Manager(config *S3Config) (*S3manager, error) { + // Right now the AWS session is only used by this manager, but if it ends + // up being needed elsewhere then it should probably be moved to an arg + awsSession, err := session.NewSession() + if err != nil { + return nil, trace.Wrap(err, "failed to create a new AWS session") + } + + syncManagerMaxConcurrentSyncs := config.maxConcurrentSyncs + if syncManagerMaxConcurrentSyncs < 0 { + // This isn't unlimited but due to the s3sync library's parallelism implementation + // this must be limited to a "reasonable" number + syncManagerMaxConcurrentSyncs = 128 + } + + s := &S3manager{ + bucketName: config.bucketName, + bucketURL: &url.URL{ + Scheme: "s3", + Host: config.bucketName, + }, + syncManager: s3sync.New(awsSession, s3sync.WithParallel(syncManagerMaxConcurrentSyncs)), + uploader: s3manager.NewUploader(awsSession), + downloader: s3manager.NewDownloader(awsSession), + maxConcurrentSyncs: config.maxConcurrentSyncs, + } + s.ChangeLocalBucketPath(config.localBucketPath) + + s3sync.SetLogger(&s3logger{}) + + return s, nil +} + +func (s *S3manager) ChangeLocalBucketPath(newBucketPath string) error { + s.bucketLocalPath = newBucketPath + + // Ensure the local bucket path exists as it will be needed by all functions + err := os.MkdirAll(s.bucketLocalPath, 0660) + if err != nil { + return trace.Wrap(err, "failed to ensure path %q exists", s.bucketLocalPath) + } + + return nil +} + +func (s *S3manager) DownloadExistingRepo() error { + err := deleteAllFilesInDirectory(s.bucketLocalPath) + if err != nil { + return trace.Wrap(err, "failed to remove all filesystem entries in %q", s.bucketLocalPath) + } + + downloadGroup := &errgroup.Group{} + downloadGroup.SetLimit(s.maxConcurrentSyncs) + linkMap := make(map[string]string) + + var continuationToken *string + for { + listObjResponse, err := s.downloader.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &s.bucketName, + ContinuationToken: continuationToken, + }) + if err != nil { + return trace.Wrap(err, "failed to list objects for bucket %q", s.bucketName) + } + + for _, s3object := range listObjResponse.Contents { + s.processS3ObjectDownload(s3object, downloadGroup, &linkMap) + } + + continuationToken = listObjResponse.NextContinuationToken + if continuationToken == nil { + break + } + } + + // Even if an error has occurred we should wait to exit until all running syncs have + // completed, even if not successful + logrus.Info("Waiting for download to complete...") + err = downloadGroup.Wait() + if err != nil { + return trace.Wrap(err, "failed to perform S3 sync from remote bucket %q to local bucket %q", s.bucketName, s.bucketLocalPath) + } + + // Links must be created after their target exists + err = createLinks(linkMap) + if err != nil { + return trace.Wrap(err, "failed to create filesystem links for bucket %q", s.bucketName) + } + + logrus.Infof("Downloaded %s bytes", bytesize.New(float64(s.downloadedBytes))) + return nil +} + +func (s *S3manager) processS3ObjectDownload(s3object *s3.Object, downloadGroup *errgroup.Group, linkMap *map[string]string) { + downloadGroup.Go(func() error { + objectLink, err := s.getObjectLink(s3object) + if err != nil { + return trace.Wrap(err, "failed to get object link for key %q in bucket %q", *s3object.Key, s.bucketName) + } + + // If the link does not start with a '/' then it is not a filesystem link + if objectLink != nil && len(*objectLink) > 0 && (*objectLink)[0] == '/' { + localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) + linkTarget := filepath.Join(s.bucketLocalPath, *objectLink) + (*linkMap)[localObjectPath] = linkTarget + return nil + } + + err = s.downloadFile(s3object) + if err != nil { + return trace.Wrap(err, "failed to download S3 file %q from bucket %q", *s3object.Key, s.bucketName) + } + + return nil + }) +} + +func createLinks(linkMap map[string]string) error { + for file, target := range linkMap { + logrus.Infof("Creating a symlink from %q to %q", file, target) + err := os.MkdirAll(filepath.Dir(file), 0660) + if err != nil { + return trace.Wrap(err, "failed to create directory structure for %q", file) + } + + err = os.Symlink(target, file) + if err != nil { + return trace.Wrap(err, "failed to symlink %q to %q", file, target) + } + } + + return nil +} + +// This could potentially be made more efficient by running `os.RemoveAll` in a goroutine +// as random access on storage devices performs better at a higher queue depth +func deleteAllFilesInDirectory(dir string) error { + // Note that os.ReadDir does not follow/eval links which is important here + dirEntries, err := os.ReadDir(dir) + if err != nil { + return trace.Wrap(err, "failed to list directory entries for directory %q", dir) + } + + for _, dirEntry := range dirEntries { + dirEntryPath := filepath.Join(dir, dirEntry.Name()) + err = os.RemoveAll(dirEntryPath) + if err != nil { + return trace.Wrap(err, "failed to remove directory entry %q", dirEntryPath) + } + } + + return nil +} + +func (s *S3manager) getObjectLink(s3object *s3.Object) (*string, error) { + s3HeadObjectOutput, err := s.downloader.S3.HeadObject(&s3.HeadObjectInput{ + Bucket: &s.bucketName, + Key: s3object.Key, + // Probably unnecessary but this will cause an error to be thrown if somebody is + // modifying the object while this program is running + IfMatch: s3object.ETag, + IfUnmodifiedSince: s3object.LastModified, + }) + if err != nil { + return nil, trace.Wrap(err, "failed to retrieve metadata for key %q in bucket %q", *s3object.Key, s.bucketName) + } + + return s3HeadObjectOutput.WebsiteRedirectLocation, nil +} + +// s3sync has a bug when downloading a single file so this call reimplements s3sync's download +func (s *S3manager) downloadFile(s3object *s3.Object) error { + logrus.Infof("Downloading %q...", *s3object.Key) + localObjectPath := filepath.Join(s.bucketLocalPath, *s3object.Key) + + err := os.MkdirAll(filepath.Dir(localObjectPath), 0660) + if err != nil { + return trace.Wrap(err, "failed to create directory structure for %q", localObjectPath) + } + + fileWriter, err := os.Create(localObjectPath) + if err != nil { + return trace.Wrap(err, "failed to open %q for writing", localObjectPath) + } + defer fileWriter.Close() + + fileDownloadByteCount, err := s.downloader.Download(fileWriter, &s3.GetObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(*s3object.Key), + }) + if err != nil { + return trace.Wrap(err, "failed to download object %q from bucket %q to local path %q", *s3object.Key, s.bucketName, localObjectPath) + } + + s.downloadedBytes += fileDownloadByteCount + + err = os.Chtimes(localObjectPath, *s3object.LastModified, *s3object.LastModified) + if err != nil { + return trace.Wrap(err, "failed to update the access and modification time on file %q to %v", localObjectPath, *s3object.LastModified) + } + + logrus.Infof("Download %q complete", *s3object.Key) + return nil +} + +func (s *S3manager) UploadBuiltRepo() error { + err := s.sync(false) + if err != nil { + return trace.Wrap(err, "failed to upload bucket") + } + + return nil +} + +func (s *S3manager) UploadBuiltRepoWithRedirects(extensionToMatch, relativeRedirectDir string) error { + uploadGroup := &errgroup.Group{} + uploadGroup.SetLimit(s.maxConcurrentSyncs) + + walkErr := filepath.WalkDir(s.bucketLocalPath, func(absPath string, info fs.DirEntry, err error) error { + logrus.Debugf("Starting on %q...", absPath) + + if err != nil { + return trace.Wrap(err, "failed to walk over directory %q on path %q", s.bucketLocalPath) + } + + syncFunc, err := s.syncGenericFsObject(absPath, info) + if err != nil { + return trace.Wrap(err, "failed to get syncing function for %q", absPath) + } + + uploadGroup.Go(syncFunc) + logrus.Debugf("Upload for %q queued", absPath) + return nil + }) + + // Even if an error has occurred we should wait to exit until all running syncs have + // completed, even if not successful + logrus.Info("Waiting for sync to complete...") + syncErr := uploadGroup.Wait() + // Future work: add upload logging information once + // https://github.com/seqsense/s3sync/commit/29b3fcb259293d80634cb3916e0f28467d017087 has been released + logrus.Info("Sync has completed") + + errs := make([]error, 0, 2) + if walkErr != nil { + errs = append(errs, trace.Wrap(walkErr, "failed to walk over entries in %q", s.bucketLocalPath)) + } + + if syncErr != nil { + errs = append(errs, trace.Wrap(syncErr, "failed to perform S3 sync from local bucket %q to remote bucket %q", s.bucketLocalPath, s.bucketName)) + } + + if len(errs) > 0 { + return trace.Wrap(trace.NewAggregate(errs...), "one or more erros occurred while uploading built repo %q", s.bucketLocalPath) + } + + return nil +} + +func (s *S3manager) syncGenericFsObject(absPath string, dirEntryInfo fs.DirEntry) (func() error, error) { + // Don't do anything with non-empty directories as they will be caught later by their contents + if dirEntryInfo.IsDir() { + f, err := s.buildSyncDirFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build directory syncing function to sync %q", absPath) + } + + return f, nil + } else + // If symbolic link + if dirEntryInfo.Type()&fs.ModeSymlink != 0 { + f, err := s.buildSyncSymbolicLinkFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build symbolic link file syncing function to sync %q", absPath) + } + + return f, nil + } + + // sync a single file or directory + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncDirFunc(absPath string) (func() error, error) { + isDirEmpty, err := isDirectoryEmpty(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to determine if directory %q is empty", absPath) + } + + if !isDirEmpty { + logrus.Debug("Skipping non-empty directory") + return func() error { return nil }, nil + } + + // If the directory has no contents, call sync normally which will create the directory remotely if not exists + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncSymbolicLinkFunc(absPath string) (func() error, error) { + actualFilePath, err := filepath.EvalSymlinks(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to follow symlink for path %q", absPath) + } + + isInBucket, err := isPathChildOfAnother(s.bucketLocalPath, actualFilePath) + if err != nil { + return nil, trace.Wrap(err, "failed to determine if %q is a child of %q", actualFilePath, s.bucketLocalPath) + } + + if isInBucket { + // This will re-upload every redirect file ever created. Implementing "sync" functionality would + // require significantly more engineering effort and this cost is low so this shouldn't be a + // problem. + return func() error { + err := s.UploadRedirectFile(absPath, actualFilePath) + if err != nil { + return trace.Wrap(err, "failed to upload a redirect file to S3 for %q targeting %q", absPath, actualFilePath) + } + + return nil + }, nil + } + + // If not in bucket, call sync normally which will follow the symlink to the actual file and upload it + f, err := s.buildSyncSingleFsEntryFunc(absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to build single file syncing function to sync %q", absPath) + } + + return f, nil +} + +func (s *S3manager) buildSyncSingleFsEntryFunc(absPath string) (func() error, error) { + relPath, err := filepath.Rel(s.bucketLocalPath, absPath) + if err != nil { + return nil, trace.Wrap(err, "failed to get %q relative to %q", absPath, s.bucketLocalPath) + } + + remoteURL := getURLWithPath(*s.bucketURL, relPath) + return func() error { + err := s.syncManager.Sync(absPath, remoteURL) + if err != nil { + return trace.Wrap(err, "failed to sync from %q to %q", absPath, remoteURL) + } + + return nil + }, nil +} + +func getURLWithPath(baseURL url.URL, path string) string { + // Because this function is pass-by-value it should not modify `baseUrl`, where doing this directly on the + // provided parameter would modify it + baseURL.Path = path + return baseURL.String() +} + +func isPathChildOfAnother(baseAbsPath string, testAbsPath string) (bool, error) { + // General implementation from https://stackoverflow.com/questions/28024731/check-if-given-path-is-a-subdirectory-of-another-in-golang + relPath, err := filepath.Rel(baseAbsPath, testAbsPath) + if err != nil { + return false, trace.Wrap(err, "failed to get the path of %q relative to %q", testAbsPath, baseAbsPath) + } + + return !strings.HasPrefix(relPath, fmt.Sprintf("..%c", os.PathSeparator)) && relPath != "..", nil +} + +func (s *S3manager) UploadRedirectFile(localAbsSrcPath, localAbsRemoteTargetPath string) error { + relSrcPath, err := filepath.Rel(s.bucketLocalPath, localAbsSrcPath) + if err != nil { + return trace.Wrap(err, "failed to get %q relative to %q", localAbsSrcPath, s.bucketLocalPath) + } + + relTargetPath, err := filepath.Rel(s.bucketLocalPath, localAbsRemoteTargetPath) + if err != nil { + return trace.Wrap(err, "failed to get %q relative to %q", localAbsRemoteTargetPath, s.bucketLocalPath) + } + + logrus.Infof("Creating a redirect file from %q to %q", relSrcPath, relTargetPath) + // S3 requires a prepended "/" to inform the redirect metadata that the target is another S3 object + // in the same bucket + s3TargetPath := filepath.Join("/", relTargetPath) + // Upload an empty file that when requested will redirect to the real one + _, err = s.uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.bucketName, + Key: &relSrcPath, + Body: bytes.NewReader([]byte{}), + WebsiteRedirectLocation: &s3TargetPath, + }) + if err != nil { + return trace.Wrap(err, "failed to upload an empty redirect file to %q in bucket %q", relSrcPath, s.bucketName) + } + + return nil +} + +func (s *S3manager) UploadRedirectURL(remoteAbsSourcePath, targetURL string) error { + logrus.Infof("Creating redirect from %q to %q", remoteAbsSourcePath, targetURL) + + _, err := s.uploader.Upload(&s3manager.UploadInput{ + Bucket: &s.bucketName, + Key: &remoteAbsSourcePath, + Body: bytes.NewReader([]byte{}), + WebsiteRedirectLocation: &targetURL, + }) + + if err != nil { + return trace.Wrap(err, "failed to upload URL redirect file targeting %q to %q", targetURL, remoteAbsSourcePath) + } + + return nil +} + +func isDirectoryEmpty(dirPath string) (bool, error) { + // Pulled from https://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty + f, err := os.Open(dirPath) + if err != nil { + return false, trace.Wrap(err, "failed to open directory %q", dirPath) + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + + if err != nil { + return false, trace.Wrap(err, "failed to read the name of directories in %q", dirPath) + } + + return false, nil +} + +func (s *S3manager) sync(download bool) error { + var src, dest string + if download { + src = s.bucketURL.String() + dest = s.bucketLocalPath + } else { + src = s.bucketLocalPath + dest = s.bucketURL.String() + } + + logrus.Infof("Performing S3 sync from %q to %q...", src, dest) + err := s.syncManager.Sync(src, dest) + if err != nil { + return trace.Wrap(err, "failed to sync %q to %q", src, dest) + } + logrus.Infoln("S3 sync complete") + + return nil +} diff --git a/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh b/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh new file mode 100755 index 0000000000000..813c4e53d7a23 --- /dev/null +++ b/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# shellcheck disable=SC2016,SC1004,SC2174,SC2155 + +set -xeu + +# These must be set for the script to run +: "$AWS_ACCESS_KEY_ID" +: "$AWS_SECRET_ACCESS_KEY" +: "$AWS_SESSION_TOKEN" + +ART_VERSION_TAG="8.3.15" +ARTIFACT_PATH="/go/artifacts" +CACHE_DIR="/mnt/createrepo_cache" +GNUPGHOME="/tmpfs/gnupg" +REPO_S3_BUCKET="fred-test1" +BUCKET_CACHE_PATH="/mnt/bucket" +export AWS_REGION="us-west-2" + +: ' +Run command: +docker run \ + --rm -it \ + -v "$(git rev-parse --show-toplevel)":/go/src/github.com/gravitational/teleport/ \ + -v "$HOME/.aws":"/root/.aws" \ + -e AWS_PROFILE="$AWS_PROFILE" \ + -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ + -e DEBIAN_FRONTEND="noninteractive" \ + golang:1.18.4-bullseye /go/src/github.com/gravitational/teleport/build.assets/tooling/cmd/build-os-package-repos/test-rpm.sh +' + +# Download the artifacts +apt update +apt install -y wget +mkdir -pv "$ARTIFACT_PATH" +cd "$ARTIFACT_PATH" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.x86_64.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm64.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.i386.rpm" +wget "https://get.gravitational.com/teleport-${ART_VERSION_TAG}-1.arm.rpm" + +apt install -y createrepo-c gnupg +mkdir -pv "$CACHE_DIR" +mkdir -pv -m0700 "$GNUPGHOME" +chown -R root:root "$GNUPGHOME" +export GPG_TTY=$(tty) +gpg --batch --gen-key <