diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 1a093202421..f07e90b2b89 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -72,7 +72,8 @@ steps: env: PACKAGES: "docker" PLATFORMS: "linux/amd64" - command: ".buildkite/scripts/steps/integration-package.sh" + command: | + .buildkite/scripts/steps/integration-package.sh artifact_paths: - build/distributions/** agents: diff --git a/.buildkite/scripts/buildkite-integration-tests.ps1 b/.buildkite/scripts/buildkite-integration-tests.ps1 index 86b7d39c41a..45f526125eb 100644 --- a/.buildkite/scripts/buildkite-integration-tests.ps1 +++ b/.buildkite/scripts/buildkite-integration-tests.ps1 @@ -25,13 +25,28 @@ go install gotest.tools/gotestsum gotestsum --version $env:TEST_BINARY_NAME = "elastic-agent" -if (-not $env:AGENT_VERSION) { - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) - $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + +if (-not $env:AGENT_VERSION) +{ + if (Test-Path .package-version) + { + $packageContent = Get-Content .package-version -Raw | ConvertFrom-Json + $env:AGENT_VERSION = $packageContent.version + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from .package-version)" + } + else + { + # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 + $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) + $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from version/version.go)" + } +} +else +{ + Write-Output "~~~ Agent version: $env:AGENT_VERSION (specified by env var)" } -Write-Output "~~~ Agent version: $env:AGENT_VERSION" $env:SNAPSHOT = $true Write-Host "~~~ Running integration tests as $env:USERNAME" diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh index b8e56603e2c..c1f926cc03e 100755 --- a/.buildkite/scripts/buildkite-integration-tests.sh +++ b/.buildkite/scripts/buildkite-integration-tests.sh @@ -33,14 +33,20 @@ echo "~~~ Running integration tests as $USER" make install-gotestsum if [[ -z "${AGENT_VERSION:-}" ]]; then - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi -export AGENT_VERSION -echo "~~~ Agent version: ${AGENT_VERSION}" - os_data=$(uname -spr | tr ' ' '_') root_suffix="" if [ "$TEST_SUDO" == "true" ]; then diff --git a/.buildkite/scripts/buildkite-k8s-integration-tests.sh b/.buildkite/scripts/buildkite-k8s-integration-tests.sh index 93054278d14..a1df92e7c6d 100755 --- a/.buildkite/scripts/buildkite-k8s-integration-tests.sh +++ b/.buildkite/scripts/buildkite-k8s-integration-tests.sh @@ -9,9 +9,18 @@ DOCKER_VARIANTS="${DOCKER_VARIANTS:-basic,wolfi,complete,complete-wolfi,service, CLUSTER_NAME="${K8S_VERSION}-kubernetes" if [[ -z "${AGENT_VERSION:-}" ]]; then - # If not specified, use the version in version/version.go - AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi echo "~~~ Create kind cluster '${CLUSTER_NAME}'" diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1 index e0e30245d04..b1c1d46810b 100644 --- a/.buildkite/scripts/steps/ess.ps1 +++ b/.buildkite/scripts/steps/ess.ps1 @@ -1,6 +1,7 @@ function ess_up { param ( [string]$StackVersion, + [string]$StackBuildId = "", [string]$EssRegion = "gcp-us-west2" ) @@ -22,6 +23,7 @@ function ess_up { & terraform init & terraform apply -auto-approve ` -var="stack_version=$StackVersion" ` + -var="stack_build_id=$StackBuildId" ` -var="ess_region=$EssRegion" ` -var="creator=$BuildkiteBuildCreator" ` -var="buildkite_id=$BuildkiteBuildNumber" ` @@ -98,12 +100,13 @@ function Retry-Command { function Get-Ess-Stack { param ( - [string]$StackVersion + [string]$StackVersion, + [string]$StackBuildId = "" ) if ($Env:BUILDKITE_RETRY_COUNT -gt 0) { - Write-Output "The step is retried, starting the ESS stack again" - ess_up $StackVersion + Write-Output "The step is retried, starting the ESS stack again" + ess_up $StackVersion $StackBuildId Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST" } else { # For the first run, we retrieve ESS stack metadata diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh index 093e0bc309c..13b0e9a9ced 100755 --- a/.buildkite/scripts/steps/ess.sh +++ b/.buildkite/scripts/steps/ess.sh @@ -6,7 +6,8 @@ function ess_up() { local WORKSPACE=$(git rev-parse --show-toplevel) local TF_DIR="${WORKSPACE}/test_infra/ess/" local STACK_VERSION=$1 - local ESS_REGION=${2:-"gcp-us-west2"} + local STACK_BUILD_ID=${2:-""} + local ESS_REGION=${3:-"gcp-us-west2"} if [ -z "$STACK_VERSION" ]; then echo "Error: Specify stack version: ess_up [stack_version]" >&2 @@ -22,6 +23,7 @@ function ess_up() { terraform apply \ -auto-approve \ -var="stack_version=${STACK_VERSION}" \ + -var="stack_build_id=${STACK_BUILD_ID}" \ -var="ess_region=${ESS_REGION}" \ -var="creator=${BUILDKITE_BUILD_CREATOR}" \ -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \ diff --git a/.buildkite/scripts/steps/ess_start.sh b/.buildkite/scripts/steps/ess_start.sh index 2062603547f..20372379335 100755 --- a/.buildkite/scripts/steps/ess_start.sh +++ b/.buildkite/scripts/steps/ess_start.sh @@ -4,10 +4,10 @@ set -euo pipefail source .buildkite/scripts/steps/ess.sh source .buildkite/scripts/steps/fleet.sh -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" -ess_up $OVERRIDE_STACK_VERSION +ess_up "$STACK_VERSION" "$STACK_BUILD_ID" preinstall_fleet_packages diff --git a/.buildkite/scripts/steps/integration-cloud-image-push.sh b/.buildkite/scripts/steps/integration-cloud-image-push.sh new file mode 100755 index 00000000000..0292c80f6d8 --- /dev/null +++ b/.buildkite/scripts/steps/integration-cloud-image-push.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +source .buildkite/scripts/common.sh + +echo "~~~ Pushing cloud image" + +suffix="" +if [ "${FIPS:-false}" == "true" ]; then + suffix="-fips" +fi + +CI_ELASTIC_AGENT_DOCKER_IMAGE="docker.elastic.co/beats-ci/elastic-agent-cloud${suffix}" +export CI_ELASTIC_AGENT_DOCKER_IMAGE +echo "CI_ELASTIC_AGENT_DOCKER_IMAGE: ${CI_ELASTIC_AGENT_DOCKER_IMAGE}" + + +export CUSTOM_IMAGE_TAG="git-${BUILDKITE_COMMIT:0:12}" +export USE_PACKAGE_VERSION="true" + +mage cloud:push diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index cd58c2777a8..b3a1594d6ee 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -5,5 +5,6 @@ source .buildkite/scripts/common.sh export SNAPSHOT="true" export EXTERNAL="true" +export USE_PACKAGE_VERSION="true" mage package diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 834da1cd4c6..f3fe7af9a24 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -7,19 +7,9 @@ STACK_PROVISIONER="${1:-"stateful"}" MAGE_TARGET="${2:-"integration:test"}" MAGE_SUBTARGET="${3:-""}" - -# Override the stack version from `.package-version` contents -# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically -# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. - -STACK_VERSION="$(cat .package-version)" -if [[ -n "$STACK_VERSION" ]]; then - STACK_VERSION=${STACK_VERSION}"-SNAPSHOT" -fi - # Run integration tests set +e -AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET +USE_PACKAGE_VERSION=true TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/integration_tests_tf.ps1 b/.buildkite/scripts/steps/integration_tests_tf.ps1 index dda818408be..109c0115e36 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.ps1 +++ b/.buildkite/scripts/steps/integration_tests_tf.ps1 @@ -9,10 +9,15 @@ $PSVersionTable.PSVersion . "$PWD\.buildkite\scripts\steps\ess.ps1" -# Read package version from .package-version file -$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue -if ($PACKAGE_VERSION) { - $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT" +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically +# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready +$packageVersionContent = Get-Content .package-version -Raw -ErrorAction SilentlyContinue | ConvertFrom-Json +if ($packageVersionContent -and $packageVersionContent.version ) { + $STACK_VERSION = $packageVersionContent.version +} +if ($packageVersionContent -and $packageVersionContent.stack_build_id ) { + $STACK_BUILD_ID = $packageVersionContent.stack_build_id } Write-Output "~~~ Building test binaries" @@ -27,7 +32,7 @@ $TestsExitCode = 0 try { Write-Output "~~~ Running integration tests" # Get-Ess-Stack will start the ESS stack if it is a BK retry, otherwise it will retrieve ESS stack metadata - Get-Ess-Stack -StackVersion $PACKAGE_VERSION + Get-Ess-Stack -StackVersion $STACK_VERSION -StackBuildId $STACK_BUILD_ID & "$PWD\.buildkite\scripts\buildkite-integration-tests.ps1" $GROUP_NAME $TEST_SUDO $TestsExitCode = $LASTEXITCODE if ($TestsExitCode -ne 0) diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index 8b60ceae4bd..4ea93e4a00e 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -20,11 +20,11 @@ if [ -z "$TEST_SUDO" ]; then exit 1 fi -# Override the agent package version using a string with format .. -# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" echo "~~~ Building test binaries" mage build:testBinaries @@ -35,7 +35,7 @@ mage build:testBinaries if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then echo "~~~ The steps is retried, starting the ESS stack again" trap 'ess_down' EXIT - ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack") + ess_up "$STACK_VERSION" "$STACK_BUILD_ID" || (echo -e "^^^ +++\nFailed to start ESS stack") else # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data echo "~~~ Receiving ESS stack metadata" diff --git a/.github/workflows/bump-agent-versions.sh b/.github/workflows/bump-agent-versions.sh index ea3941a0ea4..8656535f77e 100755 --- a/.github/workflows/bump-agent-versions.sh +++ b/.github/workflows/bump-agent-versions.sh @@ -25,7 +25,7 @@ else git add testing/integration/testdata/.upgrade-test-agent-versions.yml .package-version nl=$'\n' # otherwise the new line character is not recognized properly - commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" + commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: \`\`\`json${nl}${package_version}${nl}\`\`\`${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" git commit -m "[$current_ref][Automation] Update versions" -m "$commit_desc" git push --set-upstream origin "$pr_branch" diff --git a/.package-version b/.package-version index ff79a643409..529181d18ff 100644 --- a/.package-version +++ b/.package-version @@ -1 +1,8 @@ -8.17.10 \ No newline at end of file +{ + "version": "8.17.10-SNAPSHOT", + "build_id": "8.17.10-3da12952", + "manifest_url": "https://snapshots.elastic.co/8.17.10-3da12952/manifest-8.17.10-SNAPSHOT.json", + "summary_url": "https://snapshots.elastic.co/8.17.10-3da12952/summary-8.17.10-SNAPSHOT.html", + "core_version": "8.17.10", + "stack_build_id": "8.17.10-3da12952-SNAPSHOT" +} \ No newline at end of file diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 190efa543ea..90b8d71a694 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -11,6 +11,7 @@ import ( "log" "os" "path/filepath" + "regexp" "strings" "github.com/josephspurrier/goversioninfo" @@ -34,11 +35,45 @@ type BuildArgs struct { WinMetadata bool // Add resource metadata to Windows binaries (like add the version number to the .exe properties). } +// buildTagRE is a regexp to match strings like "-tags=abcd" +// but does not match "-tags= " +var buildTagRE = regexp.MustCompile(`-tags=([\S]+)?`) + +// ParseBuildTags returns the ExtraFlags param where all flags that are go build tags are joined by a comma. +// +// For example if given -someflag=val1 -tags=buildtag1 -tags=buildtag2 +// It will return -someflag=val1 -tags=buildtag1,buildtag2 +func (b BuildArgs) ParseBuildTags() []string { + flags := make([]string, 0) + if len(b.ExtraFlags) == 0 { + return flags + } + + buildTags := make([]string, 0) + for _, flag := range b.ExtraFlags { + if buildTagRE.MatchString(flag) { + arr := buildTagRE.FindStringSubmatch(flag) + if len(arr) != 2 || arr[1] == "" { + log.Printf("Unexpected format found for buildargs.ExtraFlags, ignoring value %q", flag) + continue + } + buildTags = append(buildTags, arr[1]) + } else { + flags = append(flags, flag) + } + } + if len(buildTags) > 0 { + flags = append(flags, "-tags="+strings.Join(buildTags, ",")) + } + return flags +} + // DefaultBuildArgs returns the default BuildArgs for use in builds. func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + Env: map[string]string{}, Vars: map[string]string{ elasticAgentModulePath + "/version.buildTime": "{{ date }}", elasticAgentModulePath + "/version.commit": "{{ commit }}", @@ -151,6 +186,7 @@ func Build(params BuildArgs) error { if params.CGO { cgoEnabled = "1" } + env["CGO_ENABLED"] = cgoEnabled // Spec @@ -159,7 +195,7 @@ func Build(params BuildArgs) error { "-o", filepath.Join(params.OutputDir, binaryName), } - args = append(args, params.ExtraFlags...) + args = append(args, params.ParseBuildTags()...) // ldflags ldflags := params.LDFlags diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index 100d138195c..d885d70f7ca 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -9,12 +9,12 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/magefile/mage/mg" "github.com/otiai10/copy" "github.com/elastic/elastic-agent/dev-tools/mage/manifest" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ComponentSpecFileSuffix = ".spec.yml" @@ -40,23 +40,53 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) return GetSHA512Hash(targetPath) } -// This is a helper function for flattenDependencies that's used when not packaging from a manifest -func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string { - globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion)) - if mg.Verbose() { - log.Printf("Finding files to copy with %s", globExpr) - } - files, err := filepath.Glob(globExpr) - if err != nil { - panic(err) - } - if mg.Verbose() { - log.Printf("Validating checksums for %+v", files) - log.Printf("--- Copying into %s: %v", versionedDropPath, files) - } - +// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest. +// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed +// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) - for _, f := range files { + + for _, dep := range dependencies { + + if dep.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName) + } + continue + } + + if !dep.SupportsPlatform(platform) { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform) + } + continue + } + + atLeastOnePackageTypeSelected := false + for _, pkgType := range dep.PackageTypes { + if IsPackageTypeSelected(PackageType(pkgType)) { + atLeastOnePackageTypeSelected = true + break + } + } + + if !atLeastOnePackageTypeSelected { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s supported package types %v do not overlap selected package types %v, skipping", dep.ProjectName, dep.BinaryName, dep.PackageTypes, SelectedPackageTypes) + } + continue + } + + srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform)) + + if mg.Verbose() { + log.Printf("Validating checksums for %+v", dep.BinaryName) + log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir) + } + options := copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow @@ -64,217 +94,99 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", f, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath) } - err = copy.Copy(f, versionedDropPath, options) + err := copy.Copy(srcDir, versionedDropPath, options) if err != nil { - panic(err) + panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err)) } // copy spec file for match - specName := filepath.Base(f) - idx := strings.Index(specName, "-"+packageVersion) - if idx != -1 { - specName = specName[:idx] - } if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) + log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName) } - checksum, err := CopyComponentSpecs(specName, versionedDropPath) + checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath) if err != nil { panic(err) } - checksums[specName+ComponentSpecFileSuffix] = checksum + checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum } return checksums } -// This is a helper function for flattenDependencies that's used when building from a manifest -func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string { +// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest. +// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest, +// (there may be some variability there in case the manifest does not include an exact match for the expected filename), +// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) if manifestResponse == nil { return checksums } - // Iterate over the component projects in the manifest - projects := manifestResponse.Projects - for componentName := range projects { - // Iterate over the individual package files within each component project - for pkgName := range projects[componentName].Packages { - // Only care about packages that match the required package constraint (os/arch) - if strings.Contains(pkgName, requiredPackage) { - // Iterate over the external binaries that we care about for packaging agent - for _, spec := range manifest.ExpectedBinaries { - // If the individual package doesn't match the expected prefix, then continue - if !strings.HasPrefix(pkgName, spec.BinaryName) { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage) - } - - // Get the version from the component based on the version in the package name - // This is useful in the case where it's an Independent Agent Release, where - // the opted-in projects will be one patch version ahead of the rest of the - // opted-out/previously-released projects - componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName]) - if mg.Verbose() { - log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion) - } - - // Combine the package name w/ the versioned flat path - fullPath := filepath.Join(versionedFlatPath, pkgName) - - // Eliminate the file extensions to get the proper directory - // name that we need to copy - var dirToCopy string - if strings.HasSuffix(fullPath, ".tar.gz") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")] - } else if strings.HasSuffix(fullPath, ".zip") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")] - } else { - dirToCopy = fullPath - } - if mg.Verbose() { - log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy) - } - - // cloud-defend path exception - // When untarred, cloud defend untars to: - // cloud-defend-8.14.0-arm64 - // but the manifest (and most of this code) expects to be the same as - // the name in the manifest, which is: - // cloud-defend-8.14.0-linux-x86_64 - // So we have to do a bit of a transformation here - if strings.Contains(dirToCopy, "cloud-defend") { - if strings.Contains(dirToCopy, "x86_64") { - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "x86_64", "amd64") - } - if strings.Contains(dirToCopy, "arm64") { - // Not actually replacing the arch, but removing the "linux" - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "arm64", "arm64") - } - if mg.Verbose() { - log.Printf(">>>>>>> Adjusted cloud-defend directory to copy: [%s]", dirToCopy) - } - } - - // Set copy options - options := copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow - }, - Sync: true, - } - if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath) - } - - // Do the copy - err := copy.Copy(dirToCopy, versionedDropPath, options) - if err != nil { - panic(err) - } - - // copy spec file for match - specName := filepath.Base(dirToCopy) - idx := strings.Index(specName, "-"+componentVersion) - if idx != -1 { - specName = specName[:idx] - } - if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) - } - - checksum, err := CopyComponentSpecs(specName, versionedDropPath) - if err != nil { - panic(err) - } - - checksums[specName+ComponentSpecFileSuffix] = checksum - } + // Iterate over the external binaries that we care about for packaging agent + for _, spec := range dependencies { + + if spec.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName) } + continue } - } - return checksums -} + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform) + continue + } -// This function is used when building with a Manifest. In that manifest, it's possible -// for projects in an Independent Agent Release to have different versions since the opted-in -// ones will be one patch version higher than the opted-out/previously released projects. -// This function tries to find the versions from the package name -func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string { - var componentVersion string - var foundIt bool - // Iterate over all the packages in the component project - for pkgName := range componentProject.Packages { - // Only care about the external binaries that we want to package - for _, spec := range manifest.ExpectedBinaries { - // If the given component name doesn't match the external binary component, skip - if componentName != spec.ProjectName { - continue + manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform) + if err != nil { + if mg.Verbose() { + log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform) } + continue + } - // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform) - // Get the second part of the first split - secondHalf := firstSplit[1] - if len(secondHalf) < 2 { - continue - } + // Combine the package name w/ the versioned flat path + fullPath := filepath.Join(versionedFlatPath, rootDir) - // Make sure the second half matches the required package - if strings.Contains(secondHalf, requiredPackage) { - // ignore packages with names where this splitting doesn't results in proper version - if strings.Contains(secondHalf, "docker-image") { - continue - } - if strings.Contains(secondHalf, "oss-") { - continue - } - - // The component version should be the first entry after splitting w/ the requiredPackage - componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0] - foundIt = true - // break out of inner loop - break - } - } - if foundIt { - // break out of outer loop - break + if mg.Verbose() { + log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath) } - } - if componentVersion == "" { - errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName) - panic(errMsg) - } + // Set copy options + options := copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + } + if mg.Verbose() { + log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath) + } - return componentVersion -} + // Do the copy + err = copy.Copy(fullPath, versionedDropPath, options) + if err != nil { + panic(err) + } -// This is a helper function for the cloud-defend package. -// When it is untarred, it does not have the same dirname as the package name. -// This adjusts for that and returns the actual path on disk for cloud-defend -func fixCloudDefendDirPath(dirPath string, componentVersion string, expectedArch string, actualArch string) string { - fixedDirPath := dirPath + checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath) + if err != nil { + panic(err) + } - cloudDefendExpectedDirName := fmt.Sprintf("cloud-defend-%s-linux-%s", componentVersion, expectedArch) - cloudDefendActualDirName := fmt.Sprintf("cloud-defend-%s-%s", componentVersion, actualArch) - if strings.Contains(fixedDirPath, cloudDefendExpectedDirName) { - fixedDirPath = strings.ReplaceAll(fixedDirPath, cloudDefendExpectedDirName, cloudDefendActualDirName) + checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum } - return fixedDirPath + return checksums } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index e99da60cb7d..23b8db0a00d 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild" var Platforms = BuildPlatforms.Defaults() // SelectedPackageTypes is the list of package types. If empty, all packages types -// are considered to be selected (see isPackageTypeSelected). +// are considered to be selected (see IsPackageTypeSelected). var SelectedPackageTypes []PackageType // SelectedDockerVariants is the list of docker variants. If empty, all docker variants -// are considered to be selected (see isDockerVariantSelected). +// are considered to be selected (see IsDockerVariantSelected). var SelectedDockerVariants []DockerVariant func init() { diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 0e0bd2078ca..a3290d69a1a 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -182,6 +182,7 @@ func (b *dockerBuilder) dockerBuild() (string, error) { if b.Snapshot { tag = tag + "-SNAPSHOT" } + if repository := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) } diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index debbcbad6d8..dc9c7d0f66d 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -46,7 +46,7 @@ type TestBinaryArgs struct { } func makeGoTestArgs(name string) GoTestArgs { - fileName := fmt.Sprintf("build/TEST-go-%s", strings.Replace(strings.ToLower(name), " ", "_", -1)) + fileName := fmt.Sprintf("build/TEST-go-%s", strings.ReplaceAll(strings.ToLower(name), " ", "_")) params := GoTestArgs{ LogName: name, Race: RaceDetector, @@ -63,8 +63,8 @@ func makeGoTestArgs(name string) GoTestArgs { func makeGoTestArgsForModule(name, module string) GoTestArgs { fileName := fmt.Sprintf("build/TEST-go-%s-%s", - strings.Replace(strings.ToLower(name), " ", "_", -1), - strings.Replace(strings.ToLower(module), " ", "_", -1), + strings.ReplaceAll(strings.ToLower(name), " ", "_"), + strings.ReplaceAll(strings.ToLower(module), " ", "_"), ) params := GoTestArgs{ LogName: fmt.Sprintf("%s-%s", name, module), diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 1db9c21dbe1..ddc495cb1d3 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -13,13 +13,14 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" "github.com/magefile/mage/mg" "golang.org/x/sync/errgroup" - "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" "github.com/elastic/elastic-agent/pkg/version" ) @@ -94,73 +95,6 @@ var PlatformPackages = map[string]string{ "windows/amd64": "windows-x86_64.zip", } -// ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. -// The project names are those used in the "projects" list in the unified release manifest. -// See the sample manifests in the testdata directory. -var ExpectedBinaries = []BinarySpec{ - {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloud-defend", ProjectName: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: []pkgcommon.PackageType{pkgcommon.Docker}}, - {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}}, - {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, -} - -type BinarySpec struct { - BinaryName string - ProjectName string - Platforms []Platform - PythonWheel bool - PackageTypes []pkgcommon.PackageType -} - -func (proj BinarySpec) SupportsPlatform(platform string) bool { - for _, p := range proj.Platforms { - if p.Platform() == platform { - return true - } - } - return false -} - -func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool { - for _, p := range proj.PackageTypes { - if p == pkgType { - return true - } - } - return false -} - -func (proj BinarySpec) GetPackageName(version string, platform string) string { - if proj.PythonWheel { - return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) - } - return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) -} - -type Platform struct { - OS string - Arch string -} - -// Converts to the format expected on the mage command line "linux", "x86_64" = "linux/amd64" -func (p Platform) Platform() string { - if p.Arch == "x86_64" { - p.Arch = "amd64" - } - if p.Arch == "aarch64" { - p.Arch = "arm64" - } - return p.OS + "/" + p.Arch -} - -var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}} - // DownloadManifest is going to download the given manifest file and return the ManifestResponse func DownloadManifest(ctx context.Context, manifest string) (Build, error) { manifestUrl, urlError := url.Parse(manifest) @@ -192,7 +126,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) { // DownloadComponents is going to download a set of components from the given manifest into the destination // dropPath folder in order to later use that folder for packaging -func DownloadComponents(ctx context.Context, manifest string, platforms []string, dropPath string) error { +func DownloadComponents(ctx context.Context, expectedBinaries []packaging.BinarySpec, manifest string, platforms []string, dropPath string) error { manifestResponse, err := DownloadManifest(ctx, manifest) if err != nil { return fmt.Errorf("failed to download remote manifest file %w", err) @@ -211,27 +145,27 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for _, spec := range ExpectedBinaries { + for _, spec := range expectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) if err != nil { return fmt.Errorf("failed to create directory %s", targetPath) } - log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform) + fmt.Printf("Prepare to download [%s] project [%s] for [%s]\n", spec.BinaryName, spec.ProjectName, platform) if !spec.SupportsPlatform(platform) { - log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform) + fmt.Printf("Binary [%s] does not support platform [%s]\n", spec.BinaryName, platform) continue } - pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) + resolvedPackage, err := ResolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } - for _, p := range pkgURL { - log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) + for _, p := range resolvedPackage.URLs { + fmt.Printf("Downloading [%s] [%s]\n", spec.BinaryName, p) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -252,76 +186,121 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return nil } -func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { - var val Package - var ok bool +type ResolvedPackage struct { + Name string + ActualVersion string + URLs []string +} + +func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { // Try the normal/easy case first - packageName := spec.GetPackageName(version, platform) - val, ok = project.Packages[packageName] - if !ok { - // If we didn't find it, it may be an Independent Agent Release, where - // the opted-in projects will have a patch version one higher than - // the rest of the projects, so we need to seek that out + packageName := spec.GetPackageName(dependencyVersion, platform) + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Got packagename [%s], looking for exact match", packageName) + } + + if exactMatch, ok := project.Packages[packageName]; ok { + // We found the exact filename we are looking for if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) + log.Printf(">>>>>>>>>>> Found exact match packageName for [%s, %s]: %s", project.Branch, project.CommitHash, exactMatch) } - var foundIt bool - for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, spec.BinaryName) { - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + return &ResolvedPackage{ + Name: packageName, + ActualVersion: dependencyVersion, + URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, + }, nil + } - secondHalf := firstSplit[1] - // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, PlatformPackages[platform]) { - - // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) - if len(secondSplit) < 2 { - continue - } - - // The first element after the split should normally be the version - pkgVersion := secondSplit[0] - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion) - } - - // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) - } - - // Get the package value, if it exists - val, ok = project.Packages[foundPkgKey] - if !ok { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey) - } - - foundIt = true - } - } - } + // If we didn't find it, it may be an Independent Agent Release, where + // the opted-in projects will have a patch version one higher than + // the rest of the projects, so we "relax" the version constraint + return resolveManifestPackageUsingRelaxedVersion(project, spec, dependencyVersion, platform) +} - if !foundIt { - return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL) - } +func resolveManifestPackageUsingRelaxedVersion(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { + // start with the rendered package name + packageName := spec.GetPackageName(dependencyVersion, platform) + + // Find the original version in the rendered filename + versionIndex := strings.Index(packageName, dependencyVersion) + if versionIndex == -1 { + return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion) + } + + // obtain a regexp from the exact version string that allows for some flexibility on patch version, prerelease and build metadata tokens + relaxedVersion, err := relaxVersion(dependencyVersion) + if err != nil { + return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err) } if mg.Verbose() { - log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash) + log.Printf(">>>>>>>>>>> Couldn't find exact match, relaxing agent dependencyVersion to %s", relaxedVersion) } - return []string{val.URL, val.ShaURL, val.AscURL}, nil + // locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that + relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex]) + relaxedPackageName += `(?P` + relaxedVersion + `)` + relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):]) + + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Attempting to match a filename with %s", relaxedPackageName) + } + + relaxedPackageNameRegexp, err := regexp.Compile(relaxedPackageName) + if err != nil { + return nil, fmt.Errorf("compiling relaxed package name regex %q: %w", relaxedPackageName, err) + } + + for pkgName, pkg := range project.Packages { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName) + } + if submatches := relaxedPackageNameRegexp.FindStringSubmatch(pkgName); len(submatches) > 0 { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName) + } + return &ResolvedPackage{ + Name: pkgName, + ActualVersion: submatches[1], + URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, + }, nil + } + } + + return nil, fmt.Errorf("package [%s] not found in project manifest at %s using relaxed version %q", packageName, project.ExternalArtifactsManifestURL, relaxedPackageName) +} + +// versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?) +const versionRegexp = `^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` +const anyPatchVersionRegexp = `(?:0|[1-9]\d*)` + +var versionRegExp = regexp.MustCompile(versionRegexp) + +func relaxVersion(version string) (string, error) { + matchIndices := versionRegExp.FindSubmatchIndex([]byte(version)) + // Matches index pairs are (0,1) for the whole regexp and (2,3) for the patch group + // check that we have matched correctly + if len(matchIndices) < 4 { + return "", fmt.Errorf("failed to match regexp for version [%s]", version) + } + + // take the starting index of the patch version + patchStartIndex := matchIndices[2] + // copy everything before the patch version escaping the regexp + relaxedVersion := regexp.QuoteMeta(version[:patchStartIndex]) + // add the patch regexp + relaxedVersion += anyPatchVersionRegexp + // check if there's more characters after the patch version + remainderIndex := matchIndices[3] + if remainderIndex < len(version) { + // This is a looser regexp that allows anything beyond the major version to change (while still enforcing a valid patch version though) + // see TestResolveManifestPackage/Independent_Agent_Staging_8.14_apm-server and TestResolveManifestPackage/Independent_Agent_Staging_8.14_endpoint-dev + // Be more relaxed and allow for any character sequence after this + relaxedVersion += `.*` + } + return relaxedVersion, nil } func DownloadPackage(ctx context.Context, downloadUrl string, target string) error { @@ -331,12 +310,12 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err } valid := false for _, manifestHost := range AllowedManifestHosts { - if manifestHost == parsedURL.Host { + if manifestHost == parsedURL.Hostname() { valid = true } } if !valid { - log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts) + log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Hostname(), AllowedManifestHosts) return errorNotAllowedManifestURL } cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path) diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index b975149aa27..76a0d53cd2e 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -7,12 +7,15 @@ package manifest import ( _ "embed" "encoding/json" + "fmt" "log" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) var ( @@ -135,7 +138,7 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - spec, ok := findBinarySpec(tc.binary) + spec, ok := findBinarySpec(t, tc.binary) assert.True(t, ok) if !spec.SupportsPlatform(tc.platform) { @@ -143,22 +146,80 @@ func TestResolveManifestPackage(t *testing.T) { return } - urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) + resolvedPackage, err := ResolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) + require.NotNil(t, resolvedPackage) - assert.Len(t, urlList, 3) - for _, url := range urlList { + assert.Len(t, resolvedPackage.URLs, 3) + for _, url := range resolvedPackage.URLs { assert.Contains(t, tc.expectedUrlList, url) } }) } } -func findBinarySpec(name string) (BinarySpec, bool) { - for _, spec := range ExpectedBinaries { +func findBinarySpec(t *testing.T, name string) (packaging.BinarySpec, bool) { + components, err := packaging.Components() + require.NoError(t, err, "error loading components from packages.yml") + + for _, spec := range components { if spec.BinaryName == name { return spec, true } } - return BinarySpec{}, false + return packaging.BinarySpec{}, false +} + +func TestRelaxVersion(t *testing.T) { + type args struct { + version string + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "major-minor-patch", + args: args{ + version: "1.2.3", + }, + want: `1\.2\.(?:0|[1-9]\d*)`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot", + args: args{ + version: "1.2.3-SNAPSHOT", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot-buildmeta", + args: args{ + version: "1.2.3-SNAPSHOT+build20250328112233", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "not a semver", + args: args{ + version: "foobar", + }, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relaxVersion(tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("relaxVersion(%v)", tt.args.version)) { + return + } + assert.Equalf(t, tt.want, got, "relaxVersion(%v)", tt.args.version) + }) + } } diff --git a/dev-tools/mage/packageversion.go b/dev-tools/mage/packageversion.go new file mode 100644 index 00000000000..81f23132ab9 --- /dev/null +++ b/dev-tools/mage/packageversion.go @@ -0,0 +1,113 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package mage + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +const PackageVersionFilename = ".package-version" + +type packageVersion struct { + Version string `json:"version"` + BuildID string `json:"build_id"` + ManifestURL string `json:"manifest_url"` + SummaryURL string `json:"summary_url"` + CoreVersion string `json:"core_version"` + StackBuildID string `json:"stack_build_id"` +} + +func initPackageVersion() error { + if os.Getenv("USE_PACKAGE_VERSION") != "true" { + return nil + } + + _, err := os.Stat(PackageVersionFilename) + if err != nil { + if os.IsNotExist(err) { + log.Printf("USE_PACKAGE_VERSION is set, but %q does not exist, not overriding\n", PackageVersionFilename) + return nil + } + return fmt.Errorf("failed to stat %q: %w", PackageVersionFilename, err) + } + + pv, err := readPackageVersion() + if err != nil { + // err is wrapped in readPackageVersion + return err + } + + PackagingFromManifest = true + ManifestURL = pv.ManifestURL + agentPackageVersion = pv.CoreVersion + Snapshot = true + + _ = os.Setenv("BEAT_VERSION", pv.CoreVersion) + _ = os.Setenv("AGENT_VERSION", pv.Version) + _ = os.Setenv("AGENT_STACK_VERSION", pv.Version) + _ = os.Setenv("SNAPSHOT", "true") + + dropPath := filepath.Join("build", "distributions", "elastic-agent-drop") + dropPath, err = filepath.Abs(dropPath) + if err != nil { + return fmt.Errorf("failed to obtain absolute path for default drop path: %w", err) + } + + _ = os.Setenv("AGENT_DROP_PATH", dropPath) + + return nil +} + +func UpdatePackageVersion(version string, buildID string, manifestURL string, summaryURL string) error { + packageVersion := packageVersion{ + Version: version, + BuildID: buildID, + ManifestURL: manifestURL, + SummaryURL: summaryURL, + CoreVersion: strings.ReplaceAll(version, "-SNAPSHOT", ""), + StackBuildID: fmt.Sprintf("%s-SNAPSHOT", buildID), + } + + if err := writePackageVersion(packageVersion); err != nil { + // err is wrapped in writePackageVersion + return err + } + return nil +} + +func writePackageVersion(pv packageVersion) error { + pvBytes, err := json.MarshalIndent(pv, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal package version: %w", err) + } + + err = os.WriteFile(PackageVersionFilename, pvBytes, 0644) + if err != nil { + return fmt.Errorf("failed to write package version: %w", err) + } + + return nil +} + +func readPackageVersion() (*packageVersion, error) { + f, err := os.Open(PackageVersionFilename) + if err != nil { + return nil, fmt.Errorf("failed to open %q for read: %w", PackageVersionFilename, err) + } + defer f.Close() + + decoder := json.NewDecoder(f) + pVersion := &packageVersion{} + err = decoder.Decode(pVersion) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML from file %q: %w", PackageVersionFilename, err) + } + return pVersion, nil +} diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index a3917ddac1d..3032886ebe3 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,6 +34,15 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms + if mg.Verbose() { + debugSelectedPackageSpecsWithPlatform := make([]string, 0, len(Packages)) + for _, p := range Packages { + debugSelectedPackageSpecsWithPlatform = append(debugSelectedPackageSpecsWithPlatform, fmt.Sprintf("spec %s on %s/%s", p.Spec.Name, p.OS, p.Arch)) + } + + log.Printf("Packaging for platforms %v, packages %v", platforms, debugSelectedPackageSpecsWithPlatform) + } + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { @@ -42,12 +51,12 @@ func Package() error { } for _, pkgType := range pkg.Types { - if !isPackageTypeSelected(pkgType) { + if !IsPackageTypeSelected(pkgType) { log.Printf("Skipping %s package type because it is not selected", pkgType) continue } - if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + if pkgType == Docker && !IsDockerVariantSelected(pkg.Spec.DockerVariant) { log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) continue } @@ -99,6 +108,10 @@ func Package() error { spec = spec.Evaluate() + if mg.Verbose() { + log.Printf("Adding task for packaging %s on %s/%s", spec.Name, target.GOOS(), target.Arch()) + } + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } @@ -111,9 +124,9 @@ func Package() error { return nil } -// isPackageTypeSelected returns true if SelectedPackageTypes is empty or if +// IsPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. -func isPackageTypeSelected(pkgType PackageType) bool { +func IsPackageTypeSelected(pkgType PackageType) bool { if len(SelectedPackageTypes) == 0 { return true } @@ -126,9 +139,9 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } -// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// IsDockerVariantSelected returns true if SelectedDockerVariants is empty or if // docVariant is present on SelectedDockerVariants. It returns false otherwise. -func isDockerVariantSelected(docVariant DockerVariant) bool { +func IsDockerVariantSelected(docVariant DockerVariant) bool { if len(SelectedDockerVariants) == 0 { return true } @@ -221,7 +234,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-v") } - args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go")) + args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/testing/package_test.go")) if params.HasModules { args = append(args, "--modules") diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index c1bcee157f9..a3189eff559 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -28,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ( @@ -104,6 +105,7 @@ type PackageSpec struct { Qualifier string `yaml:"qualifier,omitempty"` // Optional OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional + Components []packaging.BinarySpec `yaml:"components"` // Optional: Components required for this package evalContext map[string]interface{} packageDir string diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 32bc62a3e97..cec6c7b0376 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -159,6 +159,13 @@ func initGlobals() { ManifestURL = EnvOr(ManifestUrlEnvVar, "") PackagingFromManifest = ManifestURL != "" + + // order matters this must be called last as it will override some of the + // values above + err = initPackageVersion() + if err != nil { + panic(fmt.Errorf("failed to init package version: %w", err)) + } } // ProjectType specifies the type of project (OSS vs X-Pack). diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 898d963c3fa..c5856ecc342 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1,9 +1,155 @@ --- -# This file contains the package specifications for both Community Beats and -# Official Beats. The shared section contains YAML anchors that are used to +# This file contains the package specifications for Elastic Agent + +# List all the available platforms +platforms: &all-platforms + - &linux-amd64 + os: linux + arch: x86_64 + - &linux-arm64 + os: linux + arch: arm64 + - &windows-amd64 + os: windows + arch: x86_64 + - &darwin-amd64 + os: darwin + arch: x86_64 + - &darwin-arm64 + os: darwin + arch: aarch64 + +# List all the package type constants (see dev-tools/mage/pkgcommon/pkgcommon-types.go) +packageTypes: &all-package-types + - &pkg-type-rpm + 1 # RPM + - &pkg-type-deb + 2 # Deb + - &pkg-type-zip + 3 # zip + - &pkg-type-targz + 4 # tar.gz + - &pkg-type-docker + 5 # docker + +# Settings section contains general compiling and packaging settings +# settings: + +# List *all* the components available for packaging in elastic-agent +components: + # general template for new components (all attributes are mandatory) + # - &comp- + # projectName: + # packageName: