diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 242192bb682..f07e90b2b89 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -34,23 +34,6 @@ steps: machineType: "n2-standard-8" image: "${IMAGE_UBUNTU_2204_X86_64}" - - label: "Packaging: Ubuntu x86_64 FIPS" - key: "packaging-ubuntu-x86-64-fips" - env: - PACKAGES: "tar.gz" - PLATFORMS: "linux/amd64" - FIPS: "true" - command: ".buildkite/scripts/steps/integration-package.sh" - artifact_paths: - - build/distributions/** - retry: - automatic: - limit: 1 - agents: - provider: "gcp" - machineType: "n2-standard-4" - image: "${IMAGE_UBUNTU_2204_X86_64}" - # Packaging linux/arm64 - label: "Packaging: linux/arm64 tar.gz" key: packaging-ubuntu-arm64 @@ -68,23 +51,6 @@ steps: instanceType: "c6g.4xlarge" image: "${IMAGE_UBUNTU_2204_ARM_64}" - - label: "Packaging: Ubuntu arm64 FIPS" - key: "packaging-ubuntu-arm64-fips" - env: - PACKAGES: "tar.gz" - PLATFORMS: "linux/arm64" - FIPS: "true" - command: ".buildkite/scripts/steps/integration-package.sh" - artifact_paths: - - build/distributions/** - retry: - automatic: - limit: 1 - agents: - provider: "aws" - instanceType: "c6g.4xlarge" - image: "${IMAGE_UBUNTU_2204_ARM_64}" - - label: "Packaging: windows/amd64 zip" key: packaging-windows env: @@ -106,7 +72,8 @@ steps: env: PACKAGES: "docker" PLATFORMS: "linux/amd64" - command: ".buildkite/scripts/steps/integration-package.sh" + command: | + .buildkite/scripts/steps/integration-package.sh artifact_paths: - build/distributions/** agents: diff --git a/.buildkite/pipeline.elastic-agent-binary-dra.yml b/.buildkite/pipeline.elastic-agent-binary-dra.yml index ae7b3f9a190..bc1796fd176 100644 --- a/.buildkite/pipeline.elastic-agent-binary-dra.yml +++ b/.buildkite/pipeline.elastic-agent-binary-dra.yml @@ -32,7 +32,7 @@ steps: env: DRA_WORKFLOW: "snapshot" PLATFORMS: "linux/amd64 windows/amd64 darwin/amd64" - + - label: ":package: linux/arm64 darwin/arm64 Elastic-Agent Core Snapshot" commands: - .buildkite/scripts/steps/build-agent-core.sh diff --git a/.buildkite/scripts/buildkite-integration-tests.ps1 b/.buildkite/scripts/buildkite-integration-tests.ps1 index 86b7d39c41a..45f526125eb 100644 --- a/.buildkite/scripts/buildkite-integration-tests.ps1 +++ b/.buildkite/scripts/buildkite-integration-tests.ps1 @@ -25,13 +25,28 @@ go install gotest.tools/gotestsum gotestsum --version $env:TEST_BINARY_NAME = "elastic-agent" -if (-not $env:AGENT_VERSION) { - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) - $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + +if (-not $env:AGENT_VERSION) +{ + if (Test-Path .package-version) + { + $packageContent = Get-Content .package-version -Raw | ConvertFrom-Json + $env:AGENT_VERSION = $packageContent.version + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from .package-version)" + } + else + { + # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 + $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) + $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from version/version.go)" + } +} +else +{ + Write-Output "~~~ Agent version: $env:AGENT_VERSION (specified by env var)" } -Write-Output "~~~ Agent version: $env:AGENT_VERSION" $env:SNAPSHOT = $true Write-Host "~~~ Running integration tests as $env:USERNAME" diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh index 36017a5ee8c..7d8df202b7b 100755 --- a/.buildkite/scripts/buildkite-integration-tests.sh +++ b/.buildkite/scripts/buildkite-integration-tests.sh @@ -33,14 +33,20 @@ echo "~~~ Running integration tests as $USER" make install-gotestsum if [[ -z "${AGENT_VERSION:-}" ]]; then - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi -export AGENT_VERSION -echo "~~~ Agent version: ${AGENT_VERSION}" - os_data=$(uname -spr | tr ' ' '_') root_suffix="" if [ "$TEST_SUDO" == "true" ]; then diff --git a/.buildkite/scripts/buildkite-k8s-integration-tests.sh b/.buildkite/scripts/buildkite-k8s-integration-tests.sh index 9ff72207eb4..84c4c06d650 100755 --- a/.buildkite/scripts/buildkite-k8s-integration-tests.sh +++ b/.buildkite/scripts/buildkite-k8s-integration-tests.sh @@ -9,9 +9,18 @@ DOCKER_VARIANTS="${DOCKER_VARIANTS:-basic,wolfi,complete,complete-wolfi,service, CLUSTER_NAME="${K8S_VERSION}-kubernetes" if [[ -z "${AGENT_VERSION:-}" ]]; then - # If not specified, use the version in version/version.go - AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi echo "~~~ Create kind cluster '${CLUSTER_NAME}'" diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1 index e0e30245d04..6ab4dceb8e9 100644 --- a/.buildkite/scripts/steps/ess.ps1 +++ b/.buildkite/scripts/steps/ess.ps1 @@ -1,11 +1,12 @@ function ess_up { param ( [string]$StackVersion, + [string]$StackBuildId = "", [string]$EssRegion = "gcp-us-west2" ) Write-Output "~~~ Starting ESS Stack" - + $Workspace = & git rev-parse --show-toplevel $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/" @@ -13,7 +14,7 @@ function ess_up { Write-Error "Error: Specify stack version: ess_up [stack_version]" return 1 } - + $BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email } $BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" } $BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" } @@ -22,6 +23,7 @@ function ess_up { & terraform init & terraform apply -auto-approve ` -var="stack_version=$StackVersion" ` + -var="stack_build_id=$StackBuildId" ` -var="ess_region=$EssRegion" ` -var="creator=$BuildkiteBuildCreator" ` -var="buildkite_id=$BuildkiteBuildNumber" ` @@ -33,10 +35,11 @@ function ess_up { $Env:KIBANA_HOST = & terraform output -raw kibana_endpoint $Env:KIBANA_USERNAME = $Env:ELASTICSEARCH_USERNAME $Env:KIBANA_PASSWORD = $Env:ELASTICSEARCH_PASSWORD + $Env:INTEGRATIONS_SERVER_HOST = & terraform output -raw integrations_server_endpoint Pop-Location } -function ess_down { +function ess_down { $Workspace = & git rev-parse --show-toplevel $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/" $stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate" @@ -80,11 +83,11 @@ function Retry-Command { $lastError = $null for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { - try { - $result = & $ScriptBlock + try { + $result = & $ScriptBlock return $result } - catch { + catch { $lastError = $_ Write-Warning "Attempt $attempt failed: $($_.Exception.Message)" Write-Warning "Retrying in $DelaySeconds seconds..." @@ -98,14 +101,17 @@ function Retry-Command { function Get-Ess-Stack { param ( - [string]$StackVersion + [string]$StackVersion, + [string]$StackBuildId = "" ) - + if ($Env:BUILDKITE_RETRY_COUNT -gt 0) { - Write-Output "The step is retried, starting the ESS stack again" - ess_up $StackVersion + Write-Output "The step is retried, starting the ESS stack again" + ess_up $StackVersion $StackBuildId Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST" } else { + # TODO: Use a metadata prefix for "fips." if we ever need to test Windows artifacts for FIPS. + # For the first run, we retrieve ESS stack metadata Write-Output "~~~ Receiving ESS stack metadata" $Env:ELASTICSEARCH_HOST = & buildkite-agent meta-data get "es.host" diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh index 093e0bc309c..13b0e9a9ced 100755 --- a/.buildkite/scripts/steps/ess.sh +++ b/.buildkite/scripts/steps/ess.sh @@ -6,7 +6,8 @@ function ess_up() { local WORKSPACE=$(git rev-parse --show-toplevel) local TF_DIR="${WORKSPACE}/test_infra/ess/" local STACK_VERSION=$1 - local ESS_REGION=${2:-"gcp-us-west2"} + local STACK_BUILD_ID=${2:-""} + local ESS_REGION=${3:-"gcp-us-west2"} if [ -z "$STACK_VERSION" ]; then echo "Error: Specify stack version: ess_up [stack_version]" >&2 @@ -22,6 +23,7 @@ function ess_up() { terraform apply \ -auto-approve \ -var="stack_version=${STACK_VERSION}" \ + -var="stack_build_id=${STACK_BUILD_ID}" \ -var="ess_region=${ESS_REGION}" \ -var="creator=${BUILDKITE_BUILD_CREATOR}" \ -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \ diff --git a/.buildkite/scripts/steps/ess_start.sh b/.buildkite/scripts/steps/ess_start.sh index 2062603547f..20372379335 100755 --- a/.buildkite/scripts/steps/ess_start.sh +++ b/.buildkite/scripts/steps/ess_start.sh @@ -4,10 +4,10 @@ set -euo pipefail source .buildkite/scripts/steps/ess.sh source .buildkite/scripts/steps/fleet.sh -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" -ess_up $OVERRIDE_STACK_VERSION +ess_up "$STACK_VERSION" "$STACK_BUILD_ID" preinstall_fleet_packages diff --git a/.buildkite/scripts/steps/integration-cloud-image-push.sh b/.buildkite/scripts/steps/integration-cloud-image-push.sh new file mode 100755 index 00000000000..0292c80f6d8 --- /dev/null +++ b/.buildkite/scripts/steps/integration-cloud-image-push.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +source .buildkite/scripts/common.sh + +echo "~~~ Pushing cloud image" + +suffix="" +if [ "${FIPS:-false}" == "true" ]; then + suffix="-fips" +fi + +CI_ELASTIC_AGENT_DOCKER_IMAGE="docker.elastic.co/beats-ci/elastic-agent-cloud${suffix}" +export CI_ELASTIC_AGENT_DOCKER_IMAGE +echo "CI_ELASTIC_AGENT_DOCKER_IMAGE: ${CI_ELASTIC_AGENT_DOCKER_IMAGE}" + + +export CUSTOM_IMAGE_TAG="git-${BUILDKITE_COMMIT:0:12}" +export USE_PACKAGE_VERSION="true" + +mage cloud:push diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index cd58c2777a8..b3a1594d6ee 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -5,5 +5,6 @@ source .buildkite/scripts/common.sh export SNAPSHOT="true" export EXTERNAL="true" +export USE_PACKAGE_VERSION="true" mage package diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 834da1cd4c6..f3fe7af9a24 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -7,19 +7,9 @@ STACK_PROVISIONER="${1:-"stateful"}" MAGE_TARGET="${2:-"integration:test"}" MAGE_SUBTARGET="${3:-""}" - -# Override the stack version from `.package-version` contents -# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically -# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. - -STACK_VERSION="$(cat .package-version)" -if [[ -n "$STACK_VERSION" ]]; then - STACK_VERSION=${STACK_VERSION}"-SNAPSHOT" -fi - # Run integration tests set +e -AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET +USE_PACKAGE_VERSION=true TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/integration_tests_tf.ps1 b/.buildkite/scripts/steps/integration_tests_tf.ps1 index dda818408be..109c0115e36 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.ps1 +++ b/.buildkite/scripts/steps/integration_tests_tf.ps1 @@ -9,10 +9,15 @@ $PSVersionTable.PSVersion . "$PWD\.buildkite\scripts\steps\ess.ps1" -# Read package version from .package-version file -$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue -if ($PACKAGE_VERSION) { - $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT" +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically +# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready +$packageVersionContent = Get-Content .package-version -Raw -ErrorAction SilentlyContinue | ConvertFrom-Json +if ($packageVersionContent -and $packageVersionContent.version ) { + $STACK_VERSION = $packageVersionContent.version +} +if ($packageVersionContent -and $packageVersionContent.stack_build_id ) { + $STACK_BUILD_ID = $packageVersionContent.stack_build_id } Write-Output "~~~ Building test binaries" @@ -27,7 +32,7 @@ $TestsExitCode = 0 try { Write-Output "~~~ Running integration tests" # Get-Ess-Stack will start the ESS stack if it is a BK retry, otherwise it will retrieve ESS stack metadata - Get-Ess-Stack -StackVersion $PACKAGE_VERSION + Get-Ess-Stack -StackVersion $STACK_VERSION -StackBuildId $STACK_BUILD_ID & "$PWD\.buildkite\scripts\buildkite-integration-tests.ps1" $GROUP_NAME $TEST_SUDO $TestsExitCode = $LASTEXITCODE if ($TestsExitCode -ne 0) diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index 8b60ceae4bd..4ea93e4a00e 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -20,11 +20,11 @@ if [ -z "$TEST_SUDO" ]; then exit 1 fi -# Override the agent package version using a string with format .. -# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" echo "~~~ Building test binaries" mage build:testBinaries @@ -35,7 +35,7 @@ mage build:testBinaries if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then echo "~~~ The steps is retried, starting the ESS stack again" trap 'ess_down' EXIT - ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack") + ess_up "$STACK_VERSION" "$STACK_BUILD_ID" || (echo -e "^^^ +++\nFailed to start ESS stack") else # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data echo "~~~ Receiving ESS stack metadata" diff --git a/.github/workflows/bump-agent-versions.sh b/.github/workflows/bump-agent-versions.sh index ea3941a0ea4..8656535f77e 100755 --- a/.github/workflows/bump-agent-versions.sh +++ b/.github/workflows/bump-agent-versions.sh @@ -25,7 +25,7 @@ else git add testing/integration/testdata/.upgrade-test-agent-versions.yml .package-version nl=$'\n' # otherwise the new line character is not recognized properly - commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" + commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: \`\`\`json${nl}${package_version}${nl}\`\`\`${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" git commit -m "[$current_ref][Automation] Update versions" -m "$commit_desc" git push --set-upstream origin "$pr_branch" diff --git a/.package-version b/.package-version index 171d800e0f7..b40dda4c48d 100644 --- a/.package-version +++ b/.package-version @@ -1 +1,8 @@ -9.0.5 \ No newline at end of file +{ + "version": "9.0.5-SNAPSHOT", + "build_id": "9.0.5-122e6633", + "manifest_url": "https://snapshots.elastic.co/9.0.5-122e6633/manifest-9.0.5-SNAPSHOT.json", + "summary_url": "https://snapshots.elastic.co/9.0.5-122e6633/summary-9.0.5-SNAPSHOT.html", + "core_version": "9.0.5", + "stack_build_id": "9.0.5-122e6633-SNAPSHOT" +} \ No newline at end of file diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 92da55273b1..90b8d71a694 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -73,6 +73,7 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + Env: map[string]string{}, Vars: map[string]string{ elasticAgentModulePath + "/version.buildTime": "{{ date }}", elasticAgentModulePath + "/version.commit": "{{ commit }}", @@ -87,11 +88,6 @@ func DefaultBuildArgs() BuildArgs { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } - if FIPSBuild { - args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips") - args.CGO = true - } - if DevBuild { // Disable optimizations (-N) and inlining (-l) for debugging. args.ExtraFlags = append(args.ExtraFlags, `-gcflags=all=-N -l`) @@ -191,11 +187,6 @@ func Build(params BuildArgs) error { cgoEnabled = "1" } - if FIPSBuild { - cgoEnabled = "1" - env["GOEXPERIMENT"] = "systemcrypto" - } - env["CGO_ENABLED"] = cgoEnabled // Spec diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index f76ab54c849..7d57c204a87 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -9,12 +9,12 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/magefile/mage/mg" "github.com/otiai10/copy" "github.com/elastic/elastic-agent/dev-tools/mage/manifest" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ComponentSpecFileSuffix = ".spec.yml" @@ -40,23 +40,36 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) return GetSHA512Hash(targetPath) } -// This is a helper function for flattenDependencies that's used when not packaging from a manifest -func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string { - globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion)) - if mg.Verbose() { - log.Printf("Finding files to copy with %s", globExpr) - } - files, err := filepath.Glob(globExpr) - if err != nil { - panic(err) - } - if mg.Verbose() { - log.Printf("Validating checksums for %+v", files) - log.Printf("--- Copying into %s: %v", versionedDropPath, files) - } - +// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest. +// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed +// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) - for _, f := range files { + + for _, dep := range dependencies { + + if dep.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName) + } + continue + } + + if !dep.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform) + continue + } + + srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform)) + + if mg.Verbose() { + log.Printf("Validating checksums for %+v", dep.BinaryName) + log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir) + } + options := copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow @@ -64,182 +77,99 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", f, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath) } - err = copy.Copy(f, versionedDropPath, options) + err := copy.Copy(srcDir, versionedDropPath, options) if err != nil { - panic(err) + panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err)) } // copy spec file for match - specName := filepath.Base(f) - idx := strings.Index(specName, "-"+packageVersion) - if idx != -1 { - specName = specName[:idx] - } if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) + log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName) } - checksum, err := CopyComponentSpecs(specName, versionedDropPath) + checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath) if err != nil { panic(err) } - checksums[specName+ComponentSpecFileSuffix] = checksum + checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum } return checksums } -// This is a helper function for flattenDependencies that's used when building from a manifest -func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string { +// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest. +// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest, +// (there may be some variability there in case the manifest does not include an exact match for the expected filename), +// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) if manifestResponse == nil { return checksums } - // Iterate over the component projects in the manifest - projects := manifestResponse.Projects - for componentName := range projects { - // Iterate over the individual package files within each component project - for pkgName := range projects[componentName].Packages { - // Only care about packages that match the required package constraint (os/arch) - if strings.Contains(pkgName, requiredPackage) { - // Iterate over the external binaries that we care about for packaging agent - for _, spec := range manifest.ExpectedBinaries { - // If the individual package doesn't match the expected prefix, then continue - if !strings.HasPrefix(pkgName, spec.BinaryName) { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage) - } - - // Get the version from the component based on the version in the package name - // This is useful in the case where it's an Independent Agent Release, where - // the opted-in projects will be one patch version ahead of the rest of the - // opted-out/previously-released projects - componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName]) - if mg.Verbose() { - log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion) - } - - // Combine the package name w/ the versioned flat path - fullPath := filepath.Join(versionedFlatPath, pkgName) - - // Eliminate the file extensions to get the proper directory - // name that we need to copy - var dirToCopy string - if strings.HasSuffix(fullPath, ".tar.gz") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")] - } else if strings.HasSuffix(fullPath, ".zip") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")] - } else { - dirToCopy = fullPath - } - if mg.Verbose() { - log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy) - } - - // Set copy options - options := copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow - }, - Sync: true, - } - if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath) - } - - // Do the copy - err := copy.Copy(dirToCopy, versionedDropPath, options) - if err != nil { - panic(err) - } - - // copy spec file for match - specName := filepath.Base(dirToCopy) - idx := strings.Index(specName, "-"+componentVersion) - if idx != -1 { - specName = specName[:idx] - } - if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) - } - - checksum, err := CopyComponentSpecs(specName, versionedDropPath) - if err != nil { - panic(err) - } - - checksums[specName+ComponentSpecFileSuffix] = checksum - } + // Iterate over the external binaries that we care about for packaging agent + for _, spec := range dependencies { + + if spec.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName) } + continue } - } - return checksums -} + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform) + continue + } -// This function is used when building with a Manifest. In that manifest, it's possible -// for projects in an Independent Agent Release to have different versions since the opted-in -// ones will be one patch version higher than the opted-out/previously released projects. -// This function tries to find the versions from the package name -func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string { - var componentVersion string - var foundIt bool - // Iterate over all the packages in the component project - for pkgName := range componentProject.Packages { - // Only care about the external binaries that we want to package - for _, spec := range manifest.ExpectedBinaries { - // If the given component name doesn't match the external binary component, skip - if componentName != spec.ProjectName { - continue + manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform) + if err != nil { + if mg.Verbose() { + log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform) } + continue + } - // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform) - // Get the second part of the first split - secondHalf := firstSplit[1] - if len(secondHalf) < 2 { - continue - } + // Combine the package name w/ the versioned flat path + fullPath := filepath.Join(versionedFlatPath, rootDir) - // Make sure the second half matches the required package - if strings.Contains(secondHalf, requiredPackage) { - // ignore packages with names where this splitting doesn't results in proper version - if strings.Contains(secondHalf, "docker-image") { - continue - } - if strings.Contains(secondHalf, "oss-") { - continue - } - - // The component version should be the first entry after splitting w/ the requiredPackage - componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0] - foundIt = true - // break out of inner loop - break - } + if mg.Verbose() { + log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath) } - if foundIt { - // break out of outer loop - break + + // Set copy options + options := copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + } + if mg.Verbose() { + log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath) + } + + // Do the copy + err = copy.Copy(fullPath, versionedDropPath, options) + if err != nil { + panic(err) } - } - if componentVersion == "" { - errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName) - panic(errMsg) + checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath) + if err != nil { + panic(err) + } + + checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum } - return componentVersion + return checksums } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index eb18acfb422..23b8db0a00d 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild" var Platforms = BuildPlatforms.Defaults() // SelectedPackageTypes is the list of package types. If empty, all packages types -// are considered to be selected (see isPackageTypeSelected). +// are considered to be selected (see IsPackageTypeSelected). var SelectedPackageTypes []PackageType // SelectedDockerVariants is the list of docker variants. If empty, all docker variants -// are considered to be selected (see isDockerVariantSelected). +// are considered to be selected (see IsDockerVariantSelected). var SelectedDockerVariants []DockerVariant func init() { @@ -249,10 +249,6 @@ func CrossBuildImage(platform string) (string, error) { return "", err } - if FIPSBuild { - tagSuffix += "-fips" - } - return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil } @@ -344,7 +340,6 @@ func (b GolangCrossBuilder) Build() error { "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), "--env", fmt.Sprintf("DEV=%v", DevBuild), "--env", fmt.Sprintf("EXTERNAL=%v", ExternalBuild), - "--env", fmt.Sprintf("FIPS=%v", FIPSBuild), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, image, diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 8bc7498d9f5..627c665bbcb 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -227,9 +227,7 @@ func (b *dockerBuilder) dockerBuild() (string, []string, error) { if b.Snapshot { mainTag = mainTag + "-SNAPSHOT" } - if b.FIPS { - mainTag = mainTag + "-fips" - } + if repository := b.ExtraVars["repository"]; repository != "" { mainTag = fmt.Sprintf("%s/%s", repository, mainTag) } diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index 7917fa68ebc..332b9ff6f9f 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -46,7 +46,7 @@ type TestBinaryArgs struct { } func makeGoTestArgs(name string) GoTestArgs { - fileName := fmt.Sprintf("build/TEST-go-%s", strings.Replace(strings.ToLower(name), " ", "_", -1)) + fileName := fmt.Sprintf("build/TEST-go-%s", strings.ReplaceAll(strings.ToLower(name), " ", "_")) params := GoTestArgs{ LogName: name, Race: RaceDetector, @@ -63,8 +63,8 @@ func makeGoTestArgs(name string) GoTestArgs { func makeGoTestArgsForModule(name, module string) GoTestArgs { fileName := fmt.Sprintf("build/TEST-go-%s-%s", - strings.Replace(strings.ToLower(name), " ", "_", -1), - strings.Replace(strings.ToLower(module), " ", "_", -1), + strings.ReplaceAll(strings.ToLower(name), " ", "_"), + strings.ReplaceAll(strings.ToLower(module), " ", "_"), ) params := GoTestArgs{ LogName: fmt.Sprintf("%s-%s", name, module), diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 11659614f67..64b95d55dc6 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -13,13 +13,14 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" "github.com/magefile/mage/mg" "golang.org/x/sync/errgroup" - "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" "github.com/elastic/elastic-agent/pkg/version" ) @@ -94,72 +95,6 @@ var PlatformPackages = map[string]string{ "windows/amd64": "windows-x86_64.zip", } -// ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. -// The project names are those used in the "projects" list in the unified release manifest. -// See the sample manifests in the testdata directory. -var ExpectedBinaries = []BinarySpec{ - {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}}, - {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, -} - -type BinarySpec struct { - BinaryName string - ProjectName string - Platforms []Platform - PythonWheel bool - PackageTypes []pkgcommon.PackageType -} - -func (proj BinarySpec) SupportsPlatform(platform string) bool { - for _, p := range proj.Platforms { - if p.Platform() == platform { - return true - } - } - return false -} - -func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool { - for _, p := range proj.PackageTypes { - if p == pkgType { - return true - } - } - return false -} - -func (proj BinarySpec) GetPackageName(version string, platform string) string { - if proj.PythonWheel { - return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) - } - return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) -} - -type Platform struct { - OS string - Arch string -} - -// Converts to the format expected on the mage command line "linux", "x86_64" = "linux/amd64" -func (p Platform) Platform() string { - if p.Arch == "x86_64" { - p.Arch = "amd64" - } - if p.Arch == "aarch64" { - p.Arch = "arm64" - } - return p.OS + "/" + p.Arch -} - -var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}} - // DownloadManifest is going to download the given manifest file and return the ManifestResponse func DownloadManifest(ctx context.Context, manifest string) (Build, error) { manifestUrl, urlError := url.Parse(manifest) @@ -191,7 +126,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) { // DownloadComponents is going to download a set of components from the given manifest into the destination // dropPath folder in order to later use that folder for packaging -func DownloadComponents(ctx context.Context, manifest string, platforms []string, dropPath string) error { +func DownloadComponents(ctx context.Context, expectedBinaries []packaging.BinarySpec, manifest string, platforms []string, dropPath string) error { manifestResponse, err := DownloadManifest(ctx, manifest) if err != nil { return fmt.Errorf("failed to download remote manifest file %w", err) @@ -210,27 +145,27 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for _, spec := range ExpectedBinaries { + for _, spec := range expectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) if err != nil { return fmt.Errorf("failed to create directory %s", targetPath) } - log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform) + fmt.Printf("Prepare to download [%s] project [%s] for [%s]\n", spec.BinaryName, spec.ProjectName, platform) if !spec.SupportsPlatform(platform) { - log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform) + fmt.Printf("Binary [%s] does not support platform [%s]\n", spec.BinaryName, platform) continue } - pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) + resolvedPackage, err := ResolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } - for _, p := range pkgURL { - log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) + for _, p := range resolvedPackage.URLs { + fmt.Printf("Downloading [%s] [%s]\n", spec.BinaryName, p) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -247,80 +182,125 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return fmt.Errorf("error downloading files: %w", err) } - log.Printf("Downloads for manifest %q complete.", manifest) + fmt.Printf("Downloads for manifest %q complete.\n", manifest) return nil } -func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { - var val Package - var ok bool +type ResolvedPackage struct { + Name string + ActualVersion string + URLs []string +} + +func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { // Try the normal/easy case first - packageName := spec.GetPackageName(version, platform) - val, ok = project.Packages[packageName] - if !ok { - // If we didn't find it, it may be an Independent Agent Release, where - // the opted-in projects will have a patch version one higher than - // the rest of the projects, so we need to seek that out + packageName := spec.GetPackageName(dependencyVersion, platform) + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Got packagename [%s], looking for exact match", packageName) + } + + if exactMatch, ok := project.Packages[packageName]; ok { + // We found the exact filename we are looking for if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) + log.Printf(">>>>>>>>>>> Found exact match packageName for [%s, %s]: %s", project.Branch, project.CommitHash, exactMatch) } - var foundIt bool - for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, spec.BinaryName) { - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + return &ResolvedPackage{ + Name: packageName, + ActualVersion: dependencyVersion, + URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, + }, nil + } - secondHalf := firstSplit[1] - // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, PlatformPackages[platform]) { - - // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) - if len(secondSplit) < 2 { - continue - } - - // The first element after the split should normally be the version - pkgVersion := secondSplit[0] - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion) - } - - // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) - } - - // Get the package value, if it exists - val, ok = project.Packages[foundPkgKey] - if !ok { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey) - } - - foundIt = true - } - } - } + // If we didn't find it, it may be an Independent Agent Release, where + // the opted-in projects will have a patch version one higher than + // the rest of the projects, so we "relax" the version constraint + return resolveManifestPackageUsingRelaxedVersion(project, spec, dependencyVersion, platform) +} - if !foundIt { - return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL) - } +func resolveManifestPackageUsingRelaxedVersion(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { + // start with the rendered package name + packageName := spec.GetPackageName(dependencyVersion, platform) + + // Find the original version in the rendered filename + versionIndex := strings.Index(packageName, dependencyVersion) + if versionIndex == -1 { + return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion) + } + + // obtain a regexp from the exact version string that allows for some flexibility on patch version, prerelease and build metadata tokens + relaxedVersion, err := relaxVersion(dependencyVersion) + if err != nil { + return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err) } if mg.Verbose() { - log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash) + log.Printf(">>>>>>>>>>> Couldn't find exact match, relaxing agent dependencyVersion to %s", relaxedVersion) } - return []string{val.URL, val.ShaURL, val.AscURL}, nil + // locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that + relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex]) + relaxedPackageName += `(?P` + relaxedVersion + `)` + relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):]) + + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Attempting to match a filename with %s", relaxedPackageName) + } + + relaxedPackageNameRegexp, err := regexp.Compile(relaxedPackageName) + if err != nil { + return nil, fmt.Errorf("compiling relaxed package name regex %q: %w", relaxedPackageName, err) + } + + for pkgName, pkg := range project.Packages { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName) + } + if submatches := relaxedPackageNameRegexp.FindStringSubmatch(pkgName); len(submatches) > 0 { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName) + } + return &ResolvedPackage{ + Name: pkgName, + ActualVersion: submatches[1], + URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, + }, nil + } + } + + return nil, fmt.Errorf("package [%s] not found in project manifest at %s using relaxed version %q", packageName, project.ExternalArtifactsManifestURL, relaxedPackageName) +} + +// versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?) +const versionRegexp = `^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` +const anyPatchVersionRegexp = `(?:0|[1-9]\d*)` + +var versionRegExp = regexp.MustCompile(versionRegexp) + +func relaxVersion(version string) (string, error) { + matchIndices := versionRegExp.FindSubmatchIndex([]byte(version)) + // Matches index pairs are (0,1) for the whole regexp and (2,3) for the patch group + // check that we have matched correctly + if len(matchIndices) < 4 { + return "", fmt.Errorf("failed to match regexp for version [%s]", version) + } + + // take the starting index of the patch version + patchStartIndex := matchIndices[2] + // copy everything before the patch version escaping the regexp + relaxedVersion := regexp.QuoteMeta(version[:patchStartIndex]) + // add the patch regexp + relaxedVersion += anyPatchVersionRegexp + // check if there's more characters after the patch version + remainderIndex := matchIndices[3] + if remainderIndex < len(version) { + // This is a looser regexp that allows anything beyond the major version to change (while still enforcing a valid patch version though) + // see TestResolveManifestPackage/Independent_Agent_Staging_8.14_apm-server and TestResolveManifestPackage/Independent_Agent_Staging_8.14_endpoint-dev + // Be more relaxed and allow for any character sequence after this + relaxedVersion += `.*` + } + return relaxedVersion, nil } func DownloadPackage(ctx context.Context, downloadUrl string, target string) error { @@ -330,12 +310,12 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err } valid := false for _, manifestHost := range AllowedManifestHosts { - if manifestHost == parsedURL.Host { + if manifestHost == parsedURL.Hostname() { valid = true } } if !valid { - log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts) + log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Hostname(), AllowedManifestHosts) return errorNotAllowedManifestURL } cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path) diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index b975149aa27..76a0d53cd2e 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -7,12 +7,15 @@ package manifest import ( _ "embed" "encoding/json" + "fmt" "log" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) var ( @@ -135,7 +138,7 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - spec, ok := findBinarySpec(tc.binary) + spec, ok := findBinarySpec(t, tc.binary) assert.True(t, ok) if !spec.SupportsPlatform(tc.platform) { @@ -143,22 +146,80 @@ func TestResolveManifestPackage(t *testing.T) { return } - urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) + resolvedPackage, err := ResolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) + require.NotNil(t, resolvedPackage) - assert.Len(t, urlList, 3) - for _, url := range urlList { + assert.Len(t, resolvedPackage.URLs, 3) + for _, url := range resolvedPackage.URLs { assert.Contains(t, tc.expectedUrlList, url) } }) } } -func findBinarySpec(name string) (BinarySpec, bool) { - for _, spec := range ExpectedBinaries { +func findBinarySpec(t *testing.T, name string) (packaging.BinarySpec, bool) { + components, err := packaging.Components() + require.NoError(t, err, "error loading components from packages.yml") + + for _, spec := range components { if spec.BinaryName == name { return spec, true } } - return BinarySpec{}, false + return packaging.BinarySpec{}, false +} + +func TestRelaxVersion(t *testing.T) { + type args struct { + version string + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "major-minor-patch", + args: args{ + version: "1.2.3", + }, + want: `1\.2\.(?:0|[1-9]\d*)`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot", + args: args{ + version: "1.2.3-SNAPSHOT", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot-buildmeta", + args: args{ + version: "1.2.3-SNAPSHOT+build20250328112233", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "not a semver", + args: args{ + version: "foobar", + }, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relaxVersion(tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("relaxVersion(%v)", tt.args.version)) { + return + } + assert.Equalf(t, tt.want, got, "relaxVersion(%v)", tt.args.version) + }) + } } diff --git a/dev-tools/mage/packageversion.go b/dev-tools/mage/packageversion.go new file mode 100644 index 00000000000..81f23132ab9 --- /dev/null +++ b/dev-tools/mage/packageversion.go @@ -0,0 +1,113 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package mage + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +const PackageVersionFilename = ".package-version" + +type packageVersion struct { + Version string `json:"version"` + BuildID string `json:"build_id"` + ManifestURL string `json:"manifest_url"` + SummaryURL string `json:"summary_url"` + CoreVersion string `json:"core_version"` + StackBuildID string `json:"stack_build_id"` +} + +func initPackageVersion() error { + if os.Getenv("USE_PACKAGE_VERSION") != "true" { + return nil + } + + _, err := os.Stat(PackageVersionFilename) + if err != nil { + if os.IsNotExist(err) { + log.Printf("USE_PACKAGE_VERSION is set, but %q does not exist, not overriding\n", PackageVersionFilename) + return nil + } + return fmt.Errorf("failed to stat %q: %w", PackageVersionFilename, err) + } + + pv, err := readPackageVersion() + if err != nil { + // err is wrapped in readPackageVersion + return err + } + + PackagingFromManifest = true + ManifestURL = pv.ManifestURL + agentPackageVersion = pv.CoreVersion + Snapshot = true + + _ = os.Setenv("BEAT_VERSION", pv.CoreVersion) + _ = os.Setenv("AGENT_VERSION", pv.Version) + _ = os.Setenv("AGENT_STACK_VERSION", pv.Version) + _ = os.Setenv("SNAPSHOT", "true") + + dropPath := filepath.Join("build", "distributions", "elastic-agent-drop") + dropPath, err = filepath.Abs(dropPath) + if err != nil { + return fmt.Errorf("failed to obtain absolute path for default drop path: %w", err) + } + + _ = os.Setenv("AGENT_DROP_PATH", dropPath) + + return nil +} + +func UpdatePackageVersion(version string, buildID string, manifestURL string, summaryURL string) error { + packageVersion := packageVersion{ + Version: version, + BuildID: buildID, + ManifestURL: manifestURL, + SummaryURL: summaryURL, + CoreVersion: strings.ReplaceAll(version, "-SNAPSHOT", ""), + StackBuildID: fmt.Sprintf("%s-SNAPSHOT", buildID), + } + + if err := writePackageVersion(packageVersion); err != nil { + // err is wrapped in writePackageVersion + return err + } + return nil +} + +func writePackageVersion(pv packageVersion) error { + pvBytes, err := json.MarshalIndent(pv, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal package version: %w", err) + } + + err = os.WriteFile(PackageVersionFilename, pvBytes, 0644) + if err != nil { + return fmt.Errorf("failed to write package version: %w", err) + } + + return nil +} + +func readPackageVersion() (*packageVersion, error) { + f, err := os.Open(PackageVersionFilename) + if err != nil { + return nil, fmt.Errorf("failed to open %q for read: %w", PackageVersionFilename, err) + } + defer f.Close() + + decoder := json.NewDecoder(f) + pVersion := &packageVersion{} + err = decoder.Decode(pVersion) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML from file %q: %w", PackageVersionFilename, err) + } + return pVersion, nil +} diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 6ec09f2e598..3032886ebe3 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,6 +34,15 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms + if mg.Verbose() { + debugSelectedPackageSpecsWithPlatform := make([]string, 0, len(Packages)) + for _, p := range Packages { + debugSelectedPackageSpecsWithPlatform = append(debugSelectedPackageSpecsWithPlatform, fmt.Sprintf("spec %s on %s/%s", p.Spec.Name, p.OS, p.Arch)) + } + + log.Printf("Packaging for platforms %v, packages %v", platforms, debugSelectedPackageSpecsWithPlatform) + } + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { @@ -42,12 +51,12 @@ func Package() error { } for _, pkgType := range pkg.Types { - if !isPackageTypeSelected(pkgType) { + if !IsPackageTypeSelected(pkgType) { log.Printf("Skipping %s package type because it is not selected", pkgType) continue } - if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + if pkgType == Docker && !IsDockerVariantSelected(pkg.Spec.DockerVariant) { log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) continue } @@ -80,7 +89,6 @@ func Package() error { spec.OS = target.GOOS() spec.Arch = packageArch spec.Snapshot = Snapshot - spec.FIPS = FIPSBuild spec.evalContext = map[string]interface{}{ "GOOS": target.GOOS(), "GOARCH": target.GOARCH(), @@ -100,6 +108,10 @@ func Package() error { spec = spec.Evaluate() + if mg.Verbose() { + log.Printf("Adding task for packaging %s on %s/%s", spec.Name, target.GOOS(), target.Arch()) + } + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } @@ -112,9 +124,9 @@ func Package() error { return nil } -// isPackageTypeSelected returns true if SelectedPackageTypes is empty or if +// IsPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. -func isPackageTypeSelected(pkgType PackageType) bool { +func IsPackageTypeSelected(pkgType PackageType) bool { if len(SelectedPackageTypes) == 0 { return true } @@ -127,9 +139,9 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } -// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// IsDockerVariantSelected returns true if SelectedDockerVariants is empty or if // docVariant is present on SelectedDockerVariants. It returns false otherwise. -func isDockerVariantSelected(docVariant DockerVariant) bool { +func IsDockerVariantSelected(docVariant DockerVariant) bool { if len(SelectedDockerVariants) == 0 { return true } @@ -149,11 +161,11 @@ type packageBuilder struct { } func (b packageBuilder) Build() error { - fmt.Printf(">> package: Building %v type=%v for platform=%v fips=%v\n", b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS) + fmt.Printf(">> package: Building %v type=%v for platform=%v\n", b.Spec.Name, b.Type, b.Platform.Name) log.Printf("Package spec: %+v", b.Spec) if err := b.Type.Build(b.Spec); err != nil { - return fmt.Errorf("failed building %v type=%v for platform=%v fips=%v : %w", - b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS, err) + return fmt.Errorf("failed building %v type=%v for platform=%v : %w", + b.Spec.Name, b.Type, b.Platform.Name, err) } return nil } @@ -222,7 +234,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-v") } - args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go")) + args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/testing/package_test.go")) if params.HasModules { args = append(args, "--modules") @@ -248,10 +260,6 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-root-owner") } - if FIPSBuild { - args = append(args, "-fips") - } - args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 704ce2a79a4..5fbd011c221 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -29,6 +29,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ( @@ -40,13 +41,13 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" // defaultRootDir is the default name of the root directory contained inside of zip and // tar.gz packages. // NOTE: This uses .BeatName instead of .Name because we wanted the internal // directory to not include "-oss". - defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}{{if .FIPS}}-fips{{end}}" + defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" componentConfigMode os.FileMode = 0600 @@ -93,7 +94,6 @@ type PackageSpec struct { Arch string `yaml:"arch,omitempty"` Vendor string `yaml:"vendor,omitempty"` Snapshot bool `yaml:"snapshot"` - FIPS bool `yaml:"fips"` Version string `yaml:"version,omitempty"` License string `yaml:"license,omitempty"` URL string `yaml:"url,omitempty"` @@ -107,6 +107,7 @@ type PackageSpec struct { OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional ExtraTags []string `yaml:"extra_tags,omitempty"` // Optional + Components []packaging.BinarySpec `yaml:"components"` // Optional: Components required for this package evalContext map[string]interface{} packageDir string @@ -760,7 +761,7 @@ func runFPM(spec PackageSpec, packageType PackageType) error { } defer os.Remove(inputTar) - outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}{{if .FIPS}}-fips{{end}}") + outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}") if err != nil { return err } diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index ab09281e015..ef333098ee5 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -88,7 +88,6 @@ var ( Snapshot bool DevBuild bool ExternalBuild bool - FIPSBuild bool versionQualified bool versionQualifier string @@ -155,17 +154,19 @@ func initGlobals() { panic(fmt.Errorf("failed to parse EXTERNAL env value: %w", err)) } - FIPSBuild, err = strconv.ParseBool(EnvOr("FIPS", "false")) - if err != nil { - panic(fmt.Errorf("failed to parse FIPS env value: %w", err)) - } - versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER") agentPackageVersion = EnvOr(agentPackageVersionEnvVar, "") ManifestURL = EnvOr(ManifestUrlEnvVar, "") PackagingFromManifest = ManifestURL != "" + + // order matters this must be called last as it will override some of the + // values above + err = initPackageVersion() + if err != nil { + panic(fmt.Errorf("failed to init package version: %w", err)) + } } // ProjectType specifies the type of project (OSS vs X-Pack). @@ -217,7 +218,6 @@ func varMap(args ...map[string]interface{}) map[string]interface{} { "Snapshot": Snapshot, "DEV": DevBuild, "EXTERNAL": ExternalBuild, - "FIPS": FIPSBuild, "Qualifier": versionQualifier, "CI": CI, } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 1bbf2316e4d..376e59cdf26 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1,9 +1,145 @@ --- -# This file contains the package specifications for both Community Beats and -# Official Beats. The shared section contains YAML anchors that are used to +# This file contains the package specifications for Elastic Agent + +# List all the available platforms +platforms: &all-platforms + - &linux-amd64 + os: linux + arch: x86_64 + - &linux-arm64 + os: linux + arch: arm64 + - &windows-amd64 + os: windows + arch: x86_64 + - &darwin-amd64 + os: darwin + arch: x86_64 + - &darwin-arm64 + os: darwin + arch: aarch64 + +# List all the package type constants (see dev-tools/mage/pkgcommon/pkgcommon-types.go) +packageTypes: &all-package-types + - &pkg-type-rpm + 1 # RPM + - &pkg-type-deb + 2 # Deb + - &pkg-type-zip + 3 # zip + - &pkg-type-targz + 4 # tar.gz + - &pkg-type-docker + 5 # docker + +# Settings section contains general compiling and packaging settings +#settings: + +# List *all* the components available for packaging in elastic-agent +components: + # general template for new components (all attributes are mandatory) + # - &comp- + # projectName: + # packageName: