From 5521c6153d46c6ba425fd25b968faf2bb8c3ff85 Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Thu, 7 Aug 2025 11:43:24 +0300 Subject: [PATCH 1/5] ci: build agent from snapshot DRA (#9048) * feat: rework .package-version and mage integration:UpdatePackageVersion to make CI build always from snapshot DRA * feat: incorporate USE_PACKAGE_VERSION in mage * experiment: bump version.go * Revert "experiment: bump version.go" This reverts commit a57ee105a188848cdfaf85f6c4066942b8d32857. * chore: bump .package-version * feat: allow AGENT_VERSION to be overridden by env var * fix: use named args for all args in integration_tests_tf.ps1 * feat: panic on err of initPackageVersion * fix: don't panic when .package-version file doesn't exist, log it instead * feat: rework fabrication of CI_ELASTIC_AGENT_DOCKER_IMAGE * feat: use os.WriteFile in writePackageVersion * chore: bump to latest snapshot DRA * fix: always DownloadManifest if PackagingFromManifest is set in mage package * fix: check err of filepath.Abs(dropPath) (cherry picked from commit a155660c439428cacb6763d2b1b29a6ac8c2de3e) # Conflicts: # .buildkite/integration.pipeline.yml # .buildkite/scripts/steps/ess.ps1 # .package-version # dev-tools/mage/manifest/manifest.go --- .buildkite/integration.pipeline.yml | 44 +++++++ .../scripts/buildkite-integration-tests.ps1 | 25 +++- .../scripts/buildkite-integration-tests.sh | 18 ++- .../buildkite-k8s-integration-tests.sh | 15 ++- .buildkite/scripts/steps/ess.ps1 | 9 +- .buildkite/scripts/steps/ess.sh | 4 +- .buildkite/scripts/steps/ess_start.sh | 6 +- .../steps/integration-cloud-image-push.sh | 21 ++++ .../scripts/steps/integration-package.sh | 1 + .buildkite/scripts/steps/integration_tests.sh | 12 +- .../scripts/steps/integration_tests_tf.ps1 | 15 ++- .../scripts/steps/integration_tests_tf.sh | 10 +- .github/workflows/bump-agent-versions.sh | 2 +- .package-version | 13 +- dev-tools/mage/manifest/manifest.go | 11 +- dev-tools/mage/packageversion.go | 113 ++++++++++++++++++ dev-tools/mage/settings.go | 7 ++ magefile.go | 30 +++-- test_infra/ess/deployment.tf | 14 ++- .../ess/upgrade_broken_package_test.go | 1 + 20 files changed, 304 insertions(+), 67 deletions(-) create mode 100755 .buildkite/scripts/steps/integration-cloud-image-push.sh create mode 100644 dev-tools/mage/packageversion.go diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 1a093202421..ae10873b7d4 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -72,7 +72,13 @@ steps: env: PACKAGES: "docker" PLATFORMS: "linux/amd64" +<<<<<<< HEAD command: ".buildkite/scripts/steps/integration-package.sh" +======= + command: | + .buildkite/scripts/steps/integration-package.sh + .buildkite/scripts/steps/integration-cloud-image-push.sh +>>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) artifact_paths: - build/distributions/** agents: @@ -96,6 +102,44 @@ steps: diskSizeGb: 200 image: "${IMAGE_UBUNTU_2204_ARM_64}" +<<<<<<< HEAD +======= + - label: "Packaging: Containers linux/amd64 FIPS" + key: packaging-containers-x86-64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/amd64" + FIPS: "true" + command: | + .buildkite/scripts/steps/integration-package.sh + .buildkite/scripts/steps/integration-cloud-image-push.sh + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + diskSizeGb: 200 + image: "${IMAGE_UBUNTU_2204_X86_64}" + plugins: + - *vault_docker_login + + - label: "Packaging: Containers linux/arm64 FIPS" + key: packaging-containers-arm64-fips + env: + PACKAGES: "docker" + PLATFORMS: "linux/arm64" + FIPS: "true" + command: | + .buildkite/scripts/steps/integration-package.sh + artifact_paths: + - build/distributions/** + agents: + provider: "aws" + instanceType: "c6g.4xlarge" + diskSizeGb: 200 + image: "${IMAGE_UBUNTU_2204_ARM_64}" + +>>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) - label: "Triggering Integration tests" depends_on: - int-packaging diff --git a/.buildkite/scripts/buildkite-integration-tests.ps1 b/.buildkite/scripts/buildkite-integration-tests.ps1 index 86b7d39c41a..45f526125eb 100644 --- a/.buildkite/scripts/buildkite-integration-tests.ps1 +++ b/.buildkite/scripts/buildkite-integration-tests.ps1 @@ -25,13 +25,28 @@ go install gotest.tools/gotestsum gotestsum --version $env:TEST_BINARY_NAME = "elastic-agent" -if (-not $env:AGENT_VERSION) { - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) - $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + +if (-not $env:AGENT_VERSION) +{ + if (Test-Path .package-version) + { + $packageContent = Get-Content .package-version -Raw | ConvertFrom-Json + $env:AGENT_VERSION = $packageContent.version + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from .package-version)" + } + else + { + # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 + $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' }) + $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT" + Write-Output "~~~ Agent version: $env:AGENT_VERSION (from version/version.go)" + } +} +else +{ + Write-Output "~~~ Agent version: $env:AGENT_VERSION (specified by env var)" } -Write-Output "~~~ Agent version: $env:AGENT_VERSION" $env:SNAPSHOT = $true Write-Host "~~~ Running integration tests as $env:USERNAME" diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh index b8e56603e2c..c1f926cc03e 100755 --- a/.buildkite/scripts/buildkite-integration-tests.sh +++ b/.buildkite/scripts/buildkite-integration-tests.sh @@ -33,14 +33,20 @@ echo "~~~ Running integration tests as $USER" make install-gotestsum if [[ -z "${AGENT_VERSION:-}" ]]; then - # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925 - AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2) + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi -export AGENT_VERSION -echo "~~~ Agent version: ${AGENT_VERSION}" - os_data=$(uname -spr | tr ' ' '_') root_suffix="" if [ "$TEST_SUDO" == "true" ]; then diff --git a/.buildkite/scripts/buildkite-k8s-integration-tests.sh b/.buildkite/scripts/buildkite-k8s-integration-tests.sh index 93054278d14..a1df92e7c6d 100755 --- a/.buildkite/scripts/buildkite-k8s-integration-tests.sh +++ b/.buildkite/scripts/buildkite-k8s-integration-tests.sh @@ -9,9 +9,18 @@ DOCKER_VARIANTS="${DOCKER_VARIANTS:-basic,wolfi,complete,complete-wolfi,service, CLUSTER_NAME="${K8S_VERSION}-kubernetes" if [[ -z "${AGENT_VERSION:-}" ]]; then - # If not specified, use the version in version/version.go - AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" - AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + if [[ -f "${WORKSPACE}/.package-version" ]]; then + AGENT_VERSION="$(jq -r '.version' .package-version)" + echo "~~~ Agent version: ${AGENT_VERSION} (from .package-version)" + else + AGENT_VERSION="$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)" + AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT" + echo "~~~ Agent version: ${AGENT_VERSION} (from version/version.go)" + fi + + export AGENT_VERSION +else + echo "~~~ Agent version: ${AGENT_VERSION} (specified by env var)" fi echo "~~~ Create kind cluster '${CLUSTER_NAME}'" diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1 index e0e30245d04..b1c1d46810b 100644 --- a/.buildkite/scripts/steps/ess.ps1 +++ b/.buildkite/scripts/steps/ess.ps1 @@ -1,6 +1,7 @@ function ess_up { param ( [string]$StackVersion, + [string]$StackBuildId = "", [string]$EssRegion = "gcp-us-west2" ) @@ -22,6 +23,7 @@ function ess_up { & terraform init & terraform apply -auto-approve ` -var="stack_version=$StackVersion" ` + -var="stack_build_id=$StackBuildId" ` -var="ess_region=$EssRegion" ` -var="creator=$BuildkiteBuildCreator" ` -var="buildkite_id=$BuildkiteBuildNumber" ` @@ -98,12 +100,13 @@ function Retry-Command { function Get-Ess-Stack { param ( - [string]$StackVersion + [string]$StackVersion, + [string]$StackBuildId = "" ) if ($Env:BUILDKITE_RETRY_COUNT -gt 0) { - Write-Output "The step is retried, starting the ESS stack again" - ess_up $StackVersion + Write-Output "The step is retried, starting the ESS stack again" + ess_up $StackVersion $StackBuildId Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST" } else { # For the first run, we retrieve ESS stack metadata diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh index 093e0bc309c..13b0e9a9ced 100755 --- a/.buildkite/scripts/steps/ess.sh +++ b/.buildkite/scripts/steps/ess.sh @@ -6,7 +6,8 @@ function ess_up() { local WORKSPACE=$(git rev-parse --show-toplevel) local TF_DIR="${WORKSPACE}/test_infra/ess/" local STACK_VERSION=$1 - local ESS_REGION=${2:-"gcp-us-west2"} + local STACK_BUILD_ID=${2:-""} + local ESS_REGION=${3:-"gcp-us-west2"} if [ -z "$STACK_VERSION" ]; then echo "Error: Specify stack version: ess_up [stack_version]" >&2 @@ -22,6 +23,7 @@ function ess_up() { terraform apply \ -auto-approve \ -var="stack_version=${STACK_VERSION}" \ + -var="stack_build_id=${STACK_BUILD_ID}" \ -var="ess_region=${ESS_REGION}" \ -var="creator=${BUILDKITE_BUILD_CREATOR}" \ -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \ diff --git a/.buildkite/scripts/steps/ess_start.sh b/.buildkite/scripts/steps/ess_start.sh index 2062603547f..20372379335 100755 --- a/.buildkite/scripts/steps/ess_start.sh +++ b/.buildkite/scripts/steps/ess_start.sh @@ -4,10 +4,10 @@ set -euo pipefail source .buildkite/scripts/steps/ess.sh source .buildkite/scripts/steps/fleet.sh -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" -ess_up $OVERRIDE_STACK_VERSION +ess_up "$STACK_VERSION" "$STACK_BUILD_ID" preinstall_fleet_packages diff --git a/.buildkite/scripts/steps/integration-cloud-image-push.sh b/.buildkite/scripts/steps/integration-cloud-image-push.sh new file mode 100755 index 00000000000..0292c80f6d8 --- /dev/null +++ b/.buildkite/scripts/steps/integration-cloud-image-push.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +source .buildkite/scripts/common.sh + +echo "~~~ Pushing cloud image" + +suffix="" +if [ "${FIPS:-false}" == "true" ]; then + suffix="-fips" +fi + +CI_ELASTIC_AGENT_DOCKER_IMAGE="docker.elastic.co/beats-ci/elastic-agent-cloud${suffix}" +export CI_ELASTIC_AGENT_DOCKER_IMAGE +echo "CI_ELASTIC_AGENT_DOCKER_IMAGE: ${CI_ELASTIC_AGENT_DOCKER_IMAGE}" + + +export CUSTOM_IMAGE_TAG="git-${BUILDKITE_COMMIT:0:12}" +export USE_PACKAGE_VERSION="true" + +mage cloud:push diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index cd58c2777a8..b3a1594d6ee 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -5,5 +5,6 @@ source .buildkite/scripts/common.sh export SNAPSHOT="true" export EXTERNAL="true" +export USE_PACKAGE_VERSION="true" mage package diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 834da1cd4c6..f3fe7af9a24 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -7,19 +7,9 @@ STACK_PROVISIONER="${1:-"stateful"}" MAGE_TARGET="${2:-"integration:test"}" MAGE_SUBTARGET="${3:-""}" - -# Override the stack version from `.package-version` contents -# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically -# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. - -STACK_VERSION="$(cat .package-version)" -if [[ -n "$STACK_VERSION" ]]; then - STACK_VERSION=${STACK_VERSION}"-SNAPSHOT" -fi - # Run integration tests set +e -AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET +USE_PACKAGE_VERSION=true TEST_INTEG_CLEAN_ON_EXIT=true STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/integration_tests_tf.ps1 b/.buildkite/scripts/steps/integration_tests_tf.ps1 index dda818408be..109c0115e36 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.ps1 +++ b/.buildkite/scripts/steps/integration_tests_tf.ps1 @@ -9,10 +9,15 @@ $PSVersionTable.PSVersion . "$PWD\.buildkite\scripts\steps\ess.ps1" -# Read package version from .package-version file -$PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue -if ($PACKAGE_VERSION) { - $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT" +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically +# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready +$packageVersionContent = Get-Content .package-version -Raw -ErrorAction SilentlyContinue | ConvertFrom-Json +if ($packageVersionContent -and $packageVersionContent.version ) { + $STACK_VERSION = $packageVersionContent.version +} +if ($packageVersionContent -and $packageVersionContent.stack_build_id ) { + $STACK_BUILD_ID = $packageVersionContent.stack_build_id } Write-Output "~~~ Building test binaries" @@ -27,7 +32,7 @@ $TestsExitCode = 0 try { Write-Output "~~~ Running integration tests" # Get-Ess-Stack will start the ESS stack if it is a BK retry, otherwise it will retrieve ESS stack metadata - Get-Ess-Stack -StackVersion $PACKAGE_VERSION + Get-Ess-Stack -StackVersion $STACK_VERSION -StackBuildId $STACK_BUILD_ID & "$PWD\.buildkite\scripts\buildkite-integration-tests.ps1" $GROUP_NAME $TEST_SUDO $TestsExitCode = $LASTEXITCODE if ($TestsExitCode -ne 0) diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index 8b60ceae4bd..4ea93e4a00e 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -20,11 +20,11 @@ if [ -z "$TEST_SUDO" ]; then exit 1 fi -# Override the agent package version using a string with format .. -# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically +# Override the stack version from `.package-version` contents +# There is a time when the current snapshot is not available on cloud yet, so we cannot use the latest version automatically # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. -OVERRIDE_STACK_VERSION="$(cat .package-version)" -OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT" +STACK_VERSION="$(jq -r '.version' .package-version)" +STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)" echo "~~~ Building test binaries" mage build:testBinaries @@ -35,7 +35,7 @@ mage build:testBinaries if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then echo "~~~ The steps is retried, starting the ESS stack again" trap 'ess_down' EXIT - ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack") + ess_up "$STACK_VERSION" "$STACK_BUILD_ID" || (echo -e "^^^ +++\nFailed to start ESS stack") else # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data echo "~~~ Receiving ESS stack metadata" diff --git a/.github/workflows/bump-agent-versions.sh b/.github/workflows/bump-agent-versions.sh index ea3941a0ea4..8656535f77e 100755 --- a/.github/workflows/bump-agent-versions.sh +++ b/.github/workflows/bump-agent-versions.sh @@ -25,7 +25,7 @@ else git add testing/integration/testdata/.upgrade-test-agent-versions.yml .package-version nl=$'\n' # otherwise the new line character is not recognized properly - commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" + commit_desc="These files are used for picking the starting (pre-upgrade) or ending (post-upgrade) agent versions in upgrade integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: \`\`\`json${nl}${package_version}${nl}\`\`\`${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" git commit -m "[$current_ref][Automation] Update versions" -m "$commit_desc" git push --set-upstream origin "$pr_branch" diff --git a/.package-version b/.package-version index ff79a643409..089f41d19c4 100644 --- a/.package-version +++ b/.package-version @@ -1 +1,12 @@ -8.17.10 \ No newline at end of file +<<<<<<< HEAD +8.17.10 +======= +{ + "version": "9.2.0-SNAPSHOT", + "build_id": "9.2.0-ffb19b67", + "manifest_url": "https://snapshots.elastic.co/9.2.0-ffb19b67/manifest-9.2.0-SNAPSHOT.json", + "summary_url": "https://snapshots.elastic.co/9.2.0-ffb19b67/summary-9.2.0-SNAPSHOT.html", + "core_version": "9.2.0", + "stack_build_id": "9.2.0-ffb19b67-SNAPSHOT" +} +>>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 1db9c21dbe1..8618c408eb5 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -218,10 +218,10 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string if err != nil { return fmt.Errorf("failed to create directory %s", targetPath) } - log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform) + fmt.Printf("Prepare to download [%s] project [%s] for [%s]\n", spec.BinaryName, spec.ProjectName, platform) if !spec.SupportsPlatform(platform) { - log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform) + fmt.Printf("Binary [%s] does not support platform [%s]\n", spec.BinaryName, platform) continue } @@ -230,8 +230,13 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return err } +<<<<<<< HEAD for _, p := range pkgURL { log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) +======= + for _, p := range resolvedPackage.URLs { + fmt.Printf("Downloading [%s] [%s]\n", spec.BinaryName, p) +>>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -248,7 +253,7 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return fmt.Errorf("error downloading files: %w", err) } - log.Printf("Downloads for manifest %q complete.", manifest) + fmt.Printf("Downloads for manifest %q complete.\n", manifest) return nil } diff --git a/dev-tools/mage/packageversion.go b/dev-tools/mage/packageversion.go new file mode 100644 index 00000000000..81f23132ab9 --- /dev/null +++ b/dev-tools/mage/packageversion.go @@ -0,0 +1,113 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package mage + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +const PackageVersionFilename = ".package-version" + +type packageVersion struct { + Version string `json:"version"` + BuildID string `json:"build_id"` + ManifestURL string `json:"manifest_url"` + SummaryURL string `json:"summary_url"` + CoreVersion string `json:"core_version"` + StackBuildID string `json:"stack_build_id"` +} + +func initPackageVersion() error { + if os.Getenv("USE_PACKAGE_VERSION") != "true" { + return nil + } + + _, err := os.Stat(PackageVersionFilename) + if err != nil { + if os.IsNotExist(err) { + log.Printf("USE_PACKAGE_VERSION is set, but %q does not exist, not overriding\n", PackageVersionFilename) + return nil + } + return fmt.Errorf("failed to stat %q: %w", PackageVersionFilename, err) + } + + pv, err := readPackageVersion() + if err != nil { + // err is wrapped in readPackageVersion + return err + } + + PackagingFromManifest = true + ManifestURL = pv.ManifestURL + agentPackageVersion = pv.CoreVersion + Snapshot = true + + _ = os.Setenv("BEAT_VERSION", pv.CoreVersion) + _ = os.Setenv("AGENT_VERSION", pv.Version) + _ = os.Setenv("AGENT_STACK_VERSION", pv.Version) + _ = os.Setenv("SNAPSHOT", "true") + + dropPath := filepath.Join("build", "distributions", "elastic-agent-drop") + dropPath, err = filepath.Abs(dropPath) + if err != nil { + return fmt.Errorf("failed to obtain absolute path for default drop path: %w", err) + } + + _ = os.Setenv("AGENT_DROP_PATH", dropPath) + + return nil +} + +func UpdatePackageVersion(version string, buildID string, manifestURL string, summaryURL string) error { + packageVersion := packageVersion{ + Version: version, + BuildID: buildID, + ManifestURL: manifestURL, + SummaryURL: summaryURL, + CoreVersion: strings.ReplaceAll(version, "-SNAPSHOT", ""), + StackBuildID: fmt.Sprintf("%s-SNAPSHOT", buildID), + } + + if err := writePackageVersion(packageVersion); err != nil { + // err is wrapped in writePackageVersion + return err + } + return nil +} + +func writePackageVersion(pv packageVersion) error { + pvBytes, err := json.MarshalIndent(pv, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal package version: %w", err) + } + + err = os.WriteFile(PackageVersionFilename, pvBytes, 0644) + if err != nil { + return fmt.Errorf("failed to write package version: %w", err) + } + + return nil +} + +func readPackageVersion() (*packageVersion, error) { + f, err := os.Open(PackageVersionFilename) + if err != nil { + return nil, fmt.Errorf("failed to open %q for read: %w", PackageVersionFilename, err) + } + defer f.Close() + + decoder := json.NewDecoder(f) + pVersion := &packageVersion{} + err = decoder.Decode(pVersion) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML from file %q: %w", PackageVersionFilename, err) + } + return pVersion, nil +} diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 32bc62a3e97..cec6c7b0376 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -159,6 +159,13 @@ func initGlobals() { ManifestURL = EnvOr(ManifestUrlEnvVar, "") PackagingFromManifest = ManifestURL != "" + + // order matters this must be called last as it will override some of the + // values above + err = initPackageVersion() + if err != nil { + panic(fmt.Errorf("failed to init package version: %w", err)) + } } // ProjectType specifies the type of project (OSS vs X-Pack). diff --git a/magefile.go b/magefile.go index 1518196acb0..92006fd0a0c 100644 --- a/magefile.go +++ b/magefile.go @@ -510,6 +510,9 @@ func Package(ctx context.Context) error { if err != nil { return fmt.Errorf("failed downloading manifest: %w", err) } + // we need that dependency to essentially download + // the components from the given manifest + mg.Deps(DownloadManifest) } var dependenciesVersion string @@ -2148,33 +2151,28 @@ func (Integration) UpdateVersions(ctx context.Context) error { // UpdatePackageVersion update the file that contains the latest available snapshot version func (Integration) UpdatePackageVersion(ctx context.Context) error { - const packageVersionFilename = ".package-version" - currentReleaseBranch, err := git.GetCurrentReleaseBranch(ctx) if err != nil { return fmt.Errorf("failed to identify the current release branch: %w", err) } - sc := snapshots.NewSnapshotsClient() - versions, err := sc.FindLatestSnapshots(ctx, []string{currentReleaseBranch}) + branchInformation, err := findLatestBuildForBranch(ctx, baseURLForSnapshotDRA, currentReleaseBranch) if err != nil { - return fmt.Errorf("failed to fetch a manifest for the latest snapshot: %w", err) + return fmt.Errorf("failed to get latest build for branch %q: %w", currentReleaseBranch, err) } - if len(versions) != 1 { - return fmt.Errorf("expected a single version, got %v", versions) - } - packageVersion := versions[0].CoreVersion() - file, err := os.OpenFile(packageVersionFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + + err = devtools.UpdatePackageVersion( + branchInformation.Version, branchInformation.BuildID, + branchInformation.ManifestURL, branchInformation.SummaryURL) if err != nil { - return fmt.Errorf("failed to open %s for write: %w", packageVersionFilename, err) + return fmt.Errorf("failed to write package version: %w", err) } - defer file.Close() - _, err = file.WriteString(packageVersion) + + packageVersionBytes, err := os.ReadFile(devtools.PackageVersionFilename) if err != nil { - return fmt.Errorf("failed to write the package version file %s: %w", packageVersionFilename, err) + return fmt.Errorf("failed to read file: %w", err) } - - fmt.Println(packageVersion) + fmt.Println(string(packageVersionBytes)) return nil } diff --git a/test_infra/ess/deployment.tf b/test_infra/ess/deployment.tf index c3945275e2e..2ba976240a3 100644 --- a/test_infra/ess/deployment.tf +++ b/test_infra/ess/deployment.tf @@ -4,6 +4,12 @@ variable "stack_version" { description = "the stack version to use" } +variable "stack_build_id" { + type = string + default = "" + description = "The build id associated with the component images of the stack" +} + variable "ess_region" { type = string default = "" @@ -73,10 +79,10 @@ locals { }, yamldecode(file("${path.module}/../../pkg/testing/ess/create_deployment_csp_configuration.yaml"))) - - integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${var.stack_version}") - elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${var.stack_version}") - kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${var.stack_version}") + images_version = coalesce(var.stack_build_id, var.stack_version) + integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${local.images_version}") + elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${local.images_version}") + kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${local.images_version}") } # If we have defined a stack version, validate that this version exists on that region and return it. diff --git a/testing/integration/ess/upgrade_broken_package_test.go b/testing/integration/ess/upgrade_broken_package_test.go index c2b348ea87a..9369a96ae1e 100644 --- a/testing/integration/ess/upgrade_broken_package_test.go +++ b/testing/integration/ess/upgrade_broken_package_test.go @@ -32,6 +32,7 @@ func TestUpgradeBrokenPackageVersion(t *testing.T) { Local: false, // requires Agent installation Sudo: true, // requires Agent installation }) + t.Skip("Skip this test because it is not compatible with .package-version pinning") ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel() From 7947bd26fffaa26c993f7e3612cf2462304b86bd Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:07:22 +0200 Subject: [PATCH 2/5] [8.x](backport #7690) Fips packaging (#7790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make components in packages configurable (#7602) * Redefine ExpectedBinaries as YAML config * Move ExpectedBinaries closer to package spec file * Fix error formatting in downloadDRAArtifacts * add packageName template to ExpectedPackages * use a relaxed dependencies version for IAR releases * Remove FIPS hack introduced in PR #7486 * Allow for a looser match on relaxing dependencies versions * Add debug logging when packaging with EXTERNAL=true * move package tests to dedicated package * Fips packaging (#7690) * Add component list to specs * extract component dependencies from the packages to be built * Refactor component extraction from package specs * Fix package tests error handling * Inject dependencies and remove references to ExpectedBinaries * Remove ExpectedBinaries global * Add rootdir to components * Extract actual version matched on the package file and use it to render RootDir * Package elastic-agent FIPS specs when FIPS=true is specified * refactor ResolveManifestPackage * Move FIPS compile settings in packages.yml * Add more FIPS components * Properly handle dependenciesVersion when calling mage package * Refactor ChecksumsWithoutManifest to use list of dependencies instead of globbing files * Rework useDRAAgentBinaryForPackage for repackaging agent Define elastic-agent-core components (both FIPS and non-FIPS variants) and define package name and root dir templates. Implement some filtering on component list to extract the correct component definition according to the FIPSBuild flag. Refactor code that downloads pre-compiled elastic-agent binaries and places them in the golangCrossBuild directory to make use of the new component definition. * Write spec FIPS flag into manifest.yaml when packaging * Add FIPS elastic agent basic and cloud docker images * Build FIPS docker images in CI packaging * Fix FIPS .tar.gz package tests * Restructure package tests * Extend FIPS check to all binaries in components directory * Create FIPS elastic-agent-core artifacts in elastic-agent-binary-dra pipeline * Cleanup ChecksumsWithManifest and improve godoc * Improve godoc for BinarySpec * Correctly inject dependency list when packaging using DROP_PATH (#7795) * Restore qualifier=core for elastic-agent-core packaging specs (#7805) Restore qualifier for elastic-agent-core packaging specs to avoid changing the rootDir name of the archives. The qualifier had been removed in PR #7690 trying to use the spec name: this worked to get the desired file name but changed the root Dir name which uses '{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}' in the template definition instead of '{{.Name}}' which would render the spec name. * Modify fips core spec qualifier and name (#7818) * Reintroduce cloud-defend component * Filter components by package-type --------- Co-authored-by: Paolo Chilà --- .buildkite/integration.pipeline.yml | 43 -- dev-tools/mage/build.go | 38 +- dev-tools/mage/checksums.go | 296 +++++------- dev-tools/mage/crossbuild.go | 4 +- dev-tools/mage/dockerbuilder.go | 1 + dev-tools/mage/manifest/manifest.go | 248 +++++----- dev-tools/mage/manifest/manifest_test.go | 75 ++- dev-tools/mage/pkg.go | 27 +- dev-tools/mage/pkgtypes.go | 2 + dev-tools/packaging/packages.yml | 179 ++++++- dev-tools/packaging/settings.go | 274 +++++++++++ .../packaging/{ => testing}/package_test.go | 180 ++++--- magefile.go | 442 ++++++++++++------ pkg/version/version_parser.go | 2 +- .../upgrade_standalone_same_commit_test.go | 2 +- 15 files changed, 1182 insertions(+), 631 deletions(-) create mode 100644 dev-tools/packaging/settings.go rename dev-tools/packaging/{ => testing}/package_test.go (87%) diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index ae10873b7d4..f07e90b2b89 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -72,13 +72,8 @@ steps: env: PACKAGES: "docker" PLATFORMS: "linux/amd64" -<<<<<<< HEAD - command: ".buildkite/scripts/steps/integration-package.sh" -======= command: | .buildkite/scripts/steps/integration-package.sh - .buildkite/scripts/steps/integration-cloud-image-push.sh ->>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) artifact_paths: - build/distributions/** agents: @@ -102,44 +97,6 @@ steps: diskSizeGb: 200 image: "${IMAGE_UBUNTU_2204_ARM_64}" -<<<<<<< HEAD -======= - - label: "Packaging: Containers linux/amd64 FIPS" - key: packaging-containers-x86-64-fips - env: - PACKAGES: "docker" - PLATFORMS: "linux/amd64" - FIPS: "true" - command: | - .buildkite/scripts/steps/integration-package.sh - .buildkite/scripts/steps/integration-cloud-image-push.sh - artifact_paths: - - build/distributions/** - agents: - provider: "gcp" - machineType: "n2-standard-8" - diskSizeGb: 200 - image: "${IMAGE_UBUNTU_2204_X86_64}" - plugins: - - *vault_docker_login - - - label: "Packaging: Containers linux/arm64 FIPS" - key: packaging-containers-arm64-fips - env: - PACKAGES: "docker" - PLATFORMS: "linux/arm64" - FIPS: "true" - command: | - .buildkite/scripts/steps/integration-package.sh - artifact_paths: - - build/distributions/** - agents: - provider: "aws" - instanceType: "c6g.4xlarge" - diskSizeGb: 200 - image: "${IMAGE_UBUNTU_2204_ARM_64}" - ->>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) - label: "Triggering Integration tests" depends_on: - int-packaging diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 190efa543ea..90b8d71a694 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -11,6 +11,7 @@ import ( "log" "os" "path/filepath" + "regexp" "strings" "github.com/josephspurrier/goversioninfo" @@ -34,11 +35,45 @@ type BuildArgs struct { WinMetadata bool // Add resource metadata to Windows binaries (like add the version number to the .exe properties). } +// buildTagRE is a regexp to match strings like "-tags=abcd" +// but does not match "-tags= " +var buildTagRE = regexp.MustCompile(`-tags=([\S]+)?`) + +// ParseBuildTags returns the ExtraFlags param where all flags that are go build tags are joined by a comma. +// +// For example if given -someflag=val1 -tags=buildtag1 -tags=buildtag2 +// It will return -someflag=val1 -tags=buildtag1,buildtag2 +func (b BuildArgs) ParseBuildTags() []string { + flags := make([]string, 0) + if len(b.ExtraFlags) == 0 { + return flags + } + + buildTags := make([]string, 0) + for _, flag := range b.ExtraFlags { + if buildTagRE.MatchString(flag) { + arr := buildTagRE.FindStringSubmatch(flag) + if len(arr) != 2 || arr[1] == "" { + log.Printf("Unexpected format found for buildargs.ExtraFlags, ignoring value %q", flag) + continue + } + buildTags = append(buildTags, arr[1]) + } else { + flags = append(flags, flag) + } + } + if len(buildTags) > 0 { + flags = append(flags, "-tags="+strings.Join(buildTags, ",")) + } + return flags +} + // DefaultBuildArgs returns the default BuildArgs for use in builds. func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + Env: map[string]string{}, Vars: map[string]string{ elasticAgentModulePath + "/version.buildTime": "{{ date }}", elasticAgentModulePath + "/version.commit": "{{ commit }}", @@ -151,6 +186,7 @@ func Build(params BuildArgs) error { if params.CGO { cgoEnabled = "1" } + env["CGO_ENABLED"] = cgoEnabled // Spec @@ -159,7 +195,7 @@ func Build(params BuildArgs) error { "-o", filepath.Join(params.OutputDir, binaryName), } - args = append(args, params.ExtraFlags...) + args = append(args, params.ParseBuildTags()...) // ldflags ldflags := params.LDFlags diff --git a/dev-tools/mage/checksums.go b/dev-tools/mage/checksums.go index 100d138195c..d885d70f7ca 100644 --- a/dev-tools/mage/checksums.go +++ b/dev-tools/mage/checksums.go @@ -9,12 +9,12 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/magefile/mage/mg" "github.com/otiai10/copy" "github.com/elastic/elastic-agent/dev-tools/mage/manifest" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ComponentSpecFileSuffix = ".spec.yml" @@ -40,23 +40,53 @@ func CopyComponentSpecs(componentName, versionedDropPath string) (string, error) return GetSHA512Hash(targetPath) } -// This is a helper function for flattenDependencies that's used when not packaging from a manifest -func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string, packageVersion string) map[string]string { - globExpr := filepath.Join(versionedFlatPath, fmt.Sprintf("*%s*", packageVersion)) - if mg.Verbose() { - log.Printf("Finding files to copy with %s", globExpr) - } - files, err := filepath.Glob(globExpr) - if err != nil { - panic(err) - } - if mg.Verbose() { - log.Printf("Validating checksums for %+v", files) - log.Printf("--- Copying into %s: %v", versionedDropPath, files) - } - +// ChecksumsWithoutManifest is a helper function for flattenDependencies that's used when not packaging from a manifest. +// This function will iterate over the dependencies, resolve *exactly* the package name for each dependency and platform using the passed +// dependenciesVersion, and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithoutManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithoutManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) - for _, f := range files { + + for _, dep := range dependencies { + + if dep.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", dep.ProjectName, dep.BinaryName) + } + continue + } + + if !dep.SupportsPlatform(platform) { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", dep.ProjectName, dep.BinaryName, platform) + } + continue + } + + atLeastOnePackageTypeSelected := false + for _, pkgType := range dep.PackageTypes { + if IsPackageTypeSelected(PackageType(pkgType)) { + atLeastOnePackageTypeSelected = true + break + } + } + + if !atLeastOnePackageTypeSelected { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s supported package types %v do not overlap selected package types %v, skipping", dep.ProjectName, dep.BinaryName, dep.PackageTypes, SelectedPackageTypes) + } + continue + } + + srcDir := filepath.Join(versionedFlatPath, dep.GetRootDir(dependenciesVersion, platform)) + + if mg.Verbose() { + log.Printf("Validating checksums for %+v", dep.BinaryName) + log.Printf("--- Copying into %s: %v", versionedDropPath, srcDir) + } + options := copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow @@ -64,217 +94,99 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string Sync: true, } if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", f, versionedDropPath) + log.Printf("> prepare to copy %s into %s ", srcDir, versionedDropPath) } - err = copy.Copy(f, versionedDropPath, options) + err := copy.Copy(srcDir, versionedDropPath, options) if err != nil { - panic(err) + panic(fmt.Errorf("copying dependency %s files from %q to %q: %w", dep.BinaryName, srcDir, versionedDropPath, err)) } // copy spec file for match - specName := filepath.Base(f) - idx := strings.Index(specName, "-"+packageVersion) - if idx != -1 { - specName = specName[:idx] - } if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) + log.Printf(">>>> Looking to copy spec file: [%s]", dep.BinaryName) } - checksum, err := CopyComponentSpecs(specName, versionedDropPath) + checksum, err := CopyComponentSpecs(dep.BinaryName, versionedDropPath) if err != nil { panic(err) } - checksums[specName+ComponentSpecFileSuffix] = checksum + checksums[dep.BinaryName+ComponentSpecFileSuffix] = checksum } return checksums } -// This is a helper function for flattenDependencies that's used when building from a manifest -func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string { +// ChecksumsWithManifest is a helper function for flattenDependencies that's used when building from a manifest. +// This function will iterate over the dependencies, resolve the package name for each dependency and platform using the manifest, +// (there may be some variability there in case the manifest does not include an exact match for the expected filename), +// and it will copy the extracted files contained in the rootDir of each dependency from the versionedFlatPath +// (a directory containing all the extracted dependencies per platform) to the versionedDropPath (a drop path by platform +// that will be used to compose the package content) +// ChecksumsWithManifest will accumulate the checksums of each component spec that is copied, and return it to the caller. +func ChecksumsWithManifest(platform string, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build, dependencies []packaging.BinarySpec) map[string]string { checksums := make(map[string]string) if manifestResponse == nil { return checksums } - // Iterate over the component projects in the manifest - projects := manifestResponse.Projects - for componentName := range projects { - // Iterate over the individual package files within each component project - for pkgName := range projects[componentName].Packages { - // Only care about packages that match the required package constraint (os/arch) - if strings.Contains(pkgName, requiredPackage) { - // Iterate over the external binaries that we care about for packaging agent - for _, spec := range manifest.ExpectedBinaries { - // If the individual package doesn't match the expected prefix, then continue - if !strings.HasPrefix(pkgName, spec.BinaryName) { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage) - } - - // Get the version from the component based on the version in the package name - // This is useful in the case where it's an Independent Agent Release, where - // the opted-in projects will be one patch version ahead of the rest of the - // opted-out/previously-released projects - componentVersion := getComponentVersion(componentName, requiredPackage, projects[componentName]) - if mg.Verbose() { - log.Printf(">>>>>>> Component [%s]/[%s] version is [%s]", componentName, requiredPackage, componentVersion) - } - - // Combine the package name w/ the versioned flat path - fullPath := filepath.Join(versionedFlatPath, pkgName) - - // Eliminate the file extensions to get the proper directory - // name that we need to copy - var dirToCopy string - if strings.HasSuffix(fullPath, ".tar.gz") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")] - } else if strings.HasSuffix(fullPath, ".zip") { - dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")] - } else { - dirToCopy = fullPath - } - if mg.Verbose() { - log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy) - } - - // cloud-defend path exception - // When untarred, cloud defend untars to: - // cloud-defend-8.14.0-arm64 - // but the manifest (and most of this code) expects to be the same as - // the name in the manifest, which is: - // cloud-defend-8.14.0-linux-x86_64 - // So we have to do a bit of a transformation here - if strings.Contains(dirToCopy, "cloud-defend") { - if strings.Contains(dirToCopy, "x86_64") { - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "x86_64", "amd64") - } - if strings.Contains(dirToCopy, "arm64") { - // Not actually replacing the arch, but removing the "linux" - dirToCopy = fixCloudDefendDirPath(dirToCopy, componentVersion, "arm64", "arm64") - } - if mg.Verbose() { - log.Printf(">>>>>>> Adjusted cloud-defend directory to copy: [%s]", dirToCopy) - } - } - - // Set copy options - options := copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow - }, - Sync: true, - } - if mg.Verbose() { - log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath) - } - - // Do the copy - err := copy.Copy(dirToCopy, versionedDropPath, options) - if err != nil { - panic(err) - } - - // copy spec file for match - specName := filepath.Base(dirToCopy) - idx := strings.Index(specName, "-"+componentVersion) - if idx != -1 { - specName = specName[:idx] - } - if mg.Verbose() { - log.Printf(">>>> Looking to copy spec file: [%s]", specName) - } - - checksum, err := CopyComponentSpecs(specName, versionedDropPath) - if err != nil { - panic(err) - } - - checksums[specName+ComponentSpecFileSuffix] = checksum - } + // Iterate over the external binaries that we care about for packaging agent + for _, spec := range dependencies { + + if spec.PythonWheel { + if mg.Verbose() { + log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName) } + continue } - } - return checksums -} + if !spec.SupportsPlatform(platform) { + log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform) + continue + } -// This function is used when building with a Manifest. In that manifest, it's possible -// for projects in an Independent Agent Release to have different versions since the opted-in -// ones will be one patch version higher than the opted-out/previously released projects. -// This function tries to find the versions from the package name -func getComponentVersion(componentName string, requiredPackage string, componentProject manifest.Project) string { - var componentVersion string - var foundIt bool - // Iterate over all the packages in the component project - for pkgName := range componentProject.Packages { - // Only care about the external binaries that we want to package - for _, spec := range manifest.ExpectedBinaries { - // If the given component name doesn't match the external binary component, skip - if componentName != spec.ProjectName { - continue + manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform) + if err != nil { + if mg.Verbose() { + log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform) } + continue + } - // Split the package name on the binary name prefix plus a dash - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + rootDir := spec.GetRootDir(manifestPackage.ActualVersion, platform) - // Get the second part of the first split - secondHalf := firstSplit[1] - if len(secondHalf) < 2 { - continue - } + // Combine the package name w/ the versioned flat path + fullPath := filepath.Join(versionedFlatPath, rootDir) - // Make sure the second half matches the required package - if strings.Contains(secondHalf, requiredPackage) { - // ignore packages with names where this splitting doesn't results in proper version - if strings.Contains(secondHalf, "docker-image") { - continue - } - if strings.Contains(secondHalf, "oss-") { - continue - } - - // The component version should be the first entry after splitting w/ the requiredPackage - componentVersion = strings.Split(secondHalf, "-"+requiredPackage)[0] - foundIt = true - // break out of inner loop - break - } - } - if foundIt { - // break out of outer loop - break + if mg.Verbose() { + log.Printf(">>>>>>> Calculated directory to copy: [%s]", fullPath) } - } - if componentVersion == "" { - errMsg := fmt.Sprintf("Unable to determine component version for [%s]", componentName) - panic(errMsg) - } + // Set copy options + options := copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + } + if mg.Verbose() { + log.Printf("> prepare to copy %s into %s ", fullPath, versionedDropPath) + } - return componentVersion -} + // Do the copy + err = copy.Copy(fullPath, versionedDropPath, options) + if err != nil { + panic(err) + } -// This is a helper function for the cloud-defend package. -// When it is untarred, it does not have the same dirname as the package name. -// This adjusts for that and returns the actual path on disk for cloud-defend -func fixCloudDefendDirPath(dirPath string, componentVersion string, expectedArch string, actualArch string) string { - fixedDirPath := dirPath + checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath) + if err != nil { + panic(err) + } - cloudDefendExpectedDirName := fmt.Sprintf("cloud-defend-%s-linux-%s", componentVersion, expectedArch) - cloudDefendActualDirName := fmt.Sprintf("cloud-defend-%s-%s", componentVersion, actualArch) - if strings.Contains(fixedDirPath, cloudDefendExpectedDirName) { - fixedDirPath = strings.ReplaceAll(fixedDirPath, cloudDefendExpectedDirName, cloudDefendActualDirName) + checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum } - return fixedDirPath + return checksums } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index e99da60cb7d..23b8db0a00d 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -30,11 +30,11 @@ const defaultCrossBuildTarget = "golangCrossBuild" var Platforms = BuildPlatforms.Defaults() // SelectedPackageTypes is the list of package types. If empty, all packages types -// are considered to be selected (see isPackageTypeSelected). +// are considered to be selected (see IsPackageTypeSelected). var SelectedPackageTypes []PackageType // SelectedDockerVariants is the list of docker variants. If empty, all docker variants -// are considered to be selected (see isDockerVariantSelected). +// are considered to be selected (see IsDockerVariantSelected). var SelectedDockerVariants []DockerVariant func init() { diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 0e0bd2078ca..a3290d69a1a 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -182,6 +182,7 @@ func (b *dockerBuilder) dockerBuild() (string, error) { if b.Snapshot { tag = tag + "-SNAPSHOT" } + if repository := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) } diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go index 8618c408eb5..ddc495cb1d3 100644 --- a/dev-tools/mage/manifest/manifest.go +++ b/dev-tools/mage/manifest/manifest.go @@ -13,13 +13,14 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" "github.com/magefile/mage/mg" "golang.org/x/sync/errgroup" - "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" "github.com/elastic/elastic-agent/pkg/version" ) @@ -94,73 +95,6 @@ var PlatformPackages = map[string]string{ "windows/amd64": "windows-x86_64.zip", } -// ExpectedBinaries is a map of binaries agent needs to their project in the unified-release manager. -// The project names are those used in the "projects" list in the unified release manifest. -// See the sample manifests in the testdata directory. -var ExpectedBinaries = []BinarySpec{ - {BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "cloud-defend", ProjectName: "cloud-defend", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: []pkgcommon.PackageType{pkgcommon.Docker}}, - {BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}}, - {BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, - {BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes}, -} - -type BinarySpec struct { - BinaryName string - ProjectName string - Platforms []Platform - PythonWheel bool - PackageTypes []pkgcommon.PackageType -} - -func (proj BinarySpec) SupportsPlatform(platform string) bool { - for _, p := range proj.Platforms { - if p.Platform() == platform { - return true - } - } - return false -} - -func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool { - for _, p := range proj.PackageTypes { - if p == pkgType { - return true - } - } - return false -} - -func (proj BinarySpec) GetPackageName(version string, platform string) string { - if proj.PythonWheel { - return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version) - } - return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform]) -} - -type Platform struct { - OS string - Arch string -} - -// Converts to the format expected on the mage command line "linux", "x86_64" = "linux/amd64" -func (p Platform) Platform() string { - if p.Arch == "x86_64" { - p.Arch = "amd64" - } - if p.Arch == "aarch64" { - p.Arch = "arm64" - } - return p.OS + "/" + p.Arch -} - -var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}} - // DownloadManifest is going to download the given manifest file and return the ManifestResponse func DownloadManifest(ctx context.Context, manifest string) (Build, error) { manifestUrl, urlError := url.Parse(manifest) @@ -192,7 +126,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) { // DownloadComponents is going to download a set of components from the given manifest into the destination // dropPath folder in order to later use that folder for packaging -func DownloadComponents(ctx context.Context, manifest string, platforms []string, dropPath string) error { +func DownloadComponents(ctx context.Context, expectedBinaries []packaging.BinarySpec, manifest string, platforms []string, dropPath string) error { manifestResponse, err := DownloadManifest(ctx, manifest) if err != nil { return fmt.Errorf("failed to download remote manifest file %w", err) @@ -211,7 +145,7 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string errGrp, downloadsCtx := errgroup.WithContext(ctx) // for project, pkgs := range expectedProjectPkgs() { - for _, spec := range ExpectedBinaries { + for _, spec := range expectedBinaries { for _, platform := range platforms { targetPath := filepath.Join(dropPath) err := os.MkdirAll(targetPath, 0755) @@ -225,18 +159,13 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string continue } - pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) + resolvedPackage, err := ResolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform) if err != nil { return err } -<<<<<<< HEAD - for _, p := range pkgURL { - log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p) -======= for _, p := range resolvedPackage.URLs { fmt.Printf("Downloading [%s] [%s]\n", spec.BinaryName, p) ->>>>>>> a155660c4 (ci: build agent from snapshot DRA (#9048)) pkgFilename := path.Base(p) downloadTarget := filepath.Join(targetPath, pkgFilename) if _, err := os.Stat(downloadTarget); err != nil { @@ -253,80 +182,125 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string return fmt.Errorf("error downloading files: %w", err) } - fmt.Printf("Downloads for manifest %q complete.\n", manifest) + log.Printf("Downloads for manifest %q complete.", manifest) return nil } -func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) { - var val Package - var ok bool +type ResolvedPackage struct { + Name string + ActualVersion string + URLs []string +} + +func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { // Try the normal/easy case first - packageName := spec.GetPackageName(version, platform) - val, ok = project.Packages[packageName] - if !ok { - // If we didn't find it, it may be an Independent Agent Release, where - // the opted-in projects will have a patch version one higher than - // the rest of the projects, so we need to seek that out + packageName := spec.GetPackageName(dependencyVersion, platform) + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Got packagename [%s], looking for exact match", packageName) + } + + if exactMatch, ok := project.Packages[packageName]; ok { + // We found the exact filename we are looking for if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform]) + log.Printf(">>>>>>>>>>> Found exact match packageName for [%s, %s]: %s", project.Branch, project.CommitHash, exactMatch) } - var foundIt bool - for pkgName := range project.Packages { - if strings.HasPrefix(pkgName, spec.BinaryName) { - firstSplit := strings.Split(pkgName, spec.BinaryName+"-") - if len(firstSplit) < 2 { - continue - } + return &ResolvedPackage{ + Name: packageName, + ActualVersion: dependencyVersion, + URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL}, + }, nil + } - secondHalf := firstSplit[1] - // Make sure we're finding one w/ the same required package type - if strings.Contains(secondHalf, PlatformPackages[platform]) { - - // Split again after the version with the required package string - secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform]) - if len(secondSplit) < 2 { - continue - } - - // The first element after the split should normally be the version - pkgVersion := secondSplit[0] - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion) - } - - // Create a project/package key with the package, derived version, and required package - foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform]) - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey) - } - - // Get the package value, if it exists - val, ok = project.Packages[foundPkgKey] - if !ok { - continue - } - - if mg.Verbose() { - log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey) - } - - foundIt = true - } - } - } + // If we didn't find it, it may be an Independent Agent Release, where + // the opted-in projects will have a patch version one higher than + // the rest of the projects, so we "relax" the version constraint + return resolveManifestPackageUsingRelaxedVersion(project, spec, dependencyVersion, platform) +} - if !foundIt { - return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL) - } +func resolveManifestPackageUsingRelaxedVersion(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) { + // start with the rendered package name + packageName := spec.GetPackageName(dependencyVersion, platform) + + // Find the original version in the rendered filename + versionIndex := strings.Index(packageName, dependencyVersion) + if versionIndex == -1 { + return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion) + } + + // obtain a regexp from the exact version string that allows for some flexibility on patch version, prerelease and build metadata tokens + relaxedVersion, err := relaxVersion(dependencyVersion) + if err != nil { + return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err) } if mg.Verbose() { - log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash) + log.Printf(">>>>>>>>>>> Couldn't find exact match, relaxing agent dependencyVersion to %s", relaxedVersion) } - return []string{val.URL, val.ShaURL, val.AscURL}, nil + // locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that + relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex]) + relaxedPackageName += `(?P` + relaxedVersion + `)` + relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):]) + + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Attempting to match a filename with %s", relaxedPackageName) + } + + relaxedPackageNameRegexp, err := regexp.Compile(relaxedPackageName) + if err != nil { + return nil, fmt.Errorf("compiling relaxed package name regex %q: %w", relaxedPackageName, err) + } + + for pkgName, pkg := range project.Packages { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName) + } + if submatches := relaxedPackageNameRegexp.FindStringSubmatch(pkgName); len(submatches) > 0 { + if mg.Verbose() { + log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName) + } + return &ResolvedPackage{ + Name: pkgName, + ActualVersion: submatches[1], + URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL}, + }, nil + } + } + + return nil, fmt.Errorf("package [%s] not found in project manifest at %s using relaxed version %q", packageName, project.ExternalArtifactsManifestURL, relaxedPackageName) +} + +// versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?) +const versionRegexp = `^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` +const anyPatchVersionRegexp = `(?:0|[1-9]\d*)` + +var versionRegExp = regexp.MustCompile(versionRegexp) + +func relaxVersion(version string) (string, error) { + matchIndices := versionRegExp.FindSubmatchIndex([]byte(version)) + // Matches index pairs are (0,1) for the whole regexp and (2,3) for the patch group + // check that we have matched correctly + if len(matchIndices) < 4 { + return "", fmt.Errorf("failed to match regexp for version [%s]", version) + } + + // take the starting index of the patch version + patchStartIndex := matchIndices[2] + // copy everything before the patch version escaping the regexp + relaxedVersion := regexp.QuoteMeta(version[:patchStartIndex]) + // add the patch regexp + relaxedVersion += anyPatchVersionRegexp + // check if there's more characters after the patch version + remainderIndex := matchIndices[3] + if remainderIndex < len(version) { + // This is a looser regexp that allows anything beyond the major version to change (while still enforcing a valid patch version though) + // see TestResolveManifestPackage/Independent_Agent_Staging_8.14_apm-server and TestResolveManifestPackage/Independent_Agent_Staging_8.14_endpoint-dev + // Be more relaxed and allow for any character sequence after this + relaxedVersion += `.*` + } + return relaxedVersion, nil } func DownloadPackage(ctx context.Context, downloadUrl string, target string) error { @@ -336,12 +310,12 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err } valid := false for _, manifestHost := range AllowedManifestHosts { - if manifestHost == parsedURL.Host { + if manifestHost == parsedURL.Hostname() { valid = true } } if !valid { - log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts) + log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Hostname(), AllowedManifestHosts) return errorNotAllowedManifestURL } cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path) diff --git a/dev-tools/mage/manifest/manifest_test.go b/dev-tools/mage/manifest/manifest_test.go index b975149aa27..76a0d53cd2e 100644 --- a/dev-tools/mage/manifest/manifest_test.go +++ b/dev-tools/mage/manifest/manifest_test.go @@ -7,12 +7,15 @@ package manifest import ( _ "embed" "encoding/json" + "fmt" "log" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/dev-tools/packaging" ) var ( @@ -135,7 +138,7 @@ func TestResolveManifestPackage(t *testing.T) { projects := manifestJson.Projects // Verify the component name is in the list of expected packages. - spec, ok := findBinarySpec(tc.binary) + spec, ok := findBinarySpec(t, tc.binary) assert.True(t, ok) if !spec.SupportsPlatform(tc.platform) { @@ -143,22 +146,80 @@ func TestResolveManifestPackage(t *testing.T) { return } - urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) + resolvedPackage, err := ResolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform) require.NoError(t, err) + require.NotNil(t, resolvedPackage) - assert.Len(t, urlList, 3) - for _, url := range urlList { + assert.Len(t, resolvedPackage.URLs, 3) + for _, url := range resolvedPackage.URLs { assert.Contains(t, tc.expectedUrlList, url) } }) } } -func findBinarySpec(name string) (BinarySpec, bool) { - for _, spec := range ExpectedBinaries { +func findBinarySpec(t *testing.T, name string) (packaging.BinarySpec, bool) { + components, err := packaging.Components() + require.NoError(t, err, "error loading components from packages.yml") + + for _, spec := range components { if spec.BinaryName == name { return spec, true } } - return BinarySpec{}, false + return packaging.BinarySpec{}, false +} + +func TestRelaxVersion(t *testing.T) { + type args struct { + version string + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "major-minor-patch", + args: args{ + version: "1.2.3", + }, + want: `1\.2\.(?:0|[1-9]\d*)`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot", + args: args{ + version: "1.2.3-SNAPSHOT", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "major-minor-patch-snapshot-buildmeta", + args: args{ + version: "1.2.3-SNAPSHOT+build20250328112233", + }, + want: `1\.2\.(?:0|[1-9]\d*).*`, + wantErr: assert.NoError, + }, + { + name: "not a semver", + args: args{ + version: "foobar", + }, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := relaxVersion(tt.args.version) + if !tt.wantErr(t, err, fmt.Sprintf("relaxVersion(%v)", tt.args.version)) { + return + } + assert.Equalf(t, tt.want, got, "relaxVersion(%v)", tt.args.version) + }) + } } diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index a3917ddac1d..3032886ebe3 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,6 +34,15 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms + if mg.Verbose() { + debugSelectedPackageSpecsWithPlatform := make([]string, 0, len(Packages)) + for _, p := range Packages { + debugSelectedPackageSpecsWithPlatform = append(debugSelectedPackageSpecsWithPlatform, fmt.Sprintf("spec %s on %s/%s", p.Spec.Name, p.OS, p.Arch)) + } + + log.Printf("Packaging for platforms %v, packages %v", platforms, debugSelectedPackageSpecsWithPlatform) + } + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { @@ -42,12 +51,12 @@ func Package() error { } for _, pkgType := range pkg.Types { - if !isPackageTypeSelected(pkgType) { + if !IsPackageTypeSelected(pkgType) { log.Printf("Skipping %s package type because it is not selected", pkgType) continue } - if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) { + if pkgType == Docker && !IsDockerVariantSelected(pkg.Spec.DockerVariant) { log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant) continue } @@ -99,6 +108,10 @@ func Package() error { spec = spec.Evaluate() + if mg.Verbose() { + log.Printf("Adding task for packaging %s on %s/%s", spec.Name, target.GOOS(), target.Arch()) + } + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } @@ -111,9 +124,9 @@ func Package() error { return nil } -// isPackageTypeSelected returns true if SelectedPackageTypes is empty or if +// IsPackageTypeSelected returns true if SelectedPackageTypes is empty or if // pkgType is present on SelectedPackageTypes. It returns false otherwise. -func isPackageTypeSelected(pkgType PackageType) bool { +func IsPackageTypeSelected(pkgType PackageType) bool { if len(SelectedPackageTypes) == 0 { return true } @@ -126,9 +139,9 @@ func isPackageTypeSelected(pkgType PackageType) bool { return false } -// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if +// IsDockerVariantSelected returns true if SelectedDockerVariants is empty or if // docVariant is present on SelectedDockerVariants. It returns false otherwise. -func isDockerVariantSelected(docVariant DockerVariant) bool { +func IsDockerVariantSelected(docVariant DockerVariant) bool { if len(SelectedDockerVariants) == 0 { return true } @@ -221,7 +234,7 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-v") } - args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/package_test.go")) + args = append(args, MustExpand("{{ elastic_beats_dir }}/dev-tools/packaging/testing/package_test.go")) if params.HasModules { args = append(args, "--modules") diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index c1bcee157f9..a3189eff559 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -28,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon" + "github.com/elastic/elastic-agent/dev-tools/packaging" ) const ( @@ -104,6 +105,7 @@ type PackageSpec struct { Qualifier string `yaml:"qualifier,omitempty"` // Optional OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional + Components []packaging.BinarySpec `yaml:"components"` // Optional: Components required for this package evalContext map[string]interface{} packageDir string diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 898d963c3fa..c5856ecc342 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1,9 +1,155 @@ --- -# This file contains the package specifications for both Community Beats and -# Official Beats. The shared section contains YAML anchors that are used to +# This file contains the package specifications for Elastic Agent + +# List all the available platforms +platforms: &all-platforms + - &linux-amd64 + os: linux + arch: x86_64 + - &linux-arm64 + os: linux + arch: arm64 + - &windows-amd64 + os: windows + arch: x86_64 + - &darwin-amd64 + os: darwin + arch: x86_64 + - &darwin-arm64 + os: darwin + arch: aarch64 + +# List all the package type constants (see dev-tools/mage/pkgcommon/pkgcommon-types.go) +packageTypes: &all-package-types + - &pkg-type-rpm + 1 # RPM + - &pkg-type-deb + 2 # Deb + - &pkg-type-zip + 3 # zip + - &pkg-type-targz + 4 # tar.gz + - &pkg-type-docker + 5 # docker + +# Settings section contains general compiling and packaging settings +# settings: + +# List *all* the components available for packaging in elastic-agent +components: + # general template for new components (all attributes are mandatory) + # - &comp- + # projectName: + # packageName: