Add a mage target that packages elastic-agent using elastic-agent-cor…

…e DRA (#4403)

* Extract dependencies download to its own function

* Add PackageUsingDRA mage target

* filter elastic-agent-core artifacts by platform

* update buildkite package script with packageUsingDRA target

* Add sha512 validation for downloaded elastic-agent-core packages

* Override the elastic agent commit hash when packaging using DRA

* change Manifest URL env variable name

(cherry picked from commit 87336db)

# Conflicts:
#	dev-tools/mage/settings.go
#	magefile.go

.buildkite/scripts/steps/package.sh

@@ -13,8 +13,8 @@ fi
 export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 
-# Download the components from the ManifestURL and then package those downloaded into the $AGENT_DROP_PATH
-mage clean downloadManifest package ironbank fixDRADockerArtifacts
+# Download the components from the MANIFEST_URL and then package those downloaded into the $AGENT_DROP_PATH
+mage clean downloadManifest packageUsingDRA ironbank fixDRADockerArtifacts
 
 echo  "+++ Generate dependencies report"
 BEAT_VERSION_FULL=$(curl -s -XGET "${MANIFEST_URL}" |jq '.version' -r )

dev-tools/mage/manifest/manifest.go

@@ -130,7 +130,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 						downloadTarget := filepath.Join(targetPath, pkgFilename)
 						if _, err := os.Stat(downloadTarget); err != nil {
 							errGrp.Go(func(ctx context.Context, url, target string) func() error {
-								return func() error { return downloadPackage(ctx, url, target) }
+								return func() error { return DownloadPackage(ctx, url, target) }
 							}(downloadsCtx, p, downloadTarget))
 						}
 					}
@@ -150,7 +150,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 	return nil
 }
 
-func downloadPackage(ctx context.Context, downloadUrl string, target string) error {
+func DownloadPackage(ctx context.Context, downloadUrl string, target string) error {
 	parsedURL, errorUrl := url.Parse(downloadUrl)
 	if errorUrl != nil {
 		return errorInvalidManifestURL

dev-tools/mage/settings.go

@@ -46,6 +46,11 @@ const (
 	agentPackageVersionEnvVar = "AGENT_PACKAGE_VERSION"
 	//ManifestUrlEnvVar is the name fo the environment variable containing the Manifest URL to be used for packaging agent
 	ManifestUrlEnvVar = "MANIFEST_URL"
+<<<<<<< HEAD
+=======
+	// AgentCommitHashEnvVar allows to override agent commit hash string during packaging
+	AgentCommitHashEnvVar
+>>>>>>> 87336dbaab (Add a mage target that packages elastic-agent using elastic-agent-core DRA (#4403))
 
 	// Mapped functions
 	agentPackageVersionMappedFunc    = "agent_package_version"
@@ -283,7 +288,12 @@ var (
 func CommitHash() (string, error) {
 	var err error
 	commitHashOnce.Do(func() {
-		commitHash, err = sh.Output("git", "rev-parse", "HEAD")
+		// Check commit hash override first
+		commitHash = EnvOr(AgentCommitHashEnvVar, "")
+		if commitHash == "" {
+			// no override found, get the hash from HEAD
+			commitHash, err = sh.Output("git", "rev-parse", "HEAD")
+		}
 	})
 	return commitHash, err
 }

internal/pkg/agent/application/upgrade/artifact/download/verifier.go

@@ -11,6 +11,7 @@ import (
 	"crypto/sha512"
 	"encoding/hex"
 	"fmt"
+	"hash"
 	"io"
 	"net/http"
 	"net/url"
@@ -130,8 +131,16 @@ func getHashFileName(filename string) string {
 // exists and that the checksum in the sidecar file matches the checksum of
 // the file. It returns an error if validation fails.
 func VerifySHA512Hash(filename string) error {
+	hasher := sha512.New()
+	checksumFileName := getHashFileName(filename)
+	return VerifyChecksum(hasher, filename, checksumFileName)
+}
+
+// VerifyChecksum checks that the hash contained in checksumFileName correspond to the hash calculated for filename using
+// hasher.Sum()
+func VerifyChecksum(hasher hash.Hash, filename, checksumFileName string) error {
 	// Read expected checksum.
-	expectedHash, err := readChecksumFile(getHashFileName(filename), filepath.Base(filename))
+	expectedHash, err := readChecksumFile(checksumFileName, filepath.Base(filename))
 	if err != nil {
 		return fmt.Errorf("could not read checksum file: %w", err)
 	}
@@ -143,12 +152,11 @@ func VerifySHA512Hash(filename string) error {
 	}
 	defer f.Close()
 
-	hash := sha512.New()
-	if _, err := io.Copy(hash, f); err != nil {
+	if _, err := io.Copy(hasher, f); err != nil {
 		return fmt.Errorf("faled to read file to calculate hash")
 	}
 
-	computedHash := hex.EncodeToString(hash.Sum(nil))
+	computedHash := hex.EncodeToString(hasher.Sum(nil))
 	if computedHash != expectedHash {
 		return &ChecksumMismatchError{
 			Expected: expectedHash,