From d37d790970fec166e99d6ab309b1e13d7fcfc407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Thu, 24 Jul 2025 15:10:27 +0200 Subject: [PATCH 1/5] Add an env variable to keep the download archive --- magefile.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/magefile.go b/magefile.go index ede727682fe..8996825f0c8 100644 --- a/magefile.go +++ b/magefile.go @@ -1199,12 +1199,16 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s log.Printf("dependencies extracted from package specs: %v", dependencies) } + keepArchive := os.Getenv("KEEP_ARCHIVE") != "" + // download/copy all the necessary dependencies for packaging elastic-agent archivePath, dropPath, dependencies := collectPackageDependencies(platforms, dependenciesVersion, packageTypes, dependencies) // cleanup after build - defer os.RemoveAll(archivePath) - defer os.RemoveAll(dropPath) + if !keepArchive { + defer os.RemoveAll(archivePath) + defer os.RemoveAll(dropPath) + } defer os.Unsetenv(agentDropPath) // create flat dir From e3db4570a168897ae4a854929fb88afa8b3cd80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Fri, 25 Jul 2025 17:26:19 +0200 Subject: [PATCH 2/5] Only download artifacts which aren't present --- dev-tools/mage/downloads/utils.go | 34 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/dev-tools/mage/downloads/utils.go b/dev-tools/mage/downloads/utils.go index 4235e522276..400cdb227ac 100644 --- a/dev-tools/mage/downloads/utils.go +++ b/dev-tools/mage/downloads/utils.go @@ -31,13 +31,7 @@ type downloadRequest struct { // It writes to the destination file as it downloads it, without // loading the entire file into memory. func downloadFile(downloadRequest *downloadRequest) error { - targetFile, err := os.Create(downloadRequest.TargetPath) - if err != nil { - return fmt.Errorf("creating file: %w", err) - } - defer func() { - _ = targetFile.Close() - }() + stat, _ := os.Stat(downloadRequest.TargetPath) exp := getExponentialBackoff(3) @@ -47,7 +41,12 @@ func downloadFile(downloadRequest *downloadRequest) error { if err != nil { return fmt.Errorf("creating request: %w", err) } - resp, err := http.DefaultClient.Do(req) + // if the target file already exists, add the If-Modified-Since header + if stat != nil { + req.Header.Add("If-Modified-Since", stat.ModTime().Format(http.TimeFormat)) + } + + resp, err := http.DefaultClient.Do(req) //nolint:bodyclose // we do close this outside of the function if err != nil { retryCount++ return fmt.Errorf("downloading file %s: %w", downloadRequest.URL, err) @@ -55,6 +54,19 @@ func downloadFile(downloadRequest *downloadRequest) error { defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified { + return nil + } + + targetFile, err := os.Create(downloadRequest.TargetPath) + if err != nil { + return fmt.Errorf("creating file: %w", err) + } + defer func() { + _ = targetFile.Close() + }() + _, err = io.Copy(targetFile, resp.Body) if err != nil { // try to drain the body before returning to ensure the connection can be reused @@ -62,16 +74,16 @@ func downloadFile(downloadRequest *downloadRequest) error { return fmt.Errorf("writing file %s: %w", targetFile.Name(), err) } + _ = os.Chmod(targetFile.Name(), 0666) + return nil } - err = backoff.Retry(download, exp) + err := backoff.Retry(download, exp) if err != nil { return err } - _ = os.Chmod(targetFile.Name(), 0666) - return nil } From 6a9f59f3996672072cf1243e0458a22c963b331c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Fri, 25 Jul 2025 19:11:31 +0200 Subject: [PATCH 3/5] Document the KEEP_ARCHIVE env variable --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8b901dcda52..fb580866891 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,11 @@ To build a local version of the agent for development, run the command below. Th DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz mage -v package ``` +If you build the same agent package often (when running integration tests, for example), +you can also set KEEP_ARCHIVE=true in your environment. The packaging step will then +avoid deleting the binary archive after the package is generated, removing the need to +re-download binaries on every invocaion. + The resulting package will be produced in the build/distributions directory. The version is controlled by the value in [version.go](version/version.go). To install the agent extract the package and run the install command: From f9f3a1d8c3dac9c79b543114a48ae270502e5aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Thu, 31 Jul 2025 10:32:36 +0200 Subject: [PATCH 4/5] Update README.md Co-authored-by: Panos Koutsovasilis --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb580866891..49427f2a598 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz mage If you build the same agent package often (when running integration tests, for example), you can also set KEEP_ARCHIVE=true in your environment. The packaging step will then avoid deleting the binary archive after the package is generated, removing the need to -re-download binaries on every invocaion. +re-download binaries on every invocation. The resulting package will be produced in the build/distributions directory. The version is controlled by the value in [version.go](version/version.go). To install the agent extract the package and run the install command: From cb046292ec71ad3f5a56bf0659a654179d53c8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 9 Sep 2025 17:37:11 +0200 Subject: [PATCH 5/5] fixup! Only download artifacts which aren't present --- dev-tools/mage/downloads/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/downloads/utils.go b/dev-tools/mage/downloads/utils.go index 400cdb227ac..e0ae9cfd20e 100644 --- a/dev-tools/mage/downloads/utils.go +++ b/dev-tools/mage/downloads/utils.go @@ -46,7 +46,7 @@ func downloadFile(downloadRequest *downloadRequest) error { req.Header.Add("If-Modified-Since", stat.ModTime().Format(http.TimeFormat)) } - resp, err := http.DefaultClient.Do(req) //nolint:bodyclose // we do close this outside of the function + resp, err := http.DefaultClient.Do(req) if err != nil { retryCount++ return fmt.Errorf("downloading file %s: %w", downloadRequest.URL, err)