Add tests to ensure remote client auth headers function correctly

NOTICE-fips.txt

@@ -1254,11 +1254,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.32.2
+Version: v0.33.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.32.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.33.2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

NOTICE.txt

@@ -1254,11 +1254,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.32.2
+Version: v0.33.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.32.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.33.2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

changelog/fragments/1772067652-Add-support-for-agent-download-auth-headers.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add support for agent download auth headers
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/12962
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae
 	github.com/elastic/elastic-agent-autodiscover v0.10.2
 	github.com/elastic/elastic-agent-client/v7 v7.18.0
-	github.com/elastic/elastic-agent-libs v0.32.2
+	github.com/elastic/elastic-agent-libs v0.33.2
 	github.com/elastic/elastic-agent-system-metrics v0.14.1
 	github.com/elastic/elastic-transport-go/v8 v8.9.0
 	github.com/elastic/go-elasticsearch/v8 v8.19.3

go.sum

@@ -171,8 +171,8 @@ github.com/elastic/elastic-agent-autodiscover v0.10.2 h1:fzi+CIcK7FKUQlQHfKP+3Ix
 github.com/elastic/elastic-agent-autodiscover v0.10.2/go.mod h1:qBoYxp3lX3qFRYjEgsOaROC+xL4ItG63GSvOg1SjjsQ=
 github.com/elastic/elastic-agent-client/v7 v7.18.0 h1:zXdoErqECfvRjj1c+1Ko4VGuUnFtJfTiC9JJz4aQCHc=
 github.com/elastic/elastic-agent-client/v7 v7.18.0/go.mod h1:ChFjpIfSsQOnhWT3se5Euw/o8P4Vf77FxeObE/vUfSc=
-github.com/elastic/elastic-agent-libs v0.32.2 h1:2CaO4TAMcZDy0qNxh48Ui1oMc+xExKL+JkpcyOxF7XA=
-github.com/elastic/elastic-agent-libs v0.32.2/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
+github.com/elastic/elastic-agent-libs v0.33.2 h1:Y0XKL8xOoHsLdu0J2Iwp57Wa2APFORRzSGWWbukrVPg=
+github.com/elastic/elastic-agent-libs v0.33.2/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
 github.com/elastic/elastic-agent-system-metrics v0.14.1 h1:XdV3KWbug/M1dkn8h9Yth9pfdyeFR2Z1iqaypX+ohBg=
 github.com/elastic/elastic-agent-system-metrics v0.14.1/go.mod h1:JNfnZrC0viAjlJRUzQKKuMpDlXgjXBn4WdWEXQF7jcA=
 github.com/elastic/elastic-transport-go/v8 v8.9.0 h1:KeT/2P54F0xS0S8Y3Pf+tFDg4HmBgReQMB+BMz8dDAs=

internal/edot/go.mod

@@ -10,7 +10,7 @@ replace github.com/elastic/beats/v7 => ../../beats
 require (
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20260227151221-1f06953f41bf
 	github.com/elastic/elastic-agent v0.0.0-00010101000000-000000000000
-	github.com/elastic/elastic-agent-libs v0.32.2
+	github.com/elastic/elastic-agent-libs v0.33.2
 	github.com/elastic/mock-es v0.0.0-20250530054253-8c3b6053f9b6
 	github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.29.0
 	github.com/elastic/opentelemetry-collector-components/connector/profilingmetricsconnector v0.29.0

internal/edot/go.sum

@@ -452,8 +452,8 @@ github.com/elastic/elastic-agent-autodiscover v0.10.2 h1:fzi+CIcK7FKUQlQHfKP+3Ix
 github.com/elastic/elastic-agent-autodiscover v0.10.2/go.mod h1:qBoYxp3lX3qFRYjEgsOaROC+xL4ItG63GSvOg1SjjsQ=
 github.com/elastic/elastic-agent-client/v7 v7.18.0 h1:zXdoErqECfvRjj1c+1Ko4VGuUnFtJfTiC9JJz4aQCHc=
 github.com/elastic/elastic-agent-client/v7 v7.18.0/go.mod h1:ChFjpIfSsQOnhWT3se5Euw/o8P4Vf77FxeObE/vUfSc=
-github.com/elastic/elastic-agent-libs v0.32.2 h1:2CaO4TAMcZDy0qNxh48Ui1oMc+xExKL+JkpcyOxF7XA=
-github.com/elastic/elastic-agent-libs v0.32.2/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
+github.com/elastic/elastic-agent-libs v0.33.2 h1:Y0XKL8xOoHsLdu0J2Iwp57Wa2APFORRzSGWWbukrVPg=
+github.com/elastic/elastic-agent-libs v0.33.2/go.mod h1:0xUg7alsNE/WhY9DZRIdTYW75nqSHC1octIAg//j/PQ=
 github.com/elastic/elastic-agent-system-metrics v0.14.1 h1:XdV3KWbug/M1dkn8h9Yth9pfdyeFR2Z1iqaypX+ohBg=
 github.com/elastic/elastic-agent-system-metrics v0.14.1/go.mod h1:JNfnZrC0viAjlJRUzQKKuMpDlXgjXBn4WdWEXQF7jcA=
 github.com/elastic/elastic-transport-go/v8 v8.9.0 h1:KeT/2P54F0xS0S8Y3Pf+tFDg4HmBgReQMB+BMz8dDAs=

internal/pkg/remote/client_test.go

@@ -369,6 +369,69 @@ func TestHTTPClient(t *testing.T) {
 			assert.Equal(t, successResp, string(body))
 		},
 	))
+
+	t.Run("Basic auth header", withServer(
+		func(t *testing.T) *http.ServeMux {
+			mux := http.NewServeMux()
+			mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+				user, pass, _ := r.BasicAuth()
+				if user != "test-user" || pass != "test-pass" {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			})
+			return mux
+		}, func(t *testing.T, host string) {
+			cfg := config.MustNewConfigFrom(map[string]any{
+				"host": host,
+				"auth": map[string]any{
+					"username": "test-user",
+					"password": "test-pass",
+				},
+			})
+			client, err := NewWithRawConfig(nil, cfg, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Send(t.Context(), http.MethodGet, "/", nil, nil, nil)
+			require.NoError(t, err)
+			resp.Body.Close()
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+		},
+	))
+
+	t.Run("wrong auth header", withServer(
+		func(t *testing.T) *http.ServeMux {
+			mux := http.NewServeMux()
+			mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+				// Ensure API key is sent but not basic auth
+				assert.Equal(t, "ApiKey test-key", r.Header.Get("Authorization"), "Expected to see ApiKey auth instead of basic auth")
+				user, pass, _ := r.BasicAuth()
+				if user != "test-user" || pass != "test-pass" {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			})
+			return mux
+		}, func(t *testing.T, host string) {
+			cfg := config.MustNewConfigFrom(map[string]any{
+				"host": host,
+				"auth": map[string]any{
+					"api_key":  "test-key",
+					"username": "test-user",
+					"password": "test-pass",
+				},
+			})
+			client, err := NewWithRawConfig(nil, cfg, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Send(t.Context(), http.MethodGet, "/", nil, nil, nil)
+			require.NoError(t, err)
+			resp.Body.Close()
+			require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+		},
+	))
 }
 
 func TestSortClients(t *testing.T) {

internal/pkg/remote/config_test.go

@@ -24,6 +24,10 @@ func TestPackUnpack(t *testing.T) {
 		Path:     "/ok",
 		Transport: httpcommon.HTTPTransportSettings{
 			Timeout: 10 * time.Second,
+			Auth: &httpcommon.HTTPAuthorization{
+				Username: "user",
+				Password: "pass",
+			},
 		},
 	}
 

testing/integration/ess/proxy_url_test.go

@@ -11,6 +11,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -1031,3 +1032,177 @@ func TestFleetDownloadProxyURL(t *testing.T) {
 	require.NotEmpty(t, artifactsProxy.ProxiedRequests(), "artifactsProxy does not have any requests")
 	require.NotEmpty(t, proxy.ProxiedRequests(), "proxy does not have any requests")
 }
+
+func TestFleetDownloadAuthUpgrade(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: integration.Fleet,
+		Stack: &define.Stack{},
+		Local: false,
+		Sudo:  true,
+	})
+
+	// Get start and end versions
+	startFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+	err = startFixture.Prepare(t.Context())
+	require.NoError(t, err)
+	startVersionInfo, err := startFixture.ExecVersion(t.Context())
+	require.NoError(t, err)
+	startParsedVersion, err := version.ParseVersion(startVersionInfo.Binary.String())
+	require.NoError(t, err)
+
+	endFixture, err := atesting.NewFixture(
+		t,
+		upgradetest.EnsureSnapshot(define.Version()),
+		atesting.WithFetcher(atesting.ArtifactFetcher()),
+	)
+	require.NoError(t, err)
+	err = endFixture.Prepare(t.Context())
+	require.NoError(t, err)
+	endVersionInfo, err := endFixture.ExecVersion(t.Context())
+	require.NoError(t, err)
+
+	// Ensure we can run the test
+	if startVersionInfo.Binary.String() == endVersionInfo.Binary.String() &&
+		startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit {
+		t.Skipf("Build under test is the same as the build from the artifacts repository (version: %s) [commit: %s]",
+			startVersionInfo.Binary.String(), startVersionInfo.Binary.Commit)
+	}
+	if startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit {
+		t.Skipf("Target version has the same commit hash %q", endVersionInfo.Binary.Commit)
+	}
+
+	// Setup a custom download source that requires auth.
+	// Re-use the existing proxytest code to be the server.
+	username := "download-user"
+	password := "download-pass"
+
+	artifactsProxy := proxytest.New(t,
+		proxytest.WithVerifyRequest(func(r *http.Request) error {
+			if !strings.HasPrefix(r.URL.Path, "/downloads/") {
+				return fmt.Errorf("unexpected artifact path %q", r.URL.Path)
+			}
+			if user, pass, _ := r.BasicAuth(); user != username || pass != password {
+				return fmt.Errorf("unexpected auth %s:%s", user, pass)
+			}
+			return nil
+		}),
+		proxytest.WithRewriteFn(func(u *url.URL) {
+			u.Scheme = "https"
+			u.Host = "snapshots.elastic.co"
+		}),
+		proxytest.WithRequestLog("artifacts-auth", t.Logf),
+		proxytest.WithVerboseLog())
+	err = artifactsProxy.Start()
+	require.NoError(t, err, "error starting artifacts proxy")
+	t.Cleanup(artifactsProxy.Close)
+
+	kibClient := info.KibanaClient
+	fleetServerURL, err := fleettools.DefaultURL(t.Context(), kibClient)
+	require.NoError(t, err)
+	testUUID, err := uuid.NewV4()
+	require.NoError(t, err, "error generating UUID for test")
+
+	// Define a download source that includes auth headers
+	downloadSource := kibana.DownloadSource{
+		Name: "LocalArtifactsAuth-" + testUUID.String(),
+		Host: artifactsProxy.LocalhostURL + "/downloads/",
+		Auth: &kibana.DownloadSourceAuth{
+			Username: username,
+			Password: password,
+		},
+	}
+	sourceResp, err := kibClient.CreateDownloadSource(t.Context(), downloadSource)
+	require.NoError(t, err, "unable to create download source")
+
+	// Create an agent policy that uses the download source
+	t.Log("Creating Agent policy...")
+	policy := kibana.AgentPolicy{
+		Name:        "test-policy-auth-" + testUUID.String(),
+		Namespace:   "default",
+		Description: "Test policy " + testUUID.String(),
+		MonitoringEnabled: []kibana.MonitoringEnabledOption{
+			kibana.MonitoringEnabledLogs,
+			kibana.MonitoringEnabledMetrics,
+		},
+		DownloadSourceID: sourceResp.Item.ID,
+		AdvancedSettings: map[string]interface{}{
+			"agent_download_timeout": "60m",
+		},
+	}
+	policyResp, err := kibClient.CreatePolicy(t.Context(), policy)
+	require.NoError(t, err)
+
+	enrollmentToken, err := kibClient.CreateEnrollmentAPIKey(t.Context(), kibana.CreateEnrollmentAPIKeyRequest{
+		PolicyID: policyResp.ID,
+	})
+	require.NoError(t, err)
+
+	err = upgradetest.ConfigureFastWatcher(t.Context(), startFixture)
+	require.NoError(t, err, "unable to write fast watcher config")
+
+	t.Log("Installing Elastic Agent...")
+	installOpts := atesting.InstallOpts{
+		Force: true,
+		EnrollOpts: atesting.EnrollOpts{
+			URL:             fleetServerURL,
+			EnrollmentToken: enrollmentToken.APIKey,
+		},
+	}
+	output, err := startFixture.Install(t.Context(), &installOpts)
+	t.Logf("Install agent output:\n%s", string(output))
+	require.NoError(t, err)
+
+	t.Log("Waiting for Agent to be correct version and healthy...")
+	err = upgradetest.WaitHealthyAndVersion(t.Context(), startFixture, startVersionInfo.Binary, 2*time.Minute, 10*time.Second, t)
+	require.NoError(t, err)
+
+	agentID, err := startFixture.AgentID(t.Context())
+	require.NoError(t, err)
+	t.Logf("Agent ID: %q", agentID)
+
+	t.Log("Waiting for enrolled Agent status to be online...")
+	require.Eventually(t, func() bool {
+		return check.FleetAgentStatus(t.Context(), t, kibClient, agentID, "online")()
+	}, time.Minute*2, time.Second, "Agent did not come online")
+
+	t.Logf("Upgrading from version \"%s-%s\" to version \"%s-%s\"...",
+		startParsedVersion, startVersionInfo.Binary.Commit,
+		endVersionInfo.Binary.String(), endVersionInfo.Binary.Commit)
+	err = fleettools.UpgradeAgent(t.Context(), kibClient, agentID, endVersionInfo.Binary.String(), true)
+	require.NoError(t, err)
+
+	t.Log("Ensure upgrade starts")
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := startFixture.ExecStatus(t.Context())
+		require.NoError(c, err)
+		require.NotNil(c, status.UpgradeDetails, "Agent status does not contain upgrade_details.")
+	}, time.Minute*5, time.Second, "Unable to verify that upgrade details appear.")
+
+	t.Log("Waiting for upgrade watcher to start...")
+	err = upgradetest.WaitForWatcher(t.Context(), 5*time.Minute, 10*time.Second)
+	require.NoError(t, err)
+	t.Log("Upgrade watcher started")
+
+	err = upgradetest.WaitHealthyAndVersion(t.Context(), startFixture, endVersionInfo.Binary, 2*time.Minute, 10*time.Second, t)
+	require.NoError(t, err)
+
+	t.Log("Waiting for upgraded Agent status to be online...")
+	require.Eventually(t, func() bool {
+		return check.FleetAgentStatus(t.Context(), t, kibClient, agentID, "online")()
+	}, time.Minute*10, time.Second*10, "Agent did not come online")
+
+	t.Log("Check agent version")
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		ver, err := fleettools.GetAgentVersion(t.Context(), kibClient, agentID)
+		require.NoError(c, err)
+		require.Equal(c, endVersionInfo.Binary.Version, ver)
+	}, time.Minute*5, time.Second)
+
+	t.Log("Waiting for upgrade watcher to finish...")
+	err = upgradetest.WaitForNoWatcher(t.Context(), 2*time.Minute, 10*time.Second, 1*time.Minute+15*time.Second)
+	require.NoError(t, err)
+
+	err = upgradetest.CheckHealthyAndVersion(t.Context(), startFixture, endVersionInfo.Binary)
+	require.NoError(t, err, "Post watcher check has failed, agent may have rolled back")
+}