Add custom ECH deployment + smoke test

.buildkite/bk.integration-fips.pipeline.yml

@@ -18,31 +18,8 @@ common:
         env_var: "EC_API_KEY"
 
 steps:
-  - label: Build and push custom elastic-agent image
-    depends_on:
-      - 'packaging-containers-x86-64-fips' # Reuse artifacts produced in .buildkite/integration.pipeline.yml
-    key: integration-fips-cloud-image
-    env:
-      FIPS: "true"
-      CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
-      CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
-      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
-    command: |
-      buildkite-agent artifact download build/distributions/elastic-agent-cloud-fips-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
-      mage cloud:load
-      mage cloud:push
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-      image: "${IMAGE_UBUNTU_2404_X86_64}"
-    plugins:
-      - elastic/vault-docker-login#v0.5.2:
-          secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
-
   - label: Start ESS stack for FIPS integration tests
     key: integration-fips-ess
-    depends_on:
-      - integration-fips-cloud-image
     env:
       ASDF_TERRAFORM_VERSION: 1.9.2
       TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
@@ -55,7 +32,7 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
     plugins:
-      - *vault_ec_key_prod  
+      - *vault_ec_key_prod
 
   - group: "fips:Stateful:Ubuntu"
     key: integration-tests-ubuntu-fips
@@ -83,7 +60,7 @@ steps:
           image: "${IMAGE_UBUNTU_X86_64_FIPS}"
           instanceType: "m5.2xlarge"
         plugins:
-          - *vault_ec_key_prod  
+          - *vault_ec_key_prod
         matrix:
           setup:
             sudo:
@@ -113,7 +90,7 @@ steps:
           image: "${IMAGE_UBUNTU_ARM64_FIPS}"
           instanceType: "m6g.2xlarge"
         plugins:
-          - *vault_ec_key_prod  
+          - *vault_ec_key_prod
         matrix:
           setup:
             sudo:
@@ -153,7 +130,7 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
     plugins:
-      - *vault_ec_key_prod  
+      - *vault_ec_key_prod
 
   - label: Aggregate test reports
     depends_on:

.buildkite/bk.integration.pipeline.yml

@@ -41,9 +41,34 @@ common:
       elastic/vault-secrets#v0.1.0:
         path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
         field: "apiKey"
-        env_var: "EC_API_KEY"  
+        env_var: "EC_API_KEY"
 
 steps:
+  - label: "Custom ECH Testing"
+    key: integration-tests-ech
+    depends_on:
+      - packaging-ubuntu-x86-64
+    env:
+      TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud:git-${BUILDKITE_COMMIT:0:12}"
+      FORCE_ESS_CREATE: "true"
+    command: |
+      #!/usr/bin/env bash
+      buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
+      .buildkite/scripts/steps/integration_tests_tf.sh ech true
+    artifact_paths:
+      - build/**
+      - build/diagnostics/**
+    retry:
+      automatic:
+        limit: 1
+    agents:
+      provider: "gcp"
+      machineType: "n2-standard-8"
+      image: "${IMAGE_UBUNTU_2404_X86_64}"
+    plugins:
+      - *vault_ec_key_prod
+
   - label: Start ESS stack for integration tests
     key: integration-ess
     notify:
@@ -113,7 +138,7 @@ steps:
           image: "${IMAGE_WIN_2025}"
         plugins:
           - *vault_ec_key_prod
-  
+
       - label: "Ubuntu:2404:amd64:sudo"
         depends_on: packaging-ubuntu-x86-64
         env:
@@ -298,7 +323,7 @@ steps:
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
         plugins:
-          - *vault_ec_key_prod  
+          - *vault_ec_key_prod
         matrix:
           - default
           - upgrade
@@ -614,6 +639,7 @@ steps:
     # Warning: The key has a hook in pre-command
     key: aggregate-reports
     depends_on:
+      - integration-tests-ech
       - integration-tests-ubuntu
       - integration-tests-win
       - integration-tests-rhel8

.buildkite/integration.pipeline.yml

@@ -3,6 +3,11 @@
 env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
+common:
+  - vault_docker_login: &vault_docker_login
+      elastic/vault-docker-login#v0.5.2:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
 steps:
   - group: "Integration tests: packaging"
     key: "int-packaging"
@@ -97,13 +102,21 @@ steps:
         env:
           PACKAGES: "docker"
           PLATFORMS: "linux/amd64"
-        command: ".buildkite/scripts/steps/integration-package.sh"
+          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud"
+        command: |
+          source .buildkite/scripts/common.sh
+          .buildkite/scripts/steps/integration-package.sh
+          echo "~~~ Pushing cloud image"
+          mage cloud:push
         artifact_paths:
           - build/distributions/**
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
           diskSizeGb: 200
+        plugins:
+          - *vault_docker_login
 
       - label: "Packaging: Containers linux/arm64"
         key: packaging-containers-arm64
@@ -126,13 +139,21 @@ steps:
           PACKAGES: "docker"
           PLATFORMS: "linux/amd64"
           FIPS: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
+          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
+        command: |
+          source .buildkite/scripts/common.sh
+          .buildkite/scripts/steps/integration-package.sh
+          echo "~~~ Pushing cloud image"
+          mage cloud:push
         artifact_paths:
           - build/distributions/**
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
           diskSizeGb: 200
+        plugins:
+          - *vault_docker_login
 
       - label: "Packaging: Containers linux/arm64 FIPS"
         key: packaging-containers-arm64-fips

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -32,7 +32,7 @@ mage build:testBinaries
 # If the step is retried, we start the stack again.
 # BUILDKITE_RETRY_COUNT == "0" for the first run
 # BUILDKITE_RETRY_COUNT > 0 for the retries
-if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
+if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 || "${FORCE_ESS_CREATE:-false}" == "true" ]]; then
   echo "~~~ The steps is retried, starting the ESS stack again"
   trap 'ess_down' EXIT
   ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack")

magefile.go

@@ -991,14 +991,11 @@ func (Cloud) Load() error {
 
 	// Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant
 	fips := os.Getenv(fipsEnv)
-	defer os.Setenv(fipsEnv, fips)
 	fipsVal, err := strconv.ParseBool(fips)
 	if err != nil {
 		fipsVal = false
 	}
-	if err := os.Setenv(fipsEnv, strconv.FormatBool(fipsVal)); err != nil {
-		return fmt.Errorf("failed to set fips env var: %w", err)
-	}
+
 	devtools.FIPSBuild = fipsVal
 
 	source := "build/distributions/elastic-agent-cloud-" + agentVersion + "-SNAPSHOT-linux-" + runtime.GOARCH + ".docker.tar.gz"

testing/integration/ess/ech_test.go

@@ -0,0 +1,125 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+//go:build integration
+
+package ess
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gofrs/uuid/v5"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent-libs/kibana"
+	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
+	atesting "github.com/elastic/elastic-agent/pkg/testing"
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
+	"github.com/elastic/elastic-agent/testing/integration"
+)
+
+func TestECH(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: integration.ECH,
+		Stack: &define.Stack{},
+		Sudo:  true,
+		Local: false,
+		OS: []define.OS{
+			{
+				Type: define.Linux,
+			},
+		},
+	})
+
+	// Check that the Fleet Server in the deployment is healthy
+	fleetServerHost, err := fleettools.DefaultURL(t.Context(), info.KibanaClient)
+	statusUrl, err := url.JoinPath(fleetServerHost, "/api/status")
+	require.NoError(t, err)
+
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		resp, err := http.Get(statusUrl)
+		require.NoError(c, err)
+		defer resp.Body.Close()
+
+		require.Equal(c, http.StatusOK, resp.StatusCode)
+
+		var body struct {
+			Name   string `json:"name"`
+			Status string `json:"status"`
+		}
+		err = json.NewDecoder(resp.Body).Decode(&body)
+		require.NoError(c, err)
+
+		t.Logf("body.status = %s", body.Status)
+		require.Equal(c, "HEALTHY", body.Status)
+	}, 5*time.Minute, 10*time.Second, "Fleet Server in ECH deployment is not healthy")
+
+	// Create a policy and install an agent
+	policyUUID := uuid.Must(uuid.NewV4()).String()
+	policy := kibana.AgentPolicy{
+		Name:              "testloglevel-policy-" + policyUUID,
+		Namespace:         "default",
+		Description:       "Test Log Level Policy " + policyUUID,
+		MonitoringEnabled: []kibana.MonitoringEnabledOption{},
+	}
+	t.Log("Creating Agent policy...")
+	policyResp, err := info.KibanaClient.CreatePolicy(t.Context(), policy)
+	require.NoError(t, err, "failed creating policy")
+
+	t.Log("Creating Agent enrollment API key...")
+	createEnrollmentApiKeyReq := kibana.CreateEnrollmentAPIKeyRequest{
+		PolicyID: policyResp.ID,
+	}
+	enrollmentToken, err := info.KibanaClient.CreateEnrollmentAPIKey(t.Context(), createEnrollmentApiKeyReq)
+	require.NoError(t, err, "failed creating enrollment API key")
+	t.Logf("Created policy %+v", policyResp.AgentPolicy)
+
+	fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+	err = fixture.Prepare(t.Context())
+	require.NoError(t, err)
+
+	opts := &atesting.InstallOpts{
+		Force:      true,
+		Privileged: true,
+		EnrollOpts: atesting.EnrollOpts{
+			URL:             fleetServerHost,
+			EnrollmentToken: enrollmentToken.APIKey,
+		},
+	}
+	out, err := fixture.Install(t.Context(), opts)
+	if err != nil {
+		t.Logf("install output: %s", out)
+		require.NoError(t, err)
+	}
+
+	var agentID string
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := fixture.ExecStatus(t.Context())
+		require.NoError(c, err)
+		statusBuffer := new(strings.Builder)
+		err = json.NewEncoder(statusBuffer).Encode(status)
+		require.NoError(c, err)
+		t.Logf("agent status: %v", statusBuffer.String())
+
+		require.Equal(c, int(cproto.State_HEALTHY), status.State, "agent state is not healthy")
+		require.Equal(c, int(cproto.State_HEALTHY), status.FleetState, "agent's fleet-server state is not healthy")
+		agentID = status.Info.ID
+	}, time.Minute, time.Second, "agent never became healthy or connected to Fleet")
+
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := fleettools.GetAgentStatus(t.Context(), info.KibanaClient, agentID)
+		require.NoError(c, err)
+		require.Equal(c, "online", status)
+	}, time.Minute, time.Second, "agent does not show as online in fleet")
+
+	t.Run("run uninstall", testUninstallAuditUnenroll(t.Context(), fixture, info))
+}

testing/integration/groups.go

@@ -12,6 +12,9 @@ const (
 	// Default group.
 	Default = define.Default
 
+	// ECH group of tests. Used to test against a custom integration server.
+	ECH = "ech"
+
 	// Fleet group of tests. Used for testing Elastic Agent with Fleet.
 	Fleet = "fleet"