diff --git a/.buildkite/bk.integration-fips.pipeline.yml b/.buildkite/bk.integration-fips.pipeline.yml index a2cc4fd69e6..8648b4997b0 100644 --- a/.buildkite/bk.integration-fips.pipeline.yml +++ b/.buildkite/bk.integration-fips.pipeline.yml @@ -18,31 +18,8 @@ common: env_var: "EC_API_KEY" steps: - - label: Build and push custom elastic-agent image - depends_on: - - 'packaging-containers-x86-64-fips' # Reuse artifacts produced in .buildkite/integration.pipeline.yml - key: integration-fips-cloud-image - env: - FIPS: "true" - CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}" - CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips" - TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" - command: | - buildkite-agent artifact download build/distributions/elastic-agent-cloud-fips-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips' - mage cloud:load - mage cloud:push - agents: - provider: "gcp" - machineType: "n1-standard-8" - image: "${IMAGE_UBUNTU_2404_X86_64}" - plugins: - - elastic/vault-docker-login#v0.5.2: - secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry' - - label: Start ESS stack for FIPS integration tests key: integration-fips-ess - depends_on: - - integration-fips-cloud-image env: ASDF_TERRAFORM_VERSION: 1.9.2 TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" @@ -55,7 +32,7 @@ steps: image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" useCustomGlobalHooks: true plugins: - - *vault_ec_key_prod + - *vault_ec_key_prod - group: "fips:Stateful:Ubuntu" key: integration-tests-ubuntu-fips @@ -83,7 +60,7 @@ steps: image: "${IMAGE_UBUNTU_X86_64_FIPS}" instanceType: "m5.2xlarge" plugins: - - *vault_ec_key_prod + - *vault_ec_key_prod matrix: setup: sudo: @@ -113,7 +90,7 @@ steps: image: "${IMAGE_UBUNTU_ARM64_FIPS}" instanceType: "m6g.2xlarge" plugins: - - *vault_ec_key_prod + - *vault_ec_key_prod matrix: setup: sudo: @@ -153,7 +130,7 @@ steps: image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" useCustomGlobalHooks: true plugins: - - *vault_ec_key_prod + - *vault_ec_key_prod - label: Aggregate test reports depends_on: diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 290c2931483..71c11427999 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -41,9 +41,34 @@ common: elastic/vault-secrets#v0.1.0: path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod" field: "apiKey" - env_var: "EC_API_KEY" + env_var: "EC_API_KEY" steps: + - label: "Custom ECH Testing" + key: integration-tests-ech + depends_on: + - packaging-ubuntu-x86-64 + env: + TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" + TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud:git-${BUILDKITE_COMMIT:0:12}" + FORCE_ESS_CREATE: "true" + command: | + #!/usr/bin/env bash + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' + .buildkite/scripts/steps/integration_tests_tf.sh ech true + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "gcp" + machineType: "n2-standard-8" + image: "${IMAGE_UBUNTU_2404_X86_64}" + plugins: + - *vault_ec_key_prod + - label: Start ESS stack for integration tests key: integration-ess notify: @@ -113,7 +138,7 @@ steps: image: "${IMAGE_WIN_2025}" plugins: - *vault_ec_key_prod - + - label: "Ubuntu:2404:amd64:sudo" depends_on: packaging-ubuntu-x86-64 env: @@ -298,7 +323,7 @@ steps: machineType: "n2-standard-8" image: "${IMAGE_UBUNTU_2404_X86_64}" plugins: - - *vault_ec_key_prod + - *vault_ec_key_prod matrix: - default - upgrade @@ -614,6 +639,7 @@ steps: # Warning: The key has a hook in pre-command key: aggregate-reports depends_on: + - integration-tests-ech - integration-tests-ubuntu - integration-tests-win - integration-tests-rhel8 diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 68283a2f4c3..59f44efaafb 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -3,6 +3,11 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" +common: + - vault_docker_login: &vault_docker_login + elastic/vault-docker-login#v0.5.2: + secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry' + steps: - group: "Integration tests: packaging" key: "int-packaging" @@ -97,13 +102,21 @@ steps: env: PACKAGES: "docker" PLATFORMS: "linux/amd64" - command: ".buildkite/scripts/steps/integration-package.sh" + CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud" + command: | + source .buildkite/scripts/common.sh + .buildkite/scripts/steps/integration-package.sh + echo "~~~ Pushing cloud image" + mage cloud:push artifact_paths: - build/distributions/** agents: provider: "gcp" machineType: "n2-standard-8" diskSizeGb: 200 + plugins: + - *vault_docker_login - label: "Packaging: Containers linux/arm64" key: packaging-containers-arm64 @@ -126,13 +139,21 @@ steps: PACKAGES: "docker" PLATFORMS: "linux/amd64" FIPS: "true" - command: ".buildkite/scripts/steps/integration-package.sh" + CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips" + command: | + source .buildkite/scripts/common.sh + .buildkite/scripts/steps/integration-package.sh + echo "~~~ Pushing cloud image" + mage cloud:push artifact_paths: - build/distributions/** agents: provider: "gcp" machineType: "n2-standard-8" diskSizeGb: 200 + plugins: + - *vault_docker_login - label: "Packaging: Containers linux/arm64 FIPS" key: packaging-containers-arm64-fips diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index 2e4284b500e..ec196e4bd97 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -32,7 +32,7 @@ mage build:testBinaries # If the step is retried, we start the stack again. # BUILDKITE_RETRY_COUNT == "0" for the first run # BUILDKITE_RETRY_COUNT > 0 for the retries -if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then +if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 || "${FORCE_ESS_CREATE:-false}" == "true" ]]; then echo "~~~ The steps is retried, starting the ESS stack again" trap 'ess_down' EXIT ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack") diff --git a/magefile.go b/magefile.go index 820bc58e06a..6eb4c927e50 100644 --- a/magefile.go +++ b/magefile.go @@ -991,14 +991,11 @@ func (Cloud) Load() error { // Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant fips := os.Getenv(fipsEnv) - defer os.Setenv(fipsEnv, fips) fipsVal, err := strconv.ParseBool(fips) if err != nil { fipsVal = false } - if err := os.Setenv(fipsEnv, strconv.FormatBool(fipsVal)); err != nil { - return fmt.Errorf("failed to set fips env var: %w", err) - } + devtools.FIPSBuild = fipsVal source := "build/distributions/elastic-agent-cloud-" + agentVersion + "-SNAPSHOT-linux-" + runtime.GOARCH + ".docker.tar.gz" diff --git a/testing/integration/ess/ech_test.go b/testing/integration/ess/ech_test.go new file mode 100644 index 00000000000..b6f536996c2 --- /dev/null +++ b/testing/integration/ess/ech_test.go @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package ess + +import ( + "encoding/json" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/gofrs/uuid/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent/pkg/control/v2/cproto" + atesting "github.com/elastic/elastic-agent/pkg/testing" + "github.com/elastic/elastic-agent/pkg/testing/define" + "github.com/elastic/elastic-agent/pkg/testing/tools/fleettools" + "github.com/elastic/elastic-agent/testing/integration" +) + +func TestECH(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: integration.ECH, + Stack: &define.Stack{}, + Sudo: true, + Local: false, + OS: []define.OS{ + { + Type: define.Linux, + }, + }, + }) + + // Check that the Fleet Server in the deployment is healthy + fleetServerHost, err := fleettools.DefaultURL(t.Context(), info.KibanaClient) + statusUrl, err := url.JoinPath(fleetServerHost, "/api/status") + require.NoError(t, err) + + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp, err := http.Get(statusUrl) + require.NoError(c, err) + defer resp.Body.Close() + + require.Equal(c, http.StatusOK, resp.StatusCode) + + var body struct { + Name string `json:"name"` + Status string `json:"status"` + } + err = json.NewDecoder(resp.Body).Decode(&body) + require.NoError(c, err) + + t.Logf("body.status = %s", body.Status) + require.Equal(c, "HEALTHY", body.Status) + }, 5*time.Minute, 10*time.Second, "Fleet Server in ECH deployment is not healthy") + + // Create a policy and install an agent + policyUUID := uuid.Must(uuid.NewV4()).String() + policy := kibana.AgentPolicy{ + Name: "testloglevel-policy-" + policyUUID, + Namespace: "default", + Description: "Test Log Level Policy " + policyUUID, + MonitoringEnabled: []kibana.MonitoringEnabledOption{}, + } + t.Log("Creating Agent policy...") + policyResp, err := info.KibanaClient.CreatePolicy(t.Context(), policy) + require.NoError(t, err, "failed creating policy") + + t.Log("Creating Agent enrollment API key...") + createEnrollmentApiKeyReq := kibana.CreateEnrollmentAPIKeyRequest{ + PolicyID: policyResp.ID, + } + enrollmentToken, err := info.KibanaClient.CreateEnrollmentAPIKey(t.Context(), createEnrollmentApiKeyReq) + require.NoError(t, err, "failed creating enrollment API key") + t.Logf("Created policy %+v", policyResp.AgentPolicy) + + fixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + err = fixture.Prepare(t.Context()) + require.NoError(t, err) + + opts := &atesting.InstallOpts{ + Force: true, + Privileged: true, + EnrollOpts: atesting.EnrollOpts{ + URL: fleetServerHost, + EnrollmentToken: enrollmentToken.APIKey, + }, + } + out, err := fixture.Install(t.Context(), opts) + if err != nil { + t.Logf("install output: %s", out) + require.NoError(t, err) + } + + var agentID string + require.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := fixture.ExecStatus(t.Context()) + require.NoError(c, err) + statusBuffer := new(strings.Builder) + err = json.NewEncoder(statusBuffer).Encode(status) + require.NoError(c, err) + t.Logf("agent status: %v", statusBuffer.String()) + + require.Equal(c, int(cproto.State_HEALTHY), status.State, "agent state is not healthy") + require.Equal(c, int(cproto.State_HEALTHY), status.FleetState, "agent's fleet-server state is not healthy") + agentID = status.Info.ID + }, time.Minute, time.Second, "agent never became healthy or connected to Fleet") + + require.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := fleettools.GetAgentStatus(t.Context(), info.KibanaClient, agentID) + require.NoError(c, err) + require.Equal(c, "online", status) + }, time.Minute, time.Second, "agent does not show as online in fleet") + + t.Run("run uninstall", testUninstallAuditUnenroll(t.Context(), fixture, info)) +} diff --git a/testing/integration/groups.go b/testing/integration/groups.go index c40918995e5..acc64159ad0 100644 --- a/testing/integration/groups.go +++ b/testing/integration/groups.go @@ -12,6 +12,9 @@ const ( // Default group. Default = define.Default + // ECH group of tests. Used to test against a custom integration server. + ECH = "ech" + // Fleet group of tests. Used for testing Elastic Agent with Fleet. Fleet = "fleet"