From 56fd3a2f8bb0b0ad6d85dd78bffb37d76d6df9e6 Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Wed, 10 Sep 2025 15:17:04 +0300 Subject: [PATCH 1/2] fix: ESS stack clean up steps (#9832) * fix: specify terraform env var everywhere * fix: proper env vars for ESS Stack clean up * fix: error when terraform fails to destroy ESS stack * fix: pass ess_region in terraform destroy (cherry picked from commit 6918b7b0793288cfe55f4017d71f8429b4abce24) # Conflicts: # .buildkite/bk.integration-fips.pipeline.yml --- .buildkite/bk.integration-fips.pipeline.yml | 168 ++++++++++++++++++++ .buildkite/bk.integration.pipeline.yml | 3 +- .buildkite/scripts/steps/ess.ps1 | 6 +- .buildkite/scripts/steps/ess.sh | 6 +- .buildkite/scripts/steps/ess_down.sh | 4 +- 5 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 .buildkite/bk.integration-fips.pipeline.yml diff --git a/.buildkite/bk.integration-fips.pipeline.yml b/.buildkite/bk.integration-fips.pipeline.yml new file mode 100644 index 00000000000..7ea7b186f3a --- /dev/null +++ b/.buildkite/bk.integration-fips.pipeline.yml @@ -0,0 +1,168 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +env: + ASDF_MAGE_VERSION: 1.14.0 + MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0" + + IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757120457" + IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1757120457" + IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1757120457" + ASDF_TERRAFORM_VERSION: 1.9.2 + +# This section is used to define the plugins that will be used in the pipeline. +# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins +common: + - vault_ec_key_staging_frh_gov: &vault_ec_key_staging_frh_gov + elastic/vault-secrets#v0.1.0: + path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov" + field: "apiKey" + env_var: "EC_API_KEY" + +steps: + - label: Start ESS stack for FIPS integration tests + key: integration-fips-ess + env: + FIPS: "true" + EC_ENDPOINT: "https://api.staging.elastic-gov.com" + ESS_REGION: "us-gov-east-1" + TF_VAR_deployment_template_id: "aws-general-purpose" + TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" + TF_VAR_docker_images_name_suffix: "-fips" + command: | + source .buildkite/scripts/steps/ess_start.sh + artifact_paths: + - test_infra/ess/*.tfstate + - test_infra/ess/*.lock.hcl + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + plugins: + - *vault_ec_key_staging_frh_gov + + - group: "fips:Stateful:Ubuntu" + key: integration-tests-ubuntu-fips + depends_on: + - integration-fips-ess + steps: + - label: "fips:x86_64:sudo-{{matrix.sudo}}:{{matrix.groups}}" + depends_on: + - packaging-ubuntu-x86-64-fips # Reuse artifacts produced in .buildkite/integration.pipeline.yml + env: + FIPS: "true" + TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" + TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" + command: | + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips' + .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}} + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "aws" + image: "${IMAGE_UBUNTU_X86_64_FIPS}" + instanceType: "m5.2xlarge" + plugins: + - *vault_ec_key_staging_frh_gov + matrix: + setup: + sudo: + - "false" + - "true" + groups: + - fleet # currently there is only a single test in the fleet group, add more tests once they have been defined + + - label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}" + depends_on: + - packaging-ubuntu-arm64-fips + env: + FIPS: "true" + TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" + TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" + command: | + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips' + .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}} + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "aws" + image: "${IMAGE_UBUNTU_ARM64_FIPS}" + instanceType: "m6g.2xlarge" + plugins: + - *vault_ec_key_staging_frh_gov + matrix: + setup: + sudo: + - "false" + - "true" + groups: + - fleet + + - label: "fips:upgrade-ech-deployment" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*(Testing:run:TestUpgradeIntegrationsServer).*/ + env: + FIPS: "true" + TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" + command: | + .buildkite/scripts/buildkite-integration-tests.sh ech-deployment false + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "aws" + image: "${IMAGE_UBUNTU_X86_64_FIPS}" + instanceType: "m5.2xlarge" + plugins: + - *vault_ec_key_staging_frh_gov + + - label: ESS FIPS stack cleanup + depends_on: + - integration-tests-ubuntu-fips + env: + FIPS: "true" + EC_ENDPOINT: "https://api.staging.elastic-gov.com" + ESS_REGION: "us-gov-east-1" + TF_VAR_deployment_template_id: "aws-general-purpose" + TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" + TF_VAR_docker_images_name_suffix: "-fips" + allow_dependency_failure: true + command: | + buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess" + ls -lah test_infra/ess + .buildkite/scripts/steps/ess_down.sh + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + plugins: + - *vault_ec_key_staging_frh_gov + + - label: Aggregate test reports + depends_on: + - integration-tests-ubuntu-fips + allow_dependency_failure: true + command: | + buildkite-agent artifact download "build/*.xml" . + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + soft_fail: + - exit_status: "*" + plugins: + - elastic/vault-secrets#v0.1.0: + path: "kv/ci-shared/platform-ingest/buildkite_analytics_token" + field: "token" + env_var: "BUILDKITE_ANALYTICS_TOKEN" + - test-collector#v1.11.0: + files: "build/*.xml" + format: "junit" + branches: "main" + debug: true diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index c3efe6265b2..d69aa9d7692 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -12,6 +12,7 @@ env: IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1757120457" IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757120457" IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1757120457" + ASDF_TERRAFORM_VERSION: 1.9.2 # This section is used to define the plugins that will be used in the pipeline. # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins @@ -49,8 +50,6 @@ steps: notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - ESS stack provision" - env: - ASDF_TERRAFORM_VERSION: 1.9.2 command: .buildkite/scripts/steps/ess_start.sh retry: automatic: diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1 index 6ab4dceb8e9..7722948273d 100644 --- a/.buildkite/scripts/steps/ess.ps1 +++ b/.buildkite/scripts/steps/ess.ps1 @@ -40,6 +40,9 @@ function ess_up { } function ess_down { + param ( + [string]$EssRegion = "gcp-us-west2" + ) $Workspace = & git rev-parse --show-toplevel $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/" $stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate" @@ -52,7 +55,8 @@ function ess_down { try { Push-Location -Path $TfDir & terraform init - & terraform destroy -auto-approve + & terraform destroy -auto-approve ` + -var="ess_region=$EssRegion" Pop-Location } catch { Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_" diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh index 13b0e9a9ced..a6cdbd4dd2a 100755 --- a/.buildkite/scripts/steps/ess.sh +++ b/.buildkite/scripts/steps/ess.sh @@ -39,13 +39,15 @@ function ess_up() { } function ess_down() { - echo "~~~ Tearing down the ESS Stack" + echo "~~~ Tearing down the ESS Stack" + local ESS_REGION=${1:-"gcp-us-west2"} local WORKSPACE=$(git rev-parse --show-toplevel) local TF_DIR="${WORKSPACE}/test_infra/ess/" pushd "${TF_DIR}" terraform init - terraform destroy -auto-approve + terraform destroy -auto-approve \ + -var="ess_region=${ESS_REGION}" popd } diff --git a/.buildkite/scripts/steps/ess_down.sh b/.buildkite/scripts/steps/ess_down.sh index 2de58a0c382..565d5fde845 100755 --- a/.buildkite/scripts/steps/ess_down.sh +++ b/.buildkite/scripts/steps/ess_down.sh @@ -3,4 +3,6 @@ set -euo pipefail source .buildkite/scripts/steps/ess.sh -ess_down || echo "Failed to stop ESS stack" >&2 +ESS_REGION="${ESS_REGION:-gcp-us-west2}" + +ess_down "$ESS_REGION" \ No newline at end of file From a21cd0aa9c2878d96822aa10f37e5207d96a734e Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Wed, 10 Sep 2025 15:21:51 +0300 Subject: [PATCH 2/2] fix: resolve conflicts --- .buildkite/bk.integration-fips.pipeline.yml | 168 -------------------- 1 file changed, 168 deletions(-) delete mode 100644 .buildkite/bk.integration-fips.pipeline.yml diff --git a/.buildkite/bk.integration-fips.pipeline.yml b/.buildkite/bk.integration-fips.pipeline.yml deleted file mode 100644 index 7ea7b186f3a..00000000000 --- a/.buildkite/bk.integration-fips.pipeline.yml +++ /dev/null @@ -1,168 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json - -env: - ASDF_MAGE_VERSION: 1.14.0 - MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0" - - IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757120457" - IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1757120457" - IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1757120457" - ASDF_TERRAFORM_VERSION: 1.9.2 - -# This section is used to define the plugins that will be used in the pipeline. -# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins -common: - - vault_ec_key_staging_frh_gov: &vault_ec_key_staging_frh_gov - elastic/vault-secrets#v0.1.0: - path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov" - field: "apiKey" - env_var: "EC_API_KEY" - -steps: - - label: Start ESS stack for FIPS integration tests - key: integration-fips-ess - env: - FIPS: "true" - EC_ENDPOINT: "https://api.staging.elastic-gov.com" - ESS_REGION: "us-gov-east-1" - TF_VAR_deployment_template_id: "aws-general-purpose" - TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" - TF_VAR_docker_images_name_suffix: "-fips" - command: | - source .buildkite/scripts/steps/ess_start.sh - artifact_paths: - - test_infra/ess/*.tfstate - - test_infra/ess/*.lock.hcl - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" - useCustomGlobalHooks: true - plugins: - - *vault_ec_key_staging_frh_gov - - - group: "fips:Stateful:Ubuntu" - key: integration-tests-ubuntu-fips - depends_on: - - integration-fips-ess - steps: - - label: "fips:x86_64:sudo-{{matrix.sudo}}:{{matrix.groups}}" - depends_on: - - packaging-ubuntu-x86-64-fips # Reuse artifacts produced in .buildkite/integration.pipeline.yml - env: - FIPS: "true" - TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" - TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" - command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips' - .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}} - artifact_paths: - - build/** - - build/diagnostics/** - retry: - automatic: - limit: 1 - agents: - provider: "aws" - image: "${IMAGE_UBUNTU_X86_64_FIPS}" - instanceType: "m5.2xlarge" - plugins: - - *vault_ec_key_staging_frh_gov - matrix: - setup: - sudo: - - "false" - - "true" - groups: - - fleet # currently there is only a single test in the fleet group, add more tests once they have been defined - - - label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}" - depends_on: - - packaging-ubuntu-arm64-fips - env: - FIPS: "true" - TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" - TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" - command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips' - .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}} - artifact_paths: - - build/** - - build/diagnostics/** - retry: - automatic: - limit: 1 - agents: - provider: "aws" - image: "${IMAGE_UBUNTU_ARM64_FIPS}" - instanceType: "m6g.2xlarge" - plugins: - - *vault_ec_key_staging_frh_gov - matrix: - setup: - sudo: - - "false" - - "true" - groups: - - fleet - - - label: "fips:upgrade-ech-deployment" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*(Testing:run:TestUpgradeIntegrationsServer).*/ - env: - FIPS: "true" - TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" - command: | - .buildkite/scripts/buildkite-integration-tests.sh ech-deployment false - artifact_paths: - - build/** - - build/diagnostics/** - retry: - automatic: - limit: 1 - agents: - provider: "aws" - image: "${IMAGE_UBUNTU_X86_64_FIPS}" - instanceType: "m5.2xlarge" - plugins: - - *vault_ec_key_staging_frh_gov - - - label: ESS FIPS stack cleanup - depends_on: - - integration-tests-ubuntu-fips - env: - FIPS: "true" - EC_ENDPOINT: "https://api.staging.elastic-gov.com" - ESS_REGION: "us-gov-east-1" - TF_VAR_deployment_template_id: "aws-general-purpose" - TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}" - TF_VAR_docker_images_name_suffix: "-fips" - allow_dependency_failure: true - command: | - buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess" - ls -lah test_infra/ess - .buildkite/scripts/steps/ess_down.sh - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" - useCustomGlobalHooks: true - plugins: - - *vault_ec_key_staging_frh_gov - - - label: Aggregate test reports - depends_on: - - integration-tests-ubuntu-fips - allow_dependency_failure: true - command: | - buildkite-agent artifact download "build/*.xml" . - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" - useCustomGlobalHooks: true - soft_fail: - - exit_status: "*" - plugins: - - elastic/vault-secrets#v0.1.0: - path: "kv/ci-shared/platform-ingest/buildkite_analytics_token" - field: "token" - env_var: "BUILDKITE_ANALYTICS_TOKEN" - - test-collector#v1.11.0: - files: "build/*.xml" - format: "junit" - branches: "main" - debug: true