From 4376e659ae6bff3899beb8e5610ab80134aa589a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 13 Nov 2023 23:54:04 +0100 Subject: [PATCH 01/18] Sync k8s migtraion to buildkite --- .buildkite/pipeline.yml | 439 ++++++++++++++------------- .buildkite/scripts/install-gh.sh | 48 +++ .buildkite/scripts/install-kind.sh | 2 +- .buildkite/scripts/steps/sync-k8s.sh | 18 ++ 4 files changed, 291 insertions(+), 216 deletions(-) create mode 100644 .buildkite/scripts/install-gh.sh create mode 100644 .buildkite/scripts/steps/sync-k8s.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 489a3202ee3..b688c2bd64a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,230 +4,239 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: - - group: "Unit tests" - key: "unit-tests" - steps: - - label: "Unit tests - Ubuntu 22.04" - key: "unit-tests-2204" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - retry: - manual: - allowed: true + - label: "Sync K8s: Create Kibana PR" + key: "sync-k8s" + command: ".buildkite/scripts/steps/sync-k8s.sh" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true + # - group: "Unit tests" + # key: "unit-tests" + # steps: + # - label: "Unit tests - Ubuntu 22.04" + # key: "unit-tests-2204" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Ubuntu 22.04 ARM64" - key: "unit-tests-2204-arm64" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "aws" - imagePrefix: "core-ubuntu-2204-aarch64" - diskSizeGb: 200 - instanceType: "m6g.xlarge" - retry: - manual: - allowed: true + # - label: "Unit tests - Ubuntu 22.04 ARM64" + # key: "unit-tests-2204-arm64" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "aws" + # imagePrefix: "core-ubuntu-2204-aarch64" + # diskSizeGb: 200 + # instanceType: "m6g.xlarge" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Windows 2022" - key: "unit-tests-win2022" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-windows-2022" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - label: "Unit tests - Windows 2022" + # key: "unit-tests-win2022" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-windows-2022" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - - label: "Unit tests - Windows 2016" - key: "unit-tests-win2016" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-windows-2016" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - label: "Unit tests - Windows 2016" + # key: "unit-tests-win2016" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-windows-2016" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - # Runs inly on the main branch - - label: "Unit tests - MacOS 13 ARM" - key: "unit-tests-macos-13-arm" - command: ".buildkite/scripts/steps/unit-tests.sh" - branches: main - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: orka - imagePrefix: generic-13-ventura-arm - retry: - manual: - allowed: true + # # Runs inly on the main branch + # - label: "Unit tests - MacOS 13 ARM" + # key: "unit-tests-macos-13-arm" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # branches: main + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: orka + # imagePrefix: generic-13-ventura-arm + # retry: + # manual: + # allowed: true - - label: "Unit tests - MacOS 13" - key: "unit-tests-macos-13" - command: ".buildkite/scripts/steps/unit-tests.sh" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: orka - imagePrefix: generic-13-ventura-x64 - retry: - manual: - allowed: true + # - label: "Unit tests - MacOS 13" + # key: "unit-tests-macos-13" + # command: ".buildkite/scripts/steps/unit-tests.sh" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: orka + # imagePrefix: generic-13-ventura-x64 + # retry: + # manual: + # allowed: true - - group: "Desktop Windows tests" - key: "extended-windows" - steps: - - label: "Unit tests - Windows 10" - key: "unit-tests-win10" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - # TODO create own image - image: "family/endpoint-windows-10-tester-rel" - machine_type: "n2-standard-8" - disk_type: "pd-ssd" - retry: - manual: - allowed: true - - label: "Unit tests - Windows 11" - key: "unit-tests-win11" - command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - # TODO create own image - image: "family/endpoint-windows-11-tester-rel" - machine_type: "n2-standard-8" - disk_type: "pd-ssd" - retry: - manual: - allowed: true + # - group: "Desktop Windows tests" + # key: "extended-windows" + # steps: + # - label: "Unit tests - Windows 10" + # key: "unit-tests-win10" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # # TODO create own image + # image: "family/endpoint-windows-10-tester-rel" + # machine_type: "n2-standard-8" + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true + # - label: "Unit tests - Windows 11" + # key: "unit-tests-win11" + # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # # TODO create own image + # image: "family/endpoint-windows-11-tester-rel" + # machine_type: "n2-standard-8" + # disk_type: "pd-ssd" + # retry: + # manual: + # allowed: true - - label: "Merge coverage reports" - key: "merge-coverage" - env: - BUILDKITE_REPO: "" - command: " - .buildkite/scripts/steps/merge.sh - unit-tests-2204 - unit-tests-2204-arm64 - unit-tests-win2016 - unit-tests-win2022 - unit-tests-macos-13 - unit-tests-win10 - unit-tests-win11 - " - artifact_paths: - - "build/TEST-**" - agents: - image: "golang:1.20.10" - depends_on: - - unit-tests - - extended-windows - allow_dependency_failure: true + # - label: "Merge coverage reports" + # key: "merge-coverage" + # env: + # BUILDKITE_REPO: "" + # command: " + # .buildkite/scripts/steps/merge.sh + # unit-tests-2204 + # unit-tests-2204-arm64 + # unit-tests-win2016 + # unit-tests-win2022 + # unit-tests-macos-13 + # unit-tests-win10 + # unit-tests-win11 + # " + # artifact_paths: + # - "build/TEST-**" + # agents: + # image: "golang:1.20.10" + # depends_on: + # - unit-tests + # - extended-windows + # allow_dependency_failure: true - - group: "K8s tests" - key: "k8s-tests" - steps: - - label: "K8s tests: {{matrix.k8s_version}}" - env: - K8S_VERSION: "v{{matrix.k8s_version}}" - KIND_VERSION: "v0.20.0" - command: ".buildkite/scripts/steps/k8s-tests.sh" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - - "coverage.out" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - matrix: - setup: - k8s_version: - - "1.28.0" - - "1.27.3" - - "1.26.6" - retry: - manual: - allowed: true + # - group: "K8s tests" + # key: "k8s-tests" + # steps: + # - label: "K8s tests: {{matrix.k8s_version}}" + # env: + # K8S_VERSION: "v{{matrix.k8s_version}}" + # KIND_VERSION: "v0.20.0" + # command: ".buildkite/scripts/steps/k8s-tests.sh" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # - "coverage.out" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # matrix: + # setup: + # k8s_version: + # - "1.28.0" + # - "1.27.3" + # - "1.26.6" + # retry: + # manual: + # allowed: true - - label: ":sonarqube: Continuous Code Inspection" - env: - VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" - agents: - image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - command: - - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." - - "/scan-source-code.sh" - depends_on: - - "merge-coverage" - retry: - manual: - allowed: true + # - label: ":sonarqube: Continuous Code Inspection" + # env: + # VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + # agents: + # image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + # command: + # - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." + # - "/scan-source-code.sh" + # depends_on: + # - "merge-coverage" + # retry: + # manual: + # allowed: true - - label: "Serverless integration test" - key: "serverless-integration-tests" - env: - TEST_INTEG_AUTH_ESS_REGION: us-east-1 - command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "n1-standard-8" + # - label: "Serverless integration test" + # key: "serverless-integration-tests" + # env: + # TEST_INTEG_AUTH_ESS_REGION: us-east-1 + # command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # agents: + # provider: "gcp" + # machineType: "n1-standard-8" - - label: "Integration tests" - key: "integration-tests" - env: - TEST_INTEG_AUTH_ESS_REGION: us-east-1 - command: ".buildkite/scripts/steps/integration_tests.sh stateful" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "n1-standard-8" + # - label: "Integration tests" + # key: "integration-tests" + # env: + # TEST_INTEG_AUTH_ESS_REGION: us-east-1 + # command: ".buildkite/scripts/steps/integration_tests.sh stateful" + # artifact_paths: + # - "build/TEST-**" + # - "build/diagnostics/*" + # agents: + # provider: "gcp" + # machineType: "n1-standard-8" - - wait: ~ - continue_on_failure: true - - label: "Processing test results" - agents: - provider: "gcp" - plugins: - - junit-annotate#v2.4.1: - artifacts: build/TEST-go-integration*.xml + # - wait: ~ + # continue_on_failure: true + # - label: "Processing test results" + # agents: + # provider: "gcp" + # plugins: + # - junit-annotate#v2.4.1: + # artifacts: build/TEST-go-integration*.xml diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh new file mode 100644 index 00000000000..512c4cc2a2e --- /dev/null +++ b/.buildkite/scripts/install-gh.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "--- Install gh cli" + +MSG="environment variable missing." +DEFAULT_HOME="/usr/local" +GH_VERSION=${GH_VERSION:?$MSG} +HOME=${HOME:?$DEFAULT_HOME} +GH_CMD="${HOME}/bin/gh" + +if command -v gh +then + set +e + echo "Found GH. Checking version.." + FOUND_GH_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') + if [ "$FOUND_GH_VERSION" == "$GH_VERSION" ] + then + echo "Versions match. No need to install gh. Exiting." + exit 0 + fi + set -e +fi + + +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') +if [ "${ARCH}" == "aarch64" ] ; then + ARCH_SUFFIX=arm64 +else + ARCH_SUFFIX=amd64 +fi + +echo "Downloading gh : ${GH_VERSION}..." +TMP_DIR=$(mktemp -d) +if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_${OS}_${ARCH}.tar.gz" ; then + mkdir -p "${HOME}/bin" + mv "${TMP_DIR}/gh_${GH_VERSION}_${OS}_${ARCH}/bin/gh" "${GH_CMD}" + rm -rf ${TMP_DIR} +else + echo "Something bad with the download, let's delete the corrupted binary" + if [ -e "${GH_CMD}" ] ; then + rm "${GH_CMD}" + fi + exit 1 +fi + + diff --git a/.buildkite/scripts/install-kind.sh b/.buildkite/scripts/install-kind.sh index 171480d7685..ee0e0039719 100644 --- a/.buildkite/scripts/install-kind.sh +++ b/.buildkite/scripts/install-kind.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -exuo pipefail +set -euo pipefail echo "--- Install Kind" diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh new file mode 100644 index 00000000000..302d48ba5a7 --- /dev/null +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +export PATH=$HOME/bin:${PATH} +source .buildkite/scripts/install-gh.sh + + +cd deploy/kubernetes + +echo "--- [File Creation] Create-Needed-Manifest" +WITHOUTCONFIG=true make generate-k8s +./creator_k8s_manifest.sh . + +echo "--- [Clone] Kibana-Repository" +make ci-clone-kibana-repository +cp Makefile ./kibana +cd kibana +make ci-create-kubernetes-templates-pull-request \ No newline at end of file From 721dcca6a3a8534fcaa6cc0bedb1a0b9da825707 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 00:02:03 +0100 Subject: [PATCH 02/18] Sync k8s migtraion to buildkite --- .buildkite/pipeline.yml | 2 ++ .buildkite/scripts/steps/sync-k8s.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b688c2bd64a..b9cdb4c7892 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -5,6 +5,8 @@ env: DOCKER_REGISTRY: "docker.elastic.co" steps: - label: "Sync K8s: Create Kibana PR" + env: + GH_VERSION: "2.4.0" key: "sync-k8s" command: ".buildkite/scripts/steps/sync-k8s.sh" agents: diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh index 302d48ba5a7..f66faa6bfd8 100644 --- a/.buildkite/scripts/steps/sync-k8s.sh +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -4,7 +4,6 @@ set -euo pipefail export PATH=$HOME/bin:${PATH} source .buildkite/scripts/install-gh.sh - cd deploy/kubernetes echo "--- [File Creation] Create-Needed-Manifest" @@ -15,4 +14,5 @@ echo "--- [Clone] Kibana-Repository" make ci-clone-kibana-repository cp Makefile ./kibana cd kibana +echo "--- Create Kibana PR" make ci-create-kubernetes-templates-pull-request \ No newline at end of file From 5167e358b66cb9e38859a5747577bf46d87025bf Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 00:07:47 +0100 Subject: [PATCH 03/18] Sync k8s migtraion to buildkite --- .buildkite/scripts/install-gh.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh index 512c4cc2a2e..6a0b0738db5 100644 --- a/.buildkite/scripts/install-gh.sh +++ b/.buildkite/scripts/install-gh.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash + +# Required environment variables: +# - GH_VERSION - the version of gh to install set -euo pipefail echo "--- Install gh cli" @@ -13,7 +16,7 @@ if command -v gh then set +e echo "Found GH. Checking version.." - FOUND_GH_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') + FOUND_GH_VERSION=$(gh --version 2>&1 >/dev/null | awk '{print $3}') if [ "$FOUND_GH_VERSION" == "$GH_VERSION" ] then echo "Versions match. No need to install gh. Exiting." @@ -22,7 +25,6 @@ then set -e fi - OS=$(uname -s| tr '[:upper:]' '[:lower:]') ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') if [ "${ARCH}" == "aarch64" ] ; then @@ -33,7 +35,7 @@ fi echo "Downloading gh : ${GH_VERSION}..." TMP_DIR=$(mktemp -d) -if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_${OS}_${ARCH}.tar.gz" ; then +if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_${OS}_${ARCH}.tar.gz" | tar xz -C $TMP_DIR ; then mkdir -p "${HOME}/bin" mv "${TMP_DIR}/gh_${GH_VERSION}_${OS}_${ARCH}/bin/gh" "${GH_CMD}" rm -rf ${TMP_DIR} From ea71fbd51c8b51d181d254343a0789b7ad4e39e7 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 00:15:18 +0100 Subject: [PATCH 04/18] Sync k8s migtraion to buildkite --- .buildkite/scripts/install-gh.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh index 6a0b0738db5..c6c0a30cf8d 100644 --- a/.buildkite/scripts/install-gh.sh +++ b/.buildkite/scripts/install-gh.sh @@ -2,7 +2,7 @@ # Required environment variables: # - GH_VERSION - the version of gh to install -set -euo pipefail +set -exuo pipefail echo "--- Install gh cli" @@ -19,8 +19,10 @@ then FOUND_GH_VERSION=$(gh --version 2>&1 >/dev/null | awk '{print $3}') if [ "$FOUND_GH_VERSION" == "$GH_VERSION" ] then - echo "Versions match. No need to install gh. Exiting." + echo "GH Versions match: $GH_VERSION. No need to install gh. Exiting." exit 0 + else + echo "GH VErsion mismatch. Desired version: $GH_VERSION, found version: $FOUND_GH_VERSION. Installing new version." fi set -e fi From 087e1799b62a8a2bf14239adf9db5db787f2f6e6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 00:23:11 +0100 Subject: [PATCH 05/18] Sync k8s migtraion to buildkite --- .buildkite/scripts/install-gh.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh index c6c0a30cf8d..56260f192b1 100644 --- a/.buildkite/scripts/install-gh.sh +++ b/.buildkite/scripts/install-gh.sh @@ -37,12 +37,12 @@ fi echo "Downloading gh : ${GH_VERSION}..." TMP_DIR=$(mktemp -d) -if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_${OS}_${ARCH}.tar.gz" | tar xz -C $TMP_DIR ; then +if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" | tar xz -C $TMP_DIR ; then mkdir -p "${HOME}/bin" - mv "${TMP_DIR}/gh_${GH_VERSION}_${OS}_${ARCH}/bin/gh" "${GH_CMD}" + mv "${TMP_DIR}/gh_${GH_VERSION}_linux_amd64/bin/gh" "${GH_CMD}" rm -rf ${TMP_DIR} -else - echo "Something bad with the download, let's delete the corrupted binary" +else + echo "Something bad with the download, deleting the binary" if [ -e "${GH_CMD}" ] ; then rm "${GH_CMD}" fi From c967144f53673e30ccd5e609e2faabb689afd791 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 12:32:33 +0100 Subject: [PATCH 06/18] Sync k8s migtraion to buildkite --- .buildkite/scripts/steps/sync-k8s.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh index f66faa6bfd8..dc7651c3699 100644 --- a/.buildkite/scripts/steps/sync-k8s.sh +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -4,6 +4,8 @@ set -euo pipefail export PATH=$HOME/bin:${PATH} source .buildkite/scripts/install-gh.sh +export GITHUB_TOKEN=$(retry 5 vault kv get -field token kv/ci-shared/platform-ingest/github_token) + cd deploy/kubernetes echo "--- [File Creation] Create-Needed-Manifest" From c49186ec18ae15d826f25e6b2167a0d6c746164a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 15:55:40 +0100 Subject: [PATCH 07/18] Sync k8s migtraion to buildkite --- .buildkite/scripts/steps/sync-k8s.sh | 2 +- deploy/kubernetes/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh index dc7651c3699..846c2c8e3f4 100644 --- a/.buildkite/scripts/steps/sync-k8s.sh +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -17,4 +17,4 @@ make ci-clone-kibana-repository cp Makefile ./kibana cd kibana echo "--- Create Kibana PR" -make ci-create-kubernetes-templates-pull-request \ No newline at end of file +GH_TOKEN=$GITHUB_TOKEN make ci-create-kubernetes-templates-pull-request \ No newline at end of file diff --git a/deploy/kubernetes/Makefile b/deploy/kubernetes/Makefile index 6247c9461e1..d02fc6eeb38 100644 --- a/deploy/kubernetes/Makefile +++ b/deploy/kubernetes/Makefile @@ -81,7 +81,7 @@ else echo "INFO: create pull request" @gh pr create \ --title "Update kubernetes templates for elastic-agent" \ - --body "Automated by ${BUILD_URL}" \ + --body "Automated by ${BUILDKITE_BUILD_URL}" \ --label automation \ --label release_note:skip \ --base main \ From 09bd4ad6217546b5e90314c181b9bbddfa35a3fe Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 14 Nov 2023 17:39:20 +0100 Subject: [PATCH 08/18] Fix sync-k8s.sh --- .buildkite/scripts/steps/sync-k8s.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh index 846c2c8e3f4..2c33ae7fe29 100644 --- a/.buildkite/scripts/steps/sync-k8s.sh +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -2,7 +2,9 @@ set -euo pipefail export PATH=$HOME/bin:${PATH} + source .buildkite/scripts/install-gh.sh +source .buildkite/scripts/common.sh export GITHUB_TOKEN=$(retry 5 vault kv get -field token kv/ci-shared/platform-ingest/github_token) From 32328fe1c1dc958456c33df4d2b71e2e5be46ba4 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:34:38 +0100 Subject: [PATCH 09/18] try monorepo-diff step --- .buildkite/pipeline.yml | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 900a9a07cac..513720fea33 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,17 +4,6 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: - - label: "Sync K8s: Create Kibana PR" - env: - GH_VERSION: "2.4.0" - key: "sync-k8s" - command: ".buildkite/scripts/steps/sync-k8s.sh" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - retry: - manual: - allowed: true # - group: "Unit tests" # key: "unit-tests" # steps: @@ -217,3 +206,31 @@ steps: # plugins: # - junit-annotate#v2.4.1: # artifacts: build/TEST-go-integration*.xml + + # - label: "Sync K8s: Create Kibana PR" + # env: + # GH_VERSION: "2.4.0" + # key: "sync-k8s" + # branches: "main" + # command: ".buildkite/scripts/steps/sync-k8s.sh" + # agents: + # provider: "gcp" + # image: "family/core-ubuntu-2204" + # retry: + # manual: + # allowed: true + + - label: "Trigger k8s sync" + plugins: + - monebag/monorepo-diff#v2.5.9: + diff: "git diff --name-only HEAD~1" + watch: + - path: deploy/kubernetes/* + config: + command: ".buildkite/scripts/steps/sync-k8s.sh" + branches: main + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + env: + - GH_VERSION: "2.4.0" From 7acf887ca4a15ccf2f53b73be165de16148bb16d Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:41:10 +0100 Subject: [PATCH 10/18] try monorepo-diff step --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 513720fea33..cdd9ec577a5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -233,4 +233,4 @@ steps: provider: "gcp" image: "family/core-ubuntu-2204" env: - - GH_VERSION: "2.4.0" + - GH_VERSION=2.4.0 From dc2d17010a47ce1ed7b4c9c4dd4475da63e14783 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:45:38 +0100 Subject: [PATCH 11/18] try monorepo-diff step --- .buildkite/pipeline.yml | 4 +++- .buildkite/scripts/steps/sync-k8s.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index cdd9ec577a5..1924d8ef623 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -225,7 +225,8 @@ steps: - monebag/monorepo-diff#v2.5.9: diff: "git diff --name-only HEAD~1" watch: - - path: deploy/kubernetes/* + # - path: deploy/kubernetes/* + - path: .buildkite/* config: command: ".buildkite/scripts/steps/sync-k8s.sh" branches: main @@ -234,3 +235,4 @@ steps: image: "family/core-ubuntu-2204" env: - GH_VERSION=2.4.0 + - DRY_RUN=true diff --git a/.buildkite/scripts/steps/sync-k8s.sh b/.buildkite/scripts/steps/sync-k8s.sh index 2c33ae7fe29..ecd64e815a4 100644 --- a/.buildkite/scripts/steps/sync-k8s.sh +++ b/.buildkite/scripts/steps/sync-k8s.sh @@ -19,4 +19,4 @@ make ci-clone-kibana-repository cp Makefile ./kibana cd kibana echo "--- Create Kibana PR" -GH_TOKEN=$GITHUB_TOKEN make ci-create-kubernetes-templates-pull-request \ No newline at end of file +make ci-create-kubernetes-templates-pull-request \ No newline at end of file From ebfcf67f5257dd09448ea2bf86e589ae3248c858 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:50:29 +0100 Subject: [PATCH 12/18] try monorepo-diff step --- .buildkite/pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1924d8ef623..69242415237 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -220,7 +220,10 @@ steps: # manual: # allowed: true + # Triggers a dynamic pipeline of one step: Sync K8s + # Runs only on main and if k8s files are changed - label: "Trigger k8s sync" + branches: main plugins: - monebag/monorepo-diff#v2.5.9: diff: "git diff --name-only HEAD~1" @@ -228,8 +231,8 @@ steps: # - path: deploy/kubernetes/* - path: .buildkite/* config: + label: "Sync K8s" command: ".buildkite/scripts/steps/sync-k8s.sh" - branches: main agents: provider: "gcp" image: "family/core-ubuntu-2204" From 020a3824455a95a01830710db6e61d0302766cd6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:52:55 +0100 Subject: [PATCH 13/18] try monorepo-diff step --- .buildkite/pipeline.yml | 412 ++++++++++++++++++++-------------------- 1 file changed, 205 insertions(+), 207 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 69242415237..e23534af6fa 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,223 +4,223 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: - # - group: "Unit tests" - # key: "unit-tests" - # steps: - # - label: "Unit tests - Ubuntu 22.04" - # key: "unit-tests-2204" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # retry: - # manual: - # allowed: true + - group: "Unit tests" + key: "unit-tests" + steps: + - label: "Unit tests - Ubuntu 22.04" + key: "unit-tests-2204" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true - # - label: "Unit tests - Windows 2016" - # key: "unit-tests-win2016" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-windows-2016" - # machine_type: "n2-standard-8" - # disk_size: 200 - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true + - label: "Unit tests - Windows 2016" + key: "unit-tests-win2016" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2016" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true - # # Runs inly on the main branch - # - label: "Unit tests - MacOS 13 ARM" - # key: "unit-tests-macos-13-arm" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # branches: main - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: orka - # imagePrefix: generic-13-ventura-arm - # retry: - # manual: - # allowed: true + # Runs inly on the main branch + - label: "Unit tests - MacOS 13 ARM" + key: "unit-tests-macos-13-arm" + command: ".buildkite/scripts/steps/unit-tests.sh" + branches: main + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-arm + retry: + manual: + allowed: true - # - label: "Unit tests - MacOS 13" - # key: "unit-tests-macos-13" - # command: ".buildkite/scripts/steps/unit-tests.sh" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: orka - # imagePrefix: generic-13-ventura-x64 - # retry: - # manual: - # allowed: true + - label: "Unit tests - MacOS 13" + key: "unit-tests-macos-13" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: orka + imagePrefix: generic-13-ventura-x64 + retry: + manual: + allowed: true - # - group: "Desktop Windows tests" - # key: "extended-windows" - # steps: - # - label: "Unit tests - Windows 10" - # key: "unit-tests-win10" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # # TODO create own image - # image: "family/endpoint-windows-10-tester-rel" - # machine_type: "n2-standard-8" - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true - # - label: "Unit tests - Windows 11" - # key: "unit-tests-win11" - # command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # # TODO create own image - # image: "family/endpoint-windows-11-tester-rel" - # machine_type: "n2-standard-8" - # disk_type: "pd-ssd" - # retry: - # manual: - # allowed: true + - group: "Desktop Windows tests" + key: "extended-windows" + steps: + - label: "Unit tests - Windows 10" + key: "unit-tests-win10" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + # TODO create own image + image: "family/endpoint-windows-10-tester-rel" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + retry: + manual: + allowed: true + - label: "Unit tests - Windows 11" + key: "unit-tests-win11" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + # TODO create own image + image: "family/endpoint-windows-11-tester-rel" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + retry: + manual: + allowed: true - # - label: "Merge coverage reports" - # key: "merge-coverage" - # env: - # BUILDKITE_REPO: "" - # command: " - # .buildkite/scripts/steps/merge.sh - # unit-tests-2204 - # unit-tests-2204-arm64 - # unit-tests-win2016 - # unit-tests-win2022 - # unit-tests-macos-13 - # unit-tests-win10 - # unit-tests-win11 - # " - # artifact_paths: - # - "build/TEST-**" - # agents: - # image: "golang:1.20.10" - # depends_on: - # - unit-tests - # - extended-windows - # allow_dependency_failure: true + - label: "Merge coverage reports" + key: "merge-coverage" + env: + BUILDKITE_REPO: "" + command: " + .buildkite/scripts/steps/merge.sh + unit-tests-2204 + unit-tests-2204-arm64 + unit-tests-win2016 + unit-tests-win2022 + unit-tests-macos-13 + unit-tests-win10 + unit-tests-win11 + " + artifact_paths: + - "build/TEST-**" + agents: + image: "golang:1.20.10" + depends_on: + - unit-tests + - extended-windows + allow_dependency_failure: true - # - group: "K8s tests" - # key: "k8s-tests" - # steps: - # - label: "K8s tests: {{matrix.k8s_version}}" - # env: - # K8S_VERSION: "v{{matrix.k8s_version}}" - # KIND_VERSION: "v0.20.0" - # command: ".buildkite/scripts/steps/k8s-tests.sh" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # - "coverage.out" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # matrix: - # setup: - # k8s_version: - # - "1.28.0" - # - "1.27.3" - # - "1.26.6" - # retry: - # manual: - # allowed: true + - group: "K8s tests" + key: "k8s-tests" + steps: + - label: "K8s tests: {{matrix.k8s_version}}" + env: + K8S_VERSION: "v{{matrix.k8s_version}}" + KIND_VERSION: "v0.20.0" + command: ".buildkite/scripts/steps/k8s-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + matrix: + setup: + k8s_version: + - "1.28.0" + - "1.27.3" + - "1.26.6" + retry: + manual: + allowed: true - # - label: ":sonarqube: Continuous Code Inspection" - # env: - # VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" - # agents: - # image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - # command: - # - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." - # - "/scan-source-code.sh" - # depends_on: - # - "merge-coverage" - # retry: - # manual: - # allowed: true + - label: ":sonarqube: Continuous Code Inspection" + env: + VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + agents: + image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + command: + - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ." + - "/scan-source-code.sh" + depends_on: + - "merge-coverage" + retry: + manual: + allowed: true - # - label: "Serverless integration test" - # key: "serverless-integration-tests" - # command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # agents: - # provider: "gcp" - # machineType: "n1-standard-8" + - label: "Serverless integration test" + key: "serverless-integration-tests" + command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "n1-standard-8" - # - label: "Integration tests" - # key: "integration-tests" - # command: ".buildkite/scripts/steps/integration_tests.sh stateful" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # agents: - # provider: "gcp" - # machineType: "n1-standard-8" + - label: "Integration tests" + key: "integration-tests" + command: ".buildkite/scripts/steps/integration_tests.sh stateful" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "n1-standard-8" - # - label: "Integration tests" - # key: "integration-tests" - # env: - # TEST_INTEG_AUTH_ESS_REGION: us-east-1 - # command: ".buildkite/scripts/steps/integration_tests.sh stateful" - # artifact_paths: - # - "build/TEST-**" - # - "build/diagnostics/*" - # agents: - # provider: "gcp" - # machineType: "n1-standard-8" + - label: "Integration tests" + key: "integration-tests" + env: + TEST_INTEG_AUTH_ESS_REGION: us-east-1 + command: ".buildkite/scripts/steps/integration_tests.sh stateful" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + machineType: "n1-standard-8" - # - wait: ~ - # continue_on_failure: true - # - label: "Processing test results" - # agents: - # provider: "gcp" - # plugins: - # - junit-annotate#v2.4.1: - # artifacts: build/TEST-go-integration*.xml + - wait: ~ + continue_on_failure: true + - label: "Processing test results" + agents: + provider: "gcp" + plugins: + - junit-annotate#v2.4.1: + artifacts: build/TEST-go-integration*.xml - # - label: "Sync K8s: Create Kibana PR" - # env: - # GH_VERSION: "2.4.0" - # key: "sync-k8s" - # branches: "main" - # command: ".buildkite/scripts/steps/sync-k8s.sh" - # agents: - # provider: "gcp" - # image: "family/core-ubuntu-2204" - # retry: - # manual: - # allowed: true + - label: "Sync K8s: Create Kibana PR" + env: + GH_VERSION: "2.4.0" + key: "sync-k8s" + branches: "main" + command: ".buildkite/scripts/steps/sync-k8s.sh" + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + retry: + manual: + allowed: true - # Triggers a dynamic pipeline of one step: Sync K8s + # Triggers a dynamic step: Sync K8s # Runs only on main and if k8s files are changed - label: "Trigger k8s sync" branches: main @@ -228,8 +228,7 @@ steps: - monebag/monorepo-diff#v2.5.9: diff: "git diff --name-only HEAD~1" watch: - # - path: deploy/kubernetes/* - - path: .buildkite/* + - path: deploy/kubernetes/* config: label: "Sync K8s" command: ".buildkite/scripts/steps/sync-k8s.sh" @@ -238,4 +237,3 @@ steps: image: "family/core-ubuntu-2204" env: - GH_VERSION=2.4.0 - - DRY_RUN=true From b6618156ac7810cc94f644e4cb06032f35c075e4 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 01:59:26 +0100 Subject: [PATCH 14/18] Cleanup --- .buildkite/pipeline.yml | 43 +++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e23534af6fa..ee3cdae5b24 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -21,6 +21,39 @@ steps: manual: allowed: true + - label: "Unit tests - Ubuntu 22.04 ARM64" + key: "unit-tests-2204-arm64" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "aws" + imagePrefix: "core-ubuntu-2204-aarch64" + diskSizeGb: 200 + instanceType: "m6g.xlarge" + retry: + manual: + allowed: true + + - label: "Unit tests - Windows 2022" + key: "unit-tests-win2022" + command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + - "coverage.out" + agents: + provider: "gcp" + image: "family/core-windows-2022" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + retry: + manual: + allowed: true + - label: "Unit tests - Windows 2016" key: "unit-tests-win2016" command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1" @@ -176,16 +209,6 @@ steps: provider: "gcp" machineType: "n1-standard-8" - - label: "Integration tests" - key: "integration-tests" - command: ".buildkite/scripts/steps/integration_tests.sh stateful" - artifact_paths: - - "build/TEST-**" - - "build/diagnostics/*" - agents: - provider: "gcp" - machineType: "n1-standard-8" - - label: "Integration tests" key: "integration-tests" env: From ff1bfb1f60feba5f0a6cba8f85482372ddd74772 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 02:01:21 +0100 Subject: [PATCH 15/18] Cleanup --- .buildkite/pipeline.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ee3cdae5b24..96bc567f269 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -211,8 +211,6 @@ steps: - label: "Integration tests" key: "integration-tests" - env: - TEST_INTEG_AUTH_ESS_REGION: us-east-1 command: ".buildkite/scripts/steps/integration_tests.sh stateful" artifact_paths: - "build/TEST-**" @@ -230,19 +228,6 @@ steps: - junit-annotate#v2.4.1: artifacts: build/TEST-go-integration*.xml - - label: "Sync K8s: Create Kibana PR" - env: - GH_VERSION: "2.4.0" - key: "sync-k8s" - branches: "main" - command: ".buildkite/scripts/steps/sync-k8s.sh" - agents: - provider: "gcp" - image: "family/core-ubuntu-2204" - retry: - manual: - allowed: true - # Triggers a dynamic step: Sync K8s # Runs only on main and if k8s files are changed - label: "Trigger k8s sync" From 2f41c2ce2372531d87bb1827df27cae1c36ff0e3 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 15 Nov 2023 02:23:17 +0100 Subject: [PATCH 16/18] Cleanup --- .buildkite/pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 96bc567f269..6c47d08bf92 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -236,7 +236,9 @@ steps: - monebag/monorepo-diff#v2.5.9: diff: "git diff --name-only HEAD~1" watch: - - path: deploy/kubernetes/* + - path: + - deploy/kubernetes/* + - version/docs/version.asciidoc config: label: "Sync K8s" command: ".buildkite/scripts/steps/sync-k8s.sh" From ac0168ab38226620b9d4f35440a99c74a36abb3d Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 16 Nov 2023 14:13:28 +0100 Subject: [PATCH 17/18] Added retry fro install-gh.sh. Cleanup --- .buildkite/scripts/install-gh.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh index 56260f192b1..276209ddd5c 100644 --- a/.buildkite/scripts/install-gh.sh +++ b/.buildkite/scripts/install-gh.sh @@ -8,8 +8,8 @@ echo "--- Install gh cli" MSG="environment variable missing." DEFAULT_HOME="/usr/local" -GH_VERSION=${GH_VERSION:?$MSG} -HOME=${HOME:?$DEFAULT_HOME} +GH_VERSION=${GH_VERSION:-$MSG} +HOME=${HOME:-$DEFAULT_HOME} GH_CMD="${HOME}/bin/gh" if command -v gh @@ -27,6 +27,8 @@ then set -e fi +source .buildkite/scripts/common.sh + OS=$(uname -s| tr '[:upper:]' '[:lower:]') ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') if [ "${ARCH}" == "aarch64" ] ; then @@ -37,10 +39,10 @@ fi echo "Downloading gh : ${GH_VERSION}..." TMP_DIR=$(mktemp -d) -if curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" | tar xz -C $TMP_DIR ; then +if retry 5 curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" | tar xz -C $TMP_DIR ; then mkdir -p "${HOME}/bin" mv "${TMP_DIR}/gh_${GH_VERSION}_linux_amd64/bin/gh" "${GH_CMD}" - rm -rf ${TMP_DIR} + rm -rf "${TMP_DIR}" else echo "Something bad with the download, deleting the binary" if [ -e "${GH_CMD}" ] ; then From e0605aa944fa6c7f5946d304f0740f9010de4e4e Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 20 Nov 2023 15:20:42 +0100 Subject: [PATCH 18/18] Update .buildkite/scripts/install-gh.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paolo Chilà --- .buildkite/scripts/install-gh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/install-gh.sh b/.buildkite/scripts/install-gh.sh index 276209ddd5c..ff52687d02f 100644 --- a/.buildkite/scripts/install-gh.sh +++ b/.buildkite/scripts/install-gh.sh @@ -22,7 +22,7 @@ then echo "GH Versions match: $GH_VERSION. No need to install gh. Exiting." exit 0 else - echo "GH VErsion mismatch. Desired version: $GH_VERSION, found version: $FOUND_GH_VERSION. Installing new version." + echo "GH Version mismatch. Desired version: $GH_VERSION, found version: $FOUND_GH_VERSION. Installing new version." fi set -e fi