From 8b74368bce22122db852c29372a6b7c17417cb8e Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Thu, 29 May 2025 12:44:18 +0200 Subject: [PATCH 01/10] Create get-bc-commit.sh --- .buildkite/scripts/get-bc-commit.sh | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .buildkite/scripts/get-bc-commit.sh diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh new file mode 100644 index 0000000000000..aa408c837442f --- /dev/null +++ b/.buildkite/scripts/get-bc-commit.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". +# + +set -euo pipefail + +#TODO: all current branches? Only the "last" one? (9.0 in this case?) +#source .buildkite/scripts/branches.sh +#for BRANCH in "${BRANCHES[@]}"; do + +BRANCH=9.0 + +MANIFEST="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | +jq ".releases[] | +select(.branch == \"$BRANCH\") | +select(.active_release == true) | +.build_candidates | +to_entries | +sort_by(.value.completed_at) | +last | +.value.manifest_url")" + +BC_COMMIT_HASH="$(eval "curl -s $MANIFEST" | jq .projects.elasticsearch.commit_hash)" + +echo "steps: + - group: bc + steps: + - label: $BRANCH / bc-bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=9.1.0 -Dtests.bwc.refspec.main=$BC_COMMIT_HASH -Dtests.jvm.argline=\"-Des.serverless_transport=true\" + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + preemptible: true + retry: + automatic: + - exit_status: \"-1\" + limit: 3 + signal_reason: none + - signal_reason: agent_stop + limit: 3 +" From 049ec16cdd6096249cd352c7cb07420a62505e7d Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Fri, 30 May 2025 13:06:49 +0200 Subject: [PATCH 02/10] exit without pipeline upload for empty/failure; consider snapshots too --- .buildkite/scripts/get-bc-commit.sh | 75 +++++++++++++++-------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh index aa408c837442f..e8f1131d6c3f9 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/get-bc-commit.sh @@ -11,41 +11,46 @@ set -euo pipefail -#TODO: all current branches? Only the "last" one? (9.0 in this case?) -#source .buildkite/scripts/branches.sh -#for BRANCH in "${BRANCHES[@]}"; do - -BRANCH=9.0 - +# Select the most recent build from the current branch. +# We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last. +# So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot. MANIFEST="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | jq ".releases[] | -select(.branch == \"$BRANCH\") | +select(.branch == \"$BUILDKITE_BRANCH\") | select(.active_release == true) | -.build_candidates | -to_entries | -sort_by(.value.completed_at) | -last | -.value.manifest_url")" - -BC_COMMIT_HASH="$(eval "curl -s $MANIFEST" | jq .projects.elasticsearch.commit_hash)" - -echo "steps: - - group: bc - steps: - - label: $BRANCH / bc-bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=9.1.0 -Dtests.bwc.refspec.main=$BC_COMMIT_HASH -Dtests.jvm.argline=\"-Des.serverless_transport=true\" - timeout_in_minutes: 300 - agents: - provider: gcp - image: family/elasticsearch-ubuntu-2004 - machineType: n1-standard-32 - buildDirectory: /dev/shm/bk - preemptible: true - retry: - automatic: - - exit_status: \"-1\" - limit: 3 - signal_reason: none - - signal_reason: agent_stop - limit: 3 -" +((.snapshots | to_entries | sort_by(.value.completed_at)) + +(.build_candidates | to_entries | sort_by(.value.completed_at))) | +last | .value.manifest_url")" + +if [[ -z "$MANIFEST" ]]; then + echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]" + exit 0 +fi + + +TARGET_COMMIT_HASH="$(eval "curl -s $MANIFEST" | jq .projects.elasticsearch.commit_hash)" + +if [[ -z "$TARGET_COMMIT_HASH" ]]; then + echo "Cannot get the elasticsearch commit hash from [$MANIFEST]" + exit 1 +fi + +cat < Date: Fri, 30 May 2025 15:18:36 +0200 Subject: [PATCH 03/10] add log --- .buildkite/scripts/get-bc-commit.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh index e8f1131d6c3f9..328e5e9f436e7 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/get-bc-commit.sh @@ -35,6 +35,8 @@ if [[ -z "$TARGET_COMMIT_HASH" ]]; then exit 1 fi +echo "Running bc-bwc tests on commit [$TARGET_COMMIT_HASH]" + cat < Date: Fri, 30 May 2025 16:07:23 +0200 Subject: [PATCH 04/10] extract version from manifest --- .buildkite/scripts/get-bc-commit.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh index 328e5e9f436e7..e7c1a50f84dbc 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/get-bc-commit.sh @@ -14,7 +14,7 @@ set -euo pipefail # Select the most recent build from the current branch. # We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last. # So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot. -MANIFEST="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | +MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | jq ".releases[] | select(.branch == \"$BUILDKITE_BRANCH\") | select(.active_release == true) | @@ -22,25 +22,26 @@ select(.active_release == true) | (.build_candidates | to_entries | sort_by(.value.completed_at))) | last | .value.manifest_url")" -if [[ -z "$MANIFEST" ]]; then +if [[ -z "$MANIFEST_URL" ]]; then echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]" exit 0 fi - -TARGET_COMMIT_HASH="$(eval "curl -s $MANIFEST" | jq .projects.elasticsearch.commit_hash)" - -if [[ -z "$TARGET_COMMIT_HASH" ]]; then - echo "Cannot get the elasticsearch commit hash from [$MANIFEST]" +MANIFEST="$(eval "curl -s $MANIFEST_URL")" +if [[ -z "$MANIFEST" ]]; then + echo "Cannot get the build manifest from [$MANIFEST_URL]" exit 1 fi -echo "Running bc-bwc tests on commit [$TARGET_COMMIT_HASH]" +TARGET_COMMIT_HASH=$(echo "$MANIFEST" | jq .projects.elasticsearch.commit_hash) +TARGET_VERSION=$(echo "$MANIFEST" | jq .version) + +echo "Running bc-bwc tests on [$TARGET_VERSION] commit [$TARGET_COMMIT_HASH]" cat < Date: Mon, 2 Jun 2025 11:41:51 +0200 Subject: [PATCH 05/10] pr feedback --- .buildkite/scripts/get-bc-commit.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh index e7c1a50f84dbc..eb64cb8bcc827 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/get-bc-commit.sh @@ -23,7 +23,8 @@ select(.active_release == true) | last | .value.manifest_url")" if [[ -z "$MANIFEST_URL" ]]; then - echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]" + echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]." + echo "Skipping BC upgrade tests." exit 0 fi @@ -33,15 +34,23 @@ if [[ -z "$MANIFEST" ]]; then exit 1 fi -TARGET_COMMIT_HASH=$(echo "$MANIFEST" | jq .projects.elasticsearch.commit_hash) -TARGET_VERSION=$(echo "$MANIFEST" | jq .version) +CURRENT_VERSION = $(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties) -echo "Running bc-bwc tests on [$TARGET_VERSION] commit [$TARGET_COMMIT_HASH]" +BC_VERSION=$(echo "$MANIFEST" | jq .version) +BC_COMMIT_HASH=$(echo "$MANIFEST" | jq .projects.elasticsearch.commit_hash) + +if [ "$CURRENT_VERSION" != "$BC_VERSION" ]; then + echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]." + echo "Skipping BC upgrade tests." + exit 0 +fi + +echo "Running BC upgrade tests on version [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH]." cat < Date: Mon, 2 Jun 2025 16:47:08 +0200 Subject: [PATCH 06/10] fix whitespaces --- .buildkite/scripts/get-bc-commit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/get-bc-commit.sh index eb64cb8bcc827..1e1692a67145f 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/get-bc-commit.sh @@ -34,7 +34,7 @@ if [[ -z "$MANIFEST" ]]; then exit 1 fi -CURRENT_VERSION = $(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties) +CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties) BC_VERSION=$(echo "$MANIFEST" | jq .version) BC_COMMIT_HASH=$(echo "$MANIFEST" | jq .projects.elasticsearch.commit_hash) From becfaf2230a676515e4bd8116971083d829f5431 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 3 Jun 2025 14:10:37 +0200 Subject: [PATCH 07/10] integrate script into build step, various minor fixes --- .buildkite/pipelines/intake.template.yml | 5 +++++ .buildkite/pipelines/intake.yml | 5 +++++ ...get-bc-commit.sh => run-bc-upgrade-tests.sh} | 17 +++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) rename .buildkite/scripts/{get-bc-commit.sh => run-bc-upgrade-tests.sh} (78%) diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index 3aba186ed5abb..e2e809a2f971b 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -63,6 +63,11 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: "{{matrix.BWC_VERSION}}" + - label: bc-upgrade + command: ".buildkite/scripts/run-bc-upgrade-tests.sh" + agents: + image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5" + memory: "4G" - group: lucene-compat steps: - label: "{{matrix.LUCENE_VERSION}} / lucene-compat" diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 4adc8a1d31aa6..86fa4616d7292 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -64,6 +64,11 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: "{{matrix.BWC_VERSION}}" + - label: bc-upgrade + command: ".buildkite/scripts/run-bc-upgrade-tests.sh" + agents: + image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5" + memory: "4G" - group: lucene-compat steps: - label: "{{matrix.LUCENE_VERSION}} / lucene-compat" diff --git a/.buildkite/scripts/get-bc-commit.sh b/.buildkite/scripts/run-bc-upgrade-tests.sh similarity index 78% rename from .buildkite/scripts/get-bc-commit.sh rename to .buildkite/scripts/run-bc-upgrade-tests.sh index 1e1692a67145f..ff9d3e4fc75b7 100644 --- a/.buildkite/scripts/get-bc-commit.sh +++ b/.buildkite/scripts/run-bc-upgrade-tests.sh @@ -16,7 +16,7 @@ set -euo pipefail # So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot. MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | jq ".releases[] | -select(.branch == \"$BUILDKITE_BRANCH\") | +select(.branch == \"main\") | select(.active_release == true) | ((.snapshots | to_entries | sort_by(.value.completed_at)) + (.build_candidates | to_entries | sort_by(.value.completed_at))) | @@ -36,21 +36,22 @@ fi CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties) -BC_VERSION=$(echo "$MANIFEST" | jq .version) -BC_COMMIT_HASH=$(echo "$MANIFEST" | jq .projects.elasticsearch.commit_hash) +BC_VERSION=$(echo "$MANIFEST" | jq -r .version) +BC_BUILD_ID=$(echo "$MANIFEST" | jq -r .build_id) +BC_COMMIT_HASH=$(echo "$MANIFEST" | jq -r .projects.elasticsearch.commit_hash) -if [ "$CURRENT_VERSION" != "$BC_VERSION" ]; then +if [ "$CURRENT_VERSION-SNAPSHOT" != "$BC_VERSION" ]; then echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]." echo "Skipping BC upgrade tests." exit 0 fi -echo "Running BC upgrade tests on version [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH]." +echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH] with build id [$BC_BUILD_ID]." cat < $BUILDKITE_BRANCH + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true" timeout_in_minutes: 300 agents: provider: gcp @@ -60,7 +61,7 @@ steps: preemptible: true retry: automatic: - - exit_status: \"-1\" + - exit_status: "-1" limit: 3 signal_reason: none - signal_reason: agent_stop From a2e023530e6dabe95b63e2a7c6eab036f33989cb Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 3 Jun 2025 14:16:13 +0200 Subject: [PATCH 08/10] revert hardcoded branch filter for testing on branch --- .buildkite/scripts/run-bc-upgrade-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run-bc-upgrade-tests.sh b/.buildkite/scripts/run-bc-upgrade-tests.sh index ff9d3e4fc75b7..5f268996495ff 100644 --- a/.buildkite/scripts/run-bc-upgrade-tests.sh +++ b/.buildkite/scripts/run-bc-upgrade-tests.sh @@ -16,7 +16,7 @@ set -euo pipefail # So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot. MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json | jq ".releases[] | -select(.branch == \"main\") | +select(.branch == \"$BUILDKITE_BRANCH\") | select(.active_release == true) | ((.snapshots | to_entries | sort_by(.value.completed_at)) + (.build_candidates | to_entries | sort_by(.value.completed_at))) | From 7086a6f8fb63c4452de3f10e14421e98c006e5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Mon, 9 Jun 2025 09:46:57 +0200 Subject: [PATCH 09/10] Update .buildkite/scripts/run-bc-upgrade-tests.sh Co-authored-by: Brian Seeders --- .buildkite/scripts/run-bc-upgrade-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run-bc-upgrade-tests.sh b/.buildkite/scripts/run-bc-upgrade-tests.sh index 5f268996495ff..94343a2d5fb21 100644 --- a/.buildkite/scripts/run-bc-upgrade-tests.sh +++ b/.buildkite/scripts/run-bc-upgrade-tests.sh @@ -28,7 +28,7 @@ if [[ -z "$MANIFEST_URL" ]]; then exit 0 fi -MANIFEST="$(eval "curl -s $MANIFEST_URL")" +MANIFEST="$(curl -s "$MANIFEST_URL")" if [[ -z "$MANIFEST" ]]; then echo "Cannot get the build manifest from [$MANIFEST_URL]" exit 1 From f7f2b073e38ce83f40571f24915cdba4a0fc08cb Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Mon, 9 Jun 2025 10:09:27 +0200 Subject: [PATCH 10/10] PR comments --- .buildkite/pipelines/intake.template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index e2e809a2f971b..988f63efdad80 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -65,9 +65,6 @@ steps: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: bc-upgrade command: ".buildkite/scripts/run-bc-upgrade-tests.sh" - agents: - image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5" - memory: "4G" - group: lucene-compat steps: - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"