Extend CI intake pipeline to run BC upgrade tests (elastic#128614)

A new bc-upgrade step in the intake pipeline invokes a bash script to get the commit hash of the latest BC (or snapshot if no BC is available yet) from https://docs.elastic.dev/release/stack/release-api#future-releases for the current branch and runs BWC / upgrade tests from this commit to the HEAD commit of the current branch.

If no matching BC / snapshot exists, the upgrade tests are skipped.

Relates to ES-11905

# Conflicts:
#	.buildkite/scripts/run-bc-upgrade-tests.sh

Author: ldematte

.buildkite/pipelines/intake.template.yml

@@ -72,6 +72,8 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: bc-upgrade
+    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/pipelines/intake.yml

@@ -72,6 +72,11 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: bc-upgrade
+    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      memory: "4G"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"