Implement automated version bump pipeline for patch and minor releases

.buildkite/pipeline-resource-definitions/kibana-version-bump.yml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-version-bump
+  description: Buildkite Pipeline for kibana version bump
+  links:
+    - title: Kibana Version Bump Pipeline
+      url: https://buildkite.com/elastic/kibana-version-bump
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-version-bump
+      description: Buildkite Pipeline for kibana version bump
+    spec:
+      allow_rebuilds: true
+      cancel_intermediate_builds: false
+      branch_configuration: main
+      pipeline_file: .buildkite/version_bump.yml
+      provider_settings:
+        trigger_mode: none
+      repository: elastic/kibana
+      skip_intermediate_builds: false
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipelines/version_bump.yml

@@ -0,0 +1,61 @@
+notify:
+  - slack:
+      channels:
+        - '#kibana-operations'
+    if: (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/) && (build.state == 'passed' || build.state == 'failed')
+  - slack:
+      channels:
+        - '#kibana-operations'
+      message: |
+        🚦 Pipeline waiting for approval 🚦
+        Repo: `${REPO}`
+
+        Ready to fetch DRA artifacts - please unblock when ready.
+        New version: `${NEW_VERSION}`
+        Branch: `${BRANCH}`
+        Workflow: `${WORKFLOW}`
+        ${BUILDKITE_BUILD_URL}
+    if: build.state == "blocked"
+  - slack:
+      channels:
+        - '#mission-control'
+      message: |
+        Kibana has been bumped to `${NEW_VERSION}` on branch `${BRANCH}` and a DRA build has begun.
+        Build: ${BUILDKITE_BUILD_URL}
+    if: build.state == "passed"
+
+steps:
+  - command: .buildkite/scripts/version_bump/bump.sh
+    label: Bump version
+
+  - wait
+
+  - label: 'Fetch DRA Artifacts'
+    key: fetch-dra-artifacts
+    depends_on: block-get-dra-artifacts
+    agents:
+      image: docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest
+      cpu: 250m
+      memory: 512Mi
+      ephemeralStorage: 1Gi
+    command:
+      - echo "Starting DRA artifacts retrieval..."
+    timeout_in_minutes: 240
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+      manual:
+        permit_on_passed: true
+
+    plugins:
+      - elastic/json-watcher#v1.0.0:
+          url: 'https://artifacts-staging.elastic.co/kibana/latest/${BRANCH}.json'
+          field: '.version'
+          expected_value: '${NEW_VERSION}'
+          polling_interval: '30'
+      - elastic/json-watcher#v1.0.0:
+          url: 'https://storage.googleapis.com/elastic-artifacts-snapshot/kibana/latest/${BRANCH}.json'
+          field: '.version'
+          expected_value: '${NEW_VERSION}-SNAPSHOT'
+          polling_interval: '30'

.buildkite/pipelines/version_bump/bump_package_json_versions.yml

@@ -0,0 +1,11 @@
+steps:
+  - name: 'Bump package.json versions'
+    key: bump-package-json-versions
+    command: .buildkite/scripts/steps/version_bump/bump_package_json_versions.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4
+      preemptible: true
+      spotZones: us-central1-f,us-central1-c,us-central1-a

.buildkite/pipelines/version_bump/bump_package_json_versions_to_main.yml

@@ -0,0 +1,13 @@
+steps:
+  - name: 'Bump package.json versions (to main)'
+    key: bump-package-json-versions-to-main
+    command: .buildkite/scripts/steps/version_bump/bump_package_json_versions.sh
+    env:
+      OVERRIDE_BRANCH: main
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4
+      preemptible: true
+      spotZones: us-central1-f,us-central1-c,us-central1-a

.buildkite/pipelines/version_bump/bump_versions_json.yml

@@ -0,0 +1,11 @@
+steps:
+  - name: 'Bump versions.json'
+    key: bump-versions-json
+    command: .buildkite/scripts/steps/version_bump/bump_versions_json.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4
+      preemptible: true
+      spotZones: us-central1-f,us-central1-c,us-central1-a

.buildkite/pipelines/version_bump/create_new_branch.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Create new release branch'
+    key: create-new-branch
+    command: .buildkite/scripts/steps/version_bump/create_new_branch_of_main.sh

.buildkite/pipelines/version_bump/ensure_version_label.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Ensure version label exists'
+    key: ensure-version-label
+    command: .buildkite/scripts/steps/version_bump/ensure_version_label.sh

.buildkite/pipelines/version_bump/notify_branch_created.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Notify Slack: branch created'
+    key: notify-branch-created
+    command: .buildkite/scripts/steps/version_bump/notify_branch_created.sh

.buildkite/pipelines/version_bump/reconcile_pr_labels.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Reconcile PR Labels'
+    key: reconcile-pr-labels
+    command: .buildkite/scripts/steps/version_bump/reconcile_pr_labels.sh

.buildkite/pipelines/version_bump/trigger_dra_snapshot.yml

@@ -0,0 +1,7 @@
+steps:
+  - name: Trigger DRA Snapshot Build
+    key: trigger-dra-snapshot
+    trigger: kibana-artifacts-snapshot
+    async: false
+    build:
+      branch: 'main'

.buildkite/pipelines/version_bump/trigger_dra_staging.yml

@@ -0,0 +1,7 @@
+steps:
+  - name: Trigger DRA Staging Build
+    key: trigger-dra-staging
+    trigger: kibana-artifacts-staging
+    async: false
+    build:
+      branch: 'main'

.buildkite/pipelines/version_bump/trigger_es_build_and_promote.yml

@@ -0,0 +1,26 @@
+steps:
+  # run the bash files directly and only run the build and promote, because verify will fail.
+  # TODO: Ask @alex if this would work, especially because the meta data get happens in two separate bash files.
+
+  # Step 1: Run the ES Snapshot Build
+  - command: .buildkite/scripts/steps/es_snapshots/build.sh
+    label: Build ES Snapshot
+    timeout_in_minutes: 30
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-8
+      diskSizeGb: 130
+
+  # Step 2: Run the ES Promote
+  - label: Promote Snapshot
+    command: .buildkite/scripts/steps/es_snapshots/promote.sh
+    depends_on: build-es-snapshot
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2

.buildkite/pipelines/version_bump/update_label_color.yml

@@ -0,0 +1,11 @@
+steps:
+  - name: 'Update Label Color'
+    key: update-label-color
+    command: .buildkite/scripts/steps/version_bump/update_label_color.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4
+      preemptible: true
+      spotZones: us-central1-f,us-central1-c,us-central1-a

.buildkite/pipelines/version_bump/update_pipeline_resource_definitions.yml

@@ -0,0 +1,11 @@
+steps:
+  - name: 'Update pipeline resource definitions'
+    key: update-pipeline-resource-definitions
+    command: .buildkite/scripts/steps/version_bump/update_pipeline_resource_definitions.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4
+      preemptible: true
+      spotZones: us-central1-f,us-central1-c,us-central1-a

.buildkite/pipelines/version_bump/update_release_branch.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Update release branch config'
+    key: update-release-branch
+    command: .buildkite/scripts/steps/version_bump/update_release_branch.sh

.buildkite/scripts/steps/version_bump/bump.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# TODO: If the logic is simple within the piplein.ts file, then remove that layer of abstraction.
+ts-node .buildkite/scripts/version_bump/pipeline.ts | buildkite-agent pipeline upload
+

.buildkite/scripts/steps/version_bump/bump_package_json_versions.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "$0")/wait_for_pr_merge.sh"
+
+branch_to_merge_into="${OVERRIDE_BRANCH:-$BRANCH}"
+
+old_version=""
+store_old_version() {
+  old_version=$(jq -r '.version' package.json)
+  echo "Current version: $old_version"
+  echo "New version: $NEW_VERSION"
+
+  buildkite-agent meta-data set "OLD_VERSION" "$old_version"
+}
+
+update_package_json_version() {
+  echo --- Bump package.json versions
+  sed -i "s/\"version\": \"[0-9]\+\.[0-9]\+\.[0-9]\+\"/\"version\": \"${NEW_VERSION}\"/g" package.json
+  sed -i "s/\"version\": \"[0-9]\+\.[0-9]\+\.[0-9]\+\"/\"version\": \"${NEW_VERSION}\"/g" x-pack/package.json
+}
+
+update_initial_app_data() {
+  echo --- Bump initial_app_data kibanaVersion
+  local target="x-pack/solutions/search/plugins/enterprise_search/common/__mocks__/initial_app_data.ts"
+  sed -i "s/kibanaVersion: '[0-9]\+\.[0-9]\+\.[0-9]\+'/kibanaVersion: '${NEW_VERSION}'/" "$target"
+}
+
+update_kibana_migrator_utils() {
+  echo --- Bump BASELINE_ELASTICSEARCH_VERSION
+  local target="src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts"
+  sed -i "s/BASELINE_ELASTICSEARCH_VERSION = '[0-9]\+\.[0-9]\+\.[0-9]\+'/BASELINE_ELASTICSEARCH_VERSION = '${NEW_VERSION}'/" "$target"
+}
+
+delete_baseline_archives() {
+  echo --- Deleting baseline archives
+  local archive_dir="src/core/server/integration_tests/saved_objects/migrations/archives"
+  find "$archive_dir" -type f -name "${old_version}_baseline_*.zip" -delete
+}
+
+store_old_version
+
+update_package_json_version
+
+# If branch is 8.19, skip updating initial_app_data
+if [[ "$BRANCH" != "8.19" ]]; then
+  update_initial_app_data
+fi
+
+update_kibana_migrator_utils
+
+delete_baseline_archives
+
+# Run integration tests to update snapshots
+node scripts/jest_integration.js -u src/core/server/integration_tests/saved_objects/migrations/group1/create_test_archives.test.ts
+node scripts/jest_integration.js -u src/core/server/integration_tests/saved_objects/migrations/group1/v2_migration.test.ts
+
+
+echo --- Committing version bump changes
+git config --global user.name kibanamachine
+git config --global user.email '42973632+kibanamachine@users.noreply.github.com'
+
+head_branch="bump-versions-$(date +%F_%H-%M-%S)"
+git checkout -b "$head_branch"
+git add package.json x-pack/package.json x-pack/solutions/search/plugins/enterprise_search/common/__mocks__/initial_app_data.ts src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_archive_utils.ts src/core/server/integration_tests/saved_objects/migrations/archives
+git commit -m "[version bump] Bump version to ${NEW_VERSION}"
+git push origin "$branch"
+
+prUrl=$(gh pr create --repo elastic/kibana --base "$branch_to_merge_into" --head "$head_branch" --title "[bump version] $(date +%F) Bump package.json versions to ${NEW_VERSION}" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "backport:skip")
+echo "Opened PR: $prUrl"
+
+gh pr merge --repo elastic/kibana --auto --squash --delete-branch "$prUrl"
+
+wait_for_pr_merge "$prUrl"
+
+