[8.19] Implement automated version bump pipeline for patch and minor releases (#262224)

.buildkite/pipeline-resource-definitions/kibana-version-bump.yml

@@ -0,0 +1,42 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-version-bump
+  description: Buildkite Pipeline for kibana version bump
+  links:
+    - title: Kibana Version Bump Pipeline
+      url: https://buildkite.com/elastic/kibana-version-bump
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-version-bump
+      description: Buildkite Pipeline for kibana version bump
+    spec:
+      allow_rebuilds: true
+      cancel_intermediate_builds: false
+      branch_configuration: main
+      pipeline_file: .buildkite/pipelines/version_bump.yml
+      initial_step_plugins:
+        - sparse-checkout#v1.6.0:
+            paths:
+              - .buildkite
+            cleanup_sparse_state: true
+      provider_settings:
+        trigger_mode: none
+      repository: elastic/kibana
+      skip_intermediate_builds: false
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+        release-eng:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -13,28 +13,33 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-node-glibc-217.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-node-pointer-compression.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-scout-update-metadata.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-version-bump.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/gen-ai-evals.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
@@ -46,6 +51,3 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml

.buildkite/pipeline-utils/buildkite/utils.ts

@@ -11,8 +11,13 @@ import { execFileSync } from 'child_process';
 import fs from 'fs';
 import { load as loadYaml } from 'js-yaml';
 
-export function emitPipeline(pipelineSteps: string[]) {
+export function emitPipeline(pipelineSteps: string[], debug = false) {
   const pipelineStr = [...new Set(pipelineSteps)].join('\n');
+
+  if (debug) {
+    console.warn('Emitting pipeline:\n', pipelineStr);
+  }
+
   console.log(pipelineStr);
 }
 

.buildkite/pipelines/version_bump/bump_package_json_versions.yml

@@ -0,0 +1,9 @@
+steps:
+  - name: 'Bump package.json versions'
+    key: bump-package-json-versions
+    command: .buildkite/scripts/steps/version_bump/bump_package_json_versions.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4

.buildkite/pipelines/version_bump/bump_package_json_versions_to_main.yml

@@ -0,0 +1,11 @@
+steps:
+  - name: 'Bump package.json versions (to main)'
+    key: bump-package-json-versions-to-main
+    command: .buildkite/scripts/steps/version_bump/bump_package_json_versions.sh
+    env:
+      OVERRIDE_BRANCH: main
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4

.buildkite/pipelines/version_bump/bump_versions_json.yml

@@ -0,0 +1,9 @@
+steps:
+  - name: 'Bump versions.json'
+    key: bump-versions-json
+    command: .buildkite/scripts/steps/version_bump/bump_versions_json.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4

.buildkite/pipelines/version_bump/create_new_branch.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Create new release branch'
+    key: create-new-branch
+    command: .buildkite/scripts/steps/version_bump/create_new_branch_of_main.sh

.buildkite/pipelines/version_bump/ensure_version_label.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Ensure version label exists'
+    key: ensure-version-label
+    command: .buildkite/scripts/steps/version_bump/ensure_version_label.sh

.buildkite/pipelines/version_bump/notify_branch_created.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Notify Slack: branch created'
+    key: notify-branch-created
+    command: .buildkite/scripts/steps/version_bump/notify_branch_created.sh

.buildkite/pipelines/version_bump/reconcile_pr_labels.yml

@@ -0,0 +1,8 @@
+steps:
+  - name: 'Reconcile PR Labels'
+    key: reconcile-pr-labels
+    depends_on:
+      - update-label-color
+    command: .buildkite/scripts/steps/version_bump/reconcile_pr_labels.sh
+    env:
+      DRY_RUN: 'true'

.buildkite/pipelines/version_bump/trigger_dra_snapshot.yml

@@ -0,0 +1,12 @@
+steps:
+  - name: mock kibana snapshot build
+    if: build.env("DRY_RUN") == "true"
+    command: sleep 10
+
+  - name: Trigger DRA Snapshot Build
+    key: trigger-dra-snapshot
+    if: build.env("DRY_RUN") == "false"
+    trigger: kibana-artifacts-snapshot
+    async: false
+    build:
+      branch: '${BRANCH}'

.buildkite/pipelines/version_bump/trigger_dra_snapshot_on_main.yml

@@ -0,0 +1,8 @@
+steps:
+  - name: Trigger DRA Snapshot Build
+    key: trigger-dra-snapshot-on-main
+    trigger: kibana-artifacts-snapshot
+    if: build.env("DRY_RUN") != "true"
+    async: false
+    build:
+      branch: 'main'

.buildkite/pipelines/version_bump/trigger_dra_staging.yml

@@ -0,0 +1,8 @@
+steps:
+  - name: Trigger DRA Staging Build
+    key: trigger-dra-staging
+    trigger: kibana-artifacts-staging
+    if: build.env("DRY_RUN") != "true"
+    async: false
+    build:
+      branch: '${BRANCH}'

.buildkite/pipelines/version_bump/trigger_es_build_and_promote.yml

@@ -0,0 +1,45 @@
+steps:
+  - name: mock build es snapshot
+    key: mock-build-es-snapshot
+    if: build.env("DRY_RUN") == "true"
+    command: sleep 10
+
+  - name: mock promote es snapshot
+    if: build.env("DRY_RUN") == "true"
+    depends_on: mock-build-es-snapshot
+    command: |
+
+      buildkite-agent pipeline upload << PIPELINE
+      steps:
+        - name: 'Mock Promote ES Snapshot trigger'
+          command: sleep 10
+
+      PIPELINE
+
+  - key: build-es-snapshot
+    label: Build ES Snapshot
+    trigger: kibana-elasticsearch-snapshot-build
+    async: false
+    if: build.env("DRY_RUN") == "false"
+    build:
+      branch: $BRANCH
+      env:
+        PARENT_TRIGGER_JOB_ID: '${BUILDKITE_JOB_ID}'
+
+  - key: promote-es-snapshot
+    label: 'Upload promote step'
+    if: build.env("DRY_RUN") == "false"
+    depends_on: build-es-snapshot
+    command: |
+      ES_SNAPSHOT_MANIFEST="$(buildkite-agent meta-data get es_snapshot_manifest)"
+
+      buildkite-agent pipeline upload << PIPELINE
+      steps:
+        - label: 'Promote ES Snapshot'
+          trigger: kibana-elasticsearch-snapshot-promote
+          async: false
+          build:
+            branch: '$${BRANCH}'
+            env:
+              ES_SNAPSHOT_MANIFEST: '$${ES_SNAPSHOT_MANIFEST}'
+      PIPELINE

.buildkite/pipelines/version_bump/trigger_es_build_and_promote_on_main.yml

@@ -0,0 +1,28 @@
+steps:
+  - key: build-es-snapshot
+    label: Build ES Snapshot
+    trigger: kibana-elasticsearch-snapshot-build
+    async: false
+    if: build.env("DRY_RUN") != "true"
+    build:
+      branch: main
+      env:
+        PARENT_TRIGGER_JOB_ID: '${BUILDKITE_JOB_ID}'
+
+  - key: promote-es-snapshot
+    label: 'Upload promote step'
+    depends_on: build-es-snapshot
+    if: build.env("DRY_RUN") != "true"
+    command: |
+      ES_SNAPSHOT_MANIFEST="$(buildkite-agent meta-data get es_snapshot_manifest)"
+
+      buildkite-agent pipeline upload << PIPELINE
+      steps:
+        - label: 'Promote ES Snapshot'
+          trigger: kibana-elasticsearch-snapshot-promote
+          async: false
+          build:
+            branch: 'main'
+            env:
+              ES_SNAPSHOT_MANIFEST: '$${ES_SNAPSHOT_MANIFEST}'
+      PIPELINE

.buildkite/pipelines/version_bump/update_label_color.yml

@@ -0,0 +1,9 @@
+steps:
+  - name: 'Update Label Color'
+    key: update-label-color
+    command: .buildkite/scripts/steps/version_bump/update_label_color.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4

.buildkite/pipelines/version_bump/update_pipeline_resource_definitions.yml

@@ -0,0 +1,9 @@
+steps:
+  - name: 'Update pipeline resource definitions'
+    key: update-pipeline-resource-definitions
+    command: .buildkite/scripts/steps/version_bump/update_pipeline_resource_definitions.sh
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n4-standard-4

.buildkite/pipelines/version_bump/update_release_branch.yml

@@ -0,0 +1,4 @@
+steps:
+  - name: 'Update release branch config'
+    key: update-release-branch
+    command: .buildkite/scripts/steps/version_bump/update_release_branch.sh

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -125,6 +125,10 @@ cat << EOF | buildkite-agent annotate --style "info"
   - \`ES_SNAPSHOT_ID\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_ID)\`
 EOF
 
+if [ "$BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG" = "kibana-version-bump" ]; then
+  buildkite-agent meta-data set es_snapshot_manifest "$ES_SNAPSHOT_MANIFEST" --job "$PARENT_TRIGGER_JOB_ID"
+fi
+
 cat << EOF | buildkite-agent pipeline upload
 steps:
   - trigger: 'kibana-elasticsearch-snapshot-verify'

.buildkite/scripts/steps/version_bump/bump.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ts-node .buildkite/scripts/steps/version_bump/pipeline.ts | buildkite-agent pipeline upload
+