From df8c4816b821fab9045d2533e92fd8b23e360dc1 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Fri, 17 Jan 2025 10:09:11 +0100 Subject: [PATCH 1/5] Generation script should create prs for 8.x and main separately --- .../scripts/steps/console_definitions_sync.sh | 105 ++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/.buildkite/scripts/steps/console_definitions_sync.sh b/.buildkite/scripts/steps/console_definitions_sync.sh index 71d70138f8a1b..ae00c9947674d 100755 --- a/.buildkite/scripts/steps/console_definitions_sync.sh +++ b/.buildkite/scripts/steps/console_definitions_sync.sh @@ -3,70 +3,111 @@ set -euo pipefail GIT_SCOPE="src/platform/plugins/shared/console/server/lib/spec_definitions" +# We will generate the console definitions for both main and 8.x branches +BRANCHES=("main" "8.x") +# Github username for the Kibana machine user +KIBANA_MACHINE_USERNAME="kibanamachine" + report_main_step () { echo "--- $1" } -main () { - cd "$PARENT_DIR" +update_console_definitions() { + local branch="$1" - report_main_step "Cloning repositories" + report_main_step "=== [Branch: $branch] ===" + report_main_step "Checking out Elasticsearch Specification on branch: $branch" + + cd "$PARENT_DIR" + # Remove old copy if it exists rm -rf elasticsearch-specification - if ! git clone https://github.com/elastic/elasticsearch-specification --depth 1; then - echo "Error: Failed to clone the elasticsearch-specification repository." + + # Clone the ES spec on the given branch + if ! git clone --branch "$branch" https://github.com/elastic/elasticsearch-specification --depth 1; then + echo "Error: Failed to clone the elasticsearch-specification repository (branch: $branch)." exit 1 fi - report_main_step "Bootstrapping Kibana" + # Now switch Kibana to the same branch + report_main_step "Switching Kibana to branch: $branch" cd "$KIBANA_DIR" + git checkout "$branch" + + report_main_step "Bootstrapping Kibana (branch: $branch)" .buildkite/scripts/bootstrap.sh - report_main_step "Generating console definitions" - node scripts/generate_console_definitions.js --source "$PARENT_DIR/elasticsearch-specification" --emptyDest + report_main_step "Generating console definitions (branch: $branch)" + node scripts/generate_console_definitions.js \ + --source "$PARENT_DIR/elasticsearch-specification" \ + --emptyDest # Check if there are any differences set +e git diff --exit-code --quiet "$GIT_SCOPE" - if [ $? -eq 0 ]; then - echo "No differences found. Exiting.." - exit - fi + local diff_exit_code=$? set -e - report_main_step "Differences found. Checking for an existing pull request." + if [ $diff_exit_code -eq 0 ]; then + echo "No differences found on branch '$branch'. Moving on." + return 0 + fi - KIBANA_MACHINE_USERNAME="kibanamachine" - git config --global user.name "$KIBANA_MACHINE_USERNAME" - git config --global user.email '42973632+kibanamachine@users.noreply.github.com' + report_main_step "Differences found on branch '$branch'. Checking for an existing pull request." - PR_TITLE='[Console] Update console definitions' - PR_BODY='This PR updates the console definitions to match the latest ones from the @elastic/elasticsearch-specification repo.' + # Prepare PR title/body + local PR_TITLE="[Console] Update console definitions ($branch)" + local PR_BODY="This PR updates the console definitions to match the latest from the @elastic/elasticsearch-specification repo on the '$branch' branch." - # Check if a PR already exists - pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME" --limit 1 --json title -q ".[].title") + # Check if a PR already exists (search by title, state, and author) + local pr_search_result + pr_search_result=$(gh pr list \ + --search "$PR_TITLE" \ + --state open \ + --author "$KIBANA_MACHINE_USERNAME" \ + --limit 1 \ + --json title \ + -q ".[].title" || true) if [ "$pr_search_result" == "$PR_TITLE" ]; then - echo "PR already exists. Exiting.." - exit + echo "PR already exists on branch '$branch'. Skipping creation." + return 0 fi - echo "No existing PR found. Proceeding.." + echo "No existing PR found for branch '$branch'. Proceeding." - # Commit diff - BRANCH_NAME="console_definitions_sync_$(date +%s)" + # Commit the changes on a new branch + local TEMP_BRANCH_NAME="console_definitions_sync_${branch}_$(date +%s)" + git checkout -b "$TEMP_BRANCH_NAME" - git checkout -b "$BRANCH_NAME" - - git add $GIT_SCOPE + git add "$GIT_SCOPE" git commit -m "Update console definitions" - report_main_step "Changes committed. Creating pull request." + report_main_step "Changes committed. Pushing branch '$TEMP_BRANCH_NAME' and creating pull request." + + git push origin "$TEMP_BRANCH_NAME" - git push origin "$BRANCH_NAME" + # Create the PR, targeting the same branch in Kibana + gh pr create \ + --title "$PR_TITLE" \ + --body "$PR_BODY" \ + --base "$branch" \ + --head "${TEMP_BRANCH_NAME}" \ + --label 'release_note:skip' \ + --label 'backport:skip' \ + --label 'Feature:Console' \ + --label 'Team:Kibana Management' +} + +main() { + # Configure git user + git config --global user.name "$KIBANA_MACHINE_USERNAME" + git config --global user.email '42973632+kibanamachine@users.noreply.github.com' - # Create PR - gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Feature:Console' --label 'Team:Kibana Management' + # Run the process for each branch in sequence + for br in "${BRANCHES[@]}"; do + update_console_definitions "$br" + done } main From e2d010c10bc92afd13cd07374fd35db395bd1c68 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Fri, 17 Jan 2025 10:09:17 +0100 Subject: [PATCH 2/5] commit with @elastic email From bf26e287a20e29075acf51c7729fbac65abd7fe4 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Wed, 22 Jan 2025 09:53:51 +0100 Subject: [PATCH 3/5] Use separate pipeline for each branch we need to build for --- .../kibana-console-definitions-sync-8_x.yml | 54 ++++++++++ ... kibana-console-definitions-sync-main.yml} | 10 +- .../scripts/steps/console_definitions_sync.sh | 101 ++++++------------ 3 files changed, 93 insertions(+), 72 deletions(-) create mode 100644 .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml rename .buildkite/pipeline-resource-definitions/{kibana-console-definitions-sync.yml => kibana-console-definitions-sync-main.yml} (88%) diff --git a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml new file mode 100644 index 0000000000000..44edec21bd7ef --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml @@ -0,0 +1,54 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-console-definitions-sync-8_x + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for 8.x branch + links: + - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync-8_x' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-management' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / Console definitions sync 8.x + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for 8.x branch + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-management' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: false + branch_configuration: 8.x + default_branch: 8.x + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/console_definitions_sync.yml + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: true + teams: + kibana-management: + access_level: MANAGE_BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + schedules: + Weekly build: + cronline: 0 0 * * 1 America/New_York + message: Weekly build + branch: 8.x + tags: + - kibana diff --git a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml similarity index 88% rename from .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml rename to .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml index a228823202c01..989ec5ed8bd13 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml @@ -2,10 +2,10 @@ apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: bk-kibana-console-definitions-sync - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions + name: bk-kibana-console-definitions-sync-main + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for main branch links: - - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync' + - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync-main' title: Pipeline link spec: type: buildkite-pipeline @@ -15,8 +15,8 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: kibana / Console definitions sync - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions + name: kibana / Console definitions sync main + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for main branch spec: env: SLACK_NOTIFICATIONS_CHANNEL: '#kibana-management' diff --git a/.buildkite/scripts/steps/console_definitions_sync.sh b/.buildkite/scripts/steps/console_definitions_sync.sh index ae00c9947674d..de490770d55b1 100755 --- a/.buildkite/scripts/steps/console_definitions_sync.sh +++ b/.buildkite/scripts/steps/console_definitions_sync.sh @@ -3,111 +3,78 @@ set -euo pipefail GIT_SCOPE="src/platform/plugins/shared/console/server/lib/spec_definitions" -# We will generate the console definitions for both main and 8.x branches -BRANCHES=("main" "8.x") -# Github username for the Kibana machine user -KIBANA_MACHINE_USERNAME="kibanamachine" - report_main_step () { echo "--- $1" } -update_console_definitions() { - local branch="$1" - - report_main_step "=== [Branch: $branch] ===" - report_main_step "Checking out Elasticsearch Specification on branch: $branch" - +main () { cd "$PARENT_DIR" - # Remove old copy if it exists - rm -rf elasticsearch-specification + report_main_step "Cloning repositories" - # Clone the ES spec on the given branch - if ! git clone --branch "$branch" https://github.com/elastic/elasticsearch-specification --depth 1; then - echo "Error: Failed to clone the elasticsearch-specification repository (branch: $branch)." + rm -rf elasticsearch-specification + if ! git clone --branch "$BUILDKITE_BRANCH" https://github.com/elastic/elasticsearch-specification --depth 1; then + echo "Error: Failed to clone the elasticsearch-specification repository." exit 1 fi - # Now switch Kibana to the same branch - report_main_step "Switching Kibana to branch: $branch" + report_main_step "Bootstrapping Kibana" cd "$KIBANA_DIR" - git checkout "$branch" - - report_main_step "Bootstrapping Kibana (branch: $branch)" .buildkite/scripts/bootstrap.sh - report_main_step "Generating console definitions (branch: $branch)" - node scripts/generate_console_definitions.js \ - --source "$PARENT_DIR/elasticsearch-specification" \ - --emptyDest + report_main_step "Generating console definitions" + node scripts/generate_console_definitions.js --source "$PARENT_DIR/elasticsearch-specification" --emptyDest # Check if there are any differences set +e git diff --exit-code --quiet "$GIT_SCOPE" - local diff_exit_code=$? + if [ $? -eq 0 ]; then + echo "No differences found. Exiting.." + exit + fi set -e - if [ $diff_exit_code -eq 0 ]; then - echo "No differences found on branch '$branch'. Moving on." - return 0 - fi + report_main_step "Differences found. Checking for an existing pull request." - report_main_step "Differences found on branch '$branch'. Checking for an existing pull request." + KIBANA_MACHINE_USERNAME="kibanamachine" + git config --global user.name "$KIBANA_MACHINE_USERNAME" + git config --global user.email '42973632+kibanamachine@users.noreply.github.com' - # Prepare PR title/body - local PR_TITLE="[Console] Update console definitions ($branch)" - local PR_BODY="This PR updates the console definitions to match the latest from the @elastic/elasticsearch-specification repo on the '$branch' branch." + PR_TITLE="[Console] Update console definitions (${branch_name})" + PR_BODY='This PR updates the console definitions to match the latest ones from the @elastic/elasticsearch-specification repo.' - # Check if a PR already exists (search by title, state, and author) - local pr_search_result - pr_search_result=$(gh pr list \ - --search "$PR_TITLE" \ - --state open \ - --author "$KIBANA_MACHINE_USERNAME" \ - --limit 1 \ - --json title \ - -q ".[].title" || true) + # Check if a PR already exists + pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME" --limit 1 --json title -q ".[].title") if [ "$pr_search_result" == "$PR_TITLE" ]; then - echo "PR already exists on branch '$branch'. Skipping creation." - return 0 + echo "PR already exists. Exiting.." + exit fi - echo "No existing PR found for branch '$branch'. Proceeding." + echo "No existing PR found. Proceeding.." + + # Commit diff + BRANCH_NAME="console_definitions_sync_$(date +%s)" - # Commit the changes on a new branch - local TEMP_BRANCH_NAME="console_definitions_sync_${branch}_$(date +%s)" - git checkout -b "$TEMP_BRANCH_NAME" + git checkout -b "$BRANCH_NAME" - git add "$GIT_SCOPE" + git add $GIT_SCOPE git commit -m "Update console definitions" - report_main_step "Changes committed. Pushing branch '$TEMP_BRANCH_NAME' and creating pull request." + report_main_step "Changes committed. Creating pull request." - git push origin "$TEMP_BRANCH_NAME" + git push origin "$BRANCH_NAME" - # Create the PR, targeting the same branch in Kibana + # Create PR gh pr create \ --title "$PR_TITLE" \ --body "$PR_BODY" \ - --base "$branch" \ - --head "${TEMP_BRANCH_NAME}" \ - --label 'release_note:skip' \ + --base main \ + --head "$BRANCH_NAME" \ --label 'backport:skip' \ + --label 'release_note:skip' \ --label 'Feature:Console' \ --label 'Team:Kibana Management' } -main() { - # Configure git user - git config --global user.name "$KIBANA_MACHINE_USERNAME" - git config --global user.email '42973632+kibanamachine@users.noreply.github.com' - - # Run the process for each branch in sequence - for br in "${BRANCHES[@]}"; do - update_console_definitions "$br" - done -} - main From 8e0d31fb711010e278c8e2dc414d27c99ed68ae8 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Wed, 22 Jan 2025 09:55:27 +0100 Subject: [PATCH 4/5] Target PR to the right branch --- .buildkite/scripts/steps/console_definitions_sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/console_definitions_sync.sh b/.buildkite/scripts/steps/console_definitions_sync.sh index de490770d55b1..8431d8debc59f 100755 --- a/.buildkite/scripts/steps/console_definitions_sync.sh +++ b/.buildkite/scripts/steps/console_definitions_sync.sh @@ -69,7 +69,7 @@ main () { gh pr create \ --title "$PR_TITLE" \ --body "$PR_BODY" \ - --base main \ + --base "$BUILDKITE_BRANCH" \ --head "$BRANCH_NAME" \ --label 'backport:skip' \ --label 'release_note:skip' \ From 4d79d627075a1ea533ed6595e063d9fce73de387 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Jan 2025 10:18:06 +0100 Subject: [PATCH 5/5] Address CR changes --- .../kibana-console-definitions-sync-8_x.yml | 54 ------------------- ...ml => kibana-console-definitions-sync.yml} | 12 ++--- 2 files changed, 6 insertions(+), 60 deletions(-) delete mode 100644 .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml rename .buildkite/pipeline-resource-definitions/{kibana-console-definitions-sync-main.yml => kibana-console-definitions-sync.yml} (86%) diff --git a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml deleted file mode 100644 index 44edec21bd7ef..0000000000000 --- a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-8_x.yml +++ /dev/null @@ -1,54 +0,0 @@ -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: bk-kibana-console-definitions-sync-8_x - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for 8.x branch - links: - - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync-8_x' - title: Pipeline link -spec: - type: buildkite-pipeline - owner: 'group:kibana-management' - system: buildkite - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: kibana / Console definitions sync 8.x - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for 8.x branch - spec: - env: - SLACK_NOTIFICATIONS_CHANNEL: '#kibana-management' - ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' - allow_rebuilds: false - branch_configuration: 8.x - default_branch: 8.x - repository: elastic/kibana - pipeline_file: .buildkite/pipelines/console_definitions_sync.yml - provider_settings: - build_branches: false - build_pull_requests: false - publish_commit_status: false - trigger_mode: none - build_tags: false - prefix_pull_request_fork_branch_names: false - skip_pull_request_builds_for_existing_commits: true - teams: - kibana-management: - access_level: MANAGE_BUILD_AND_READ - kibana-operations: - access_level: MANAGE_BUILD_AND_READ - appex-qa: - access_level: MANAGE_BUILD_AND_READ - kibana-tech-leads: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: BUILD_AND_READ - schedules: - Weekly build: - cronline: 0 0 * * 1 America/New_York - message: Weekly build - branch: 8.x - tags: - - kibana diff --git a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml similarity index 86% rename from .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml rename to .buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml index 989ec5ed8bd13..b4ca2b3e00e6e 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync-main.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml @@ -2,10 +2,10 @@ apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: bk-kibana-console-definitions-sync-main - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for main branch + name: bk-kibana-console-definitions-sync + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions links: - - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync-main' + - url: 'https://buildkite.com/elastic/kibana-console-definitions-sync' title: Pipeline link spec: type: buildkite-pipeline @@ -15,14 +15,14 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: kibana / Console definitions sync main - description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions for main branch + name: kibana / Console definitions sync + description: Opens a PR if anything changes in the console definitions in elasticsearch-definitions spec: env: SLACK_NOTIFICATIONS_CHANNEL: '#kibana-management' ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' allow_rebuilds: false - branch_configuration: main + branch_configuration: main 8.x default_branch: main repository: elastic/kibana pipeline_file: .buildkite/pipelines/console_definitions_sync.yml