Merge branch 'main' into chore/puppeteer_23.3.0_upgrade

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.x",
     "8.15",
     "8.14",
     "8.13",
@@ -52,7 +53,8 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.16.0$": "main",
+    "^v9.0.0$": "main",
+    "^v8.16.0$": "8.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_platform_stateful_configs.yml

@@ -44,8 +44,7 @@ enabled:
   - test/api_integration/config.js
   - test/examples/config.js
   - test/functional/apps/bundles/config.ts
-  - test/functional/apps/console/monaco/config.ts
-  - test/functional/apps/console/ace/config.ts
+  - test/functional/apps/console/config.ts
   - test/functional/apps/context/config.ts
   - test/functional/apps/dashboard_elements/controls/common/config.ts
   - test/functional/apps/dashboard_elements/controls/options_list/config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -21,8 +21,6 @@ disabled:
   - x-pack/test/osquery_cypress/visual_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
   - x-pack/test/security_solution_cypress/config.ts
-  - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
-  - x-pack/test/threat_intelligence_cypress/config.ts
 
   # Playwright
   - x-pack/test/security_solution_playwright/playwright.config.ts

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -49,6 +49,10 @@ spec:
           cronline: 0 9 * * * America/New_York
           message: Daily build
           branch: main
+        Daily build (8.x):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: '8.x'
         Daily build (8.15):
           cronline: 0 9 * * * America/New_York
           message: Daily build
@@ -82,7 +86,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -130,7 +134,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -34,5 +34,7 @@ spec:
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ
+        kibana-security:
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 7.17 8.15
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -197,6 +197,32 @@ function getEnabledFtrConfigs(patterns?: string[]) {
   }
 }
 
+/**
+ * Collects environment variables from labels on the PR
+ * TODO: extract this (and other functions from this big file) to a separate module
+ */
+function collectEnvFromLabels() {
+  const LABEL_MAPPING: Record<string, Record<string, string>> = {
+    'ci:use-chrome-beta': {
+      USE_CHROME_BETA: 'true',
+    },
+  };
+
+  const envFromlabels: Record<string, string> = {};
+  if (!process.env.GITHUB_PR_LABELS) {
+    return envFromlabels;
+  } else {
+    const labels = process.env.GITHUB_PR_LABELS.split(',');
+    labels.forEach((label) => {
+      const env = LABEL_MAPPING[label];
+      if (env) {
+        Object.assign(envFromlabels, env);
+      }
+    });
+    return envFromlabels;
+  }
+}
+
 export async function pickTestGroupRunOrder() {
   const bk = new BuildkiteClient();
   const ciStats = new CiStatsClient();
@@ -273,9 +299,10 @@ export async function pickTestGroupRunOrder() {
           .filter(Boolean)
       : ['build'];
 
-  const FTR_EXTRA_ARGS: Record<string, string> = process.env.FTR_EXTRA_ARGS
+  const ftrExtraArgs: Record<string, string> = process.env.FTR_EXTRA_ARGS
     ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS }
     : {};
+  const envFromlabels: Record<string, string> = collectEnvFromLabels();
 
   const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
 
@@ -514,7 +541,8 @@ export async function pickTestGroupRunOrder() {
                   agents: expandAgentQueue(queue),
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,
-                    ...FTR_EXTRA_ARGS,
+                    ...ftrExtraArgs,
+                    ...envFromlabels,
                   },
                   retry: {
                     automatic: [

.buildkite/pipeline-utils/utils.test.ts

@@ -36,9 +36,6 @@ describe('getVersionsFile', () => {
     const versionsFile = getVersionsFile();
 
     expect(versionsFile.prevMajors).to.be.an('array');
-    expect(versionsFile.prevMajors.length).to.eql(1);
-    expect(versionsFile.prevMajors[0].branch).to.eql('7.17');
-
     expect(versionsFile.prevMinors).to.be.an('array');
   });
 

.buildkite/pipeline-utils/utils.ts

@@ -43,6 +43,7 @@ const getVersionsFile = (() => {
     prevMinors: Version[];
     prevMajors: Version[];
     current: Version;
+    versions: Version[];
   };
   const versionsFileName = 'versions.json';
   try {

.buildkite/pipelines/on_merge.yml

@@ -386,24 +386,6 @@ steps:
         - exit_status: '-1'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
-    label: 'Threat Intelligence Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on:
-      - build
-      - quick_checks
-    timeout_in_minutes: 60
-    parallelism: 1
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:

.buildkite/pipelines/pointer_compression.yml

@@ -321,23 +321,6 @@ steps:
         - exit_status: '-1'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
-    label: 'Threat Intelligence Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on:
-      - build
-    timeout_in_minutes: 60
-    parallelism: 1
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:

.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml

@@ -19,6 +19,7 @@ env:
   SKIP_NODE_SETUP: true
   TEAM_CHANNEL: "#kibana-mission-control"
   ENVIRONMENT: ${ENVIRONMENT?}
+  DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
 
 steps:
   - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -13,6 +13,7 @@ steps:
         CHECK_SLO_TAG: kibana
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
+        DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
     soft_fail: true
 
   - label: ":rocket: control-plane e2e tests"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -33,6 +33,7 @@ steps:
         CHECK_SYNTHETICS_MINIMUM_RUNS: 3
         MAX_FAILURES: 2
         CHECK_SYNTHETIC_MAX_POLL: 50
+        DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
     soft_fail: true
 
   - wait: ~

.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml

@@ -19,6 +19,7 @@ env:
   SKIP_NODE_SETUP: true
   TEAM_CHANNEL: "#kibana-mission-control"
   ENVIRONMENT: ${ENVIRONMENT?}
+  DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
 
 steps:
   - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -13,6 +13,7 @@ steps:
         CHECK_SLO_TAG: kibana
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
+        DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
     soft_fail: true
 
   - label: ":rocket: control-plane e2e tests"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -33,6 +33,7 @@ steps:
         CHECK_SYNTHETICS_MINIMUM_RUNS: 3
         MAX_FAILURES: 2
         CHECK_SYNTHETIC_MAX_POLL: 50
+        DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
     soft_fail: true
 
   - wait: ~

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml

@@ -13,6 +13,7 @@ steps:
           localSsds: 1
           localSsdInterface: nvme
           machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 300
         parallelism: 5
         retry:
@@ -102,6 +103,7 @@ steps:
           localSsds: 1
           localSsdInterface: nvme
           machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -119,6 +121,7 @@ steps:
           localSsds: 1
           localSsdInterface: nvme
           machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -136,6 +139,7 @@ steps:
           localSsds: 1
           localSsdInterface: nvme
           machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 120
         retry:
           automatic:
@@ -156,6 +160,7 @@ steps:
           localSsds: 1
           localSsdInterface: nvme
           machineType: n2-standard-4
+          preemptible: true
         timeout_in_minutes: 300
         parallelism: 3
         retry:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_detection_engine.yml

@@ -16,10 +16,6 @@ steps:
         # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 8
-        retry:
-          automatic:
-            - exit_status: "-1"
-              limit: 1
 
       - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
         label: "Cypress MKI - Detection Engine - Exceptions"
@@ -35,10 +31,6 @@ steps:
           # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
         timeout_in_minutes: 300
         parallelism: 6
-        retry:
-          automatic:
-            - exit_status: "-1"
-              limit: 1
 
   - group: "API MKI - Detection Engine - "
     key: api_test_detections_engine

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_entity_analytics.yml

@@ -13,10 +13,6 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 2
-    retry:
-      automatic:
-          - exit_status: '-1'
-            limit: 1
 
   - group: "API MKI - Entity Analytics"
     key: api_test_entity_analytics

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml

@@ -13,10 +13,6 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 4
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
 
   - group: "API MKI - Explore"
     key: api_test_explore

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_gen_ai.yml

@@ -13,10 +13,6 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 1
-    retry:
-      automatic:
-        - exit_status: "-1"
-          limit: 1
 
   - group: "API MKI - GenAI"
     key: api_test_ai_assistant

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml

@@ -13,10 +13,6 @@ steps:
       # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 8
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
 
   - group: "API MKI - Investigations"
     key: api_test_investigations