[Security Solution] [Sourcerer] Update to use Kibana Data Views

.bazelrc

@@ -2,12 +2,13 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-## Disabled for now
 # Remote cache settings for local env
-# build --remote_cache=https://storage.googleapis.com/kibana-bazel-cache
-# build --incompatible_remote_results_ignore_disk=true
-# build --remote_accept_cached=true
-# build --remote_upload_local_results=false
+build --remote_cache=grpcs://cloud.buildbuddy.io
+build --incompatible_remote_results_ignore_disk=true
+build --noremote_upload_local_results
+build --remote_timeout=30
+build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
+build --remote_accept_cached=true
 
 # BuildBuddy
 ## Metadata settings

.buildkite/pipelines/es_snapshots/verify.yml

@@ -19,7 +19,7 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-8
+      queue: c2-16
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
@@ -91,13 +91,5 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - plugins:
-      - junit-annotate#v1.9.0:
-          artifacts: target/junit/**/*.xml
-          job-uuid-file-pattern: '-bk__(.*).xml'
-
-  - wait: ~
-    continue_on_failure: true
-
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -0,0 +1,52 @@
+const stepInput = (key, nameOfSuite) => {
+  return {
+    key: `ftsr-suite/${key}`,
+    text: nameOfSuite,
+    required: false,
+    default: '0',
+  };
+};
+
+const OSS_CI_GROUPS = 12;
+const XPACK_CI_GROUPS = 13;
+
+const inputs = [
+  {
+    key: 'ftsr-override-count',
+    text: 'Override for all suites',
+    default: 0,
+    required: true,
+  },
+  {
+    key: 'ftsr-concurrency',
+    text: 'Max concurrency per step',
+    default: 20,
+    required: true,
+  },
+];
+
+for (let i = 1; i <= OSS_CI_GROUPS; i++) {
+  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+}
+
+for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
+  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
+}
+
+const pipeline = {
+  steps: [
+    {
+      input: 'Number of Runs',
+      fields: inputs,
+    },
+    {
+      wait: '~',
+    },
+    {
+      command: '.buildkite/pipelines/flaky_tests/runner.sh',
+      label: 'Create pipeline',
+    },
+  ],
+};
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+node .buildkite/pipelines/flaky_tests/pipeline.js | buildkite-agent pipeline upload

.buildkite/pipelines/flaky_tests/runner.js

@@ -0,0 +1,82 @@
+const { execSync } = require('child_process');
+
+const keys = execSync('buildkite-agent meta-data keys')
+  .toString()
+  .split('\n')
+  .filter((k) => k.startsWith('ftsr-suite/'));
+
+const overrideCount = parseInt(
+  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
+);
+
+const concurrency =
+  parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20;
+
+const testSuites = [];
+for (const key of keys) {
+  if (!key) {
+    continue;
+  }
+
+  const value =
+    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+  testSuites.push({
+    key: key.replace('ftsr-suite/', ''),
+    count: value === '' ? defaultCount : parseInt(value),
+  });
+}
+
+const steps = [];
+const pipeline = {
+  env: {
+    IGNORE_SHIP_CI_STATS_ERROR: 'true',
+  },
+  steps: steps,
+};
+
+steps.push({
+  command: '.buildkite/scripts/steps/build_kibana.sh',
+  label: 'Build Kibana Distribution and Plugins',
+  agents: { queue: 'c2-8' },
+  key: 'build',
+  if: "build.env('BUILD_ID_FOR_ARTIFACTS') == null || build.env('BUILD_ID_FOR_ARTIFACTS') == ''",
+});
+
+for (const testSuite of testSuites) {
+  const TEST_SUITE = testSuite.key;
+  const RUN_COUNT = testSuite.count;
+  const UUID = TEST_SUITE + process.env.UUID;
+
+  const JOB_PARTS = TEST_SUITE.split('/');
+  const IS_XPACK = JOB_PARTS[0] === 'xpack';
+  const CI_GROUP = JOB_PARTS.length > 2 ? JOB_PARTS[2] : '';
+
+  if (RUN_COUNT < 1) {
+    continue;
+  }
+
+  if (IS_XPACK) {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
+      label: `Default CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-6' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  } else {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
+      label: `OSS CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-4d' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  }
+}
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/runner.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+UUID="$(cat /proc/sys/kernel/random/uuid)"
+export UUID
+
+node .buildkite/pipelines/flaky_tests/runner.js | buildkite-agent pipeline upload

.buildkite/pipelines/hourly.yml

@@ -9,7 +9,7 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-8
+      queue: c2-16
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
@@ -98,7 +98,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-4d
+      queue: ci-group-6
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -159,13 +159,5 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - plugins:
-      - junit-annotate#v1.9.0:
-          artifacts: target/junit/**/*.xml
-          job-uuid-file-pattern: '-bk__(.*).xml'
-
-  - wait: ~
-    continue_on_failure: true
-
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/base.yml

@@ -0,0 +1,155 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: c2-16
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Default CI Group'
+    parallelism: 13
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    key: default-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Docker CI Group'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    key: default-cigroup-docker
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
+    label: 'OSS CI Group'
+    parallelism: 11
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    key: oss-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
+    label: 'OSS Accessibility Tests'
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
+    label: 'Default Accessibility Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_firefox.sh
+    label: 'OSS Firefox Tests'
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
+    label: 'Default Firefox Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_misc.sh
+    label: 'OSS Misc Functional Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
+    label: 'Saved Object Field Metrics'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/jest_integration.sh
+    label: 'Jest Integration Tests'
+    agents:
+      queue: n2-4
+    timeout_in_minutes: 120
+    key: jest-integration
+
+  - command: .buildkite/scripts/steps/test/api_integration.sh
+    label: 'API Integration Tests'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 120
+    key: api-integration
+
+  - command: .buildkite/scripts/steps/test/jest.sh
+    label: 'Jest Tests'
+    agents:
+      queue: c2-16
+    timeout_in_minutes: 120
+    key: jest
+
+  - command: .buildkite/scripts/steps/lint.sh
+    label: 'Linting'
+    agents:
+      queue: n2-2
+    key: linting
+
+  - command: .buildkite/scripts/steps/checks.sh
+    label: 'Checks'
+    agents:
+      queue: c2-4
+    key: checks
+
+  - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
+    label: 'Build Storybooks'
+    agents:
+      queue: c2-4
+    key: storybooks

.buildkite/pipelines/pull_request/post_build.yml

@@ -0,0 +1,6 @@
+steps:
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build