[CI] Move faster beta build to default

.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts

@@ -49,6 +49,13 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
     return;
   }
 
+  const SCOUT_CONFIGS_DEPS =
+    process.env.SCOUT_CONFIGS_DEPS !== undefined
+      ? process.env.SCOUT_CONFIGS_DEPS.split(',')
+          .map((t) => t.trim())
+          .filter(Boolean)
+      : ['build_scout_tests'];
+
   const scoutCiRunGroups = modulesWithTests.map((module) => {
     // Check if any config in this module uses parallel workers
     const usesParallelWorkers = module.configs.some((config) => config.usesParallelWorkers);
@@ -67,7 +74,7 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
       {
         group: 'Scout Configs',
         key: 'scout-configs',
-        depends_on: ['build_scout_tests'],
+        depends_on: SCOUT_CONFIGS_DEPS,
         steps: scoutCiRunGroups.map(
           ({ label, key, group, agents }): BuildkiteStep => ({
             label,

.buildkite/pipelines/pull_request/base.yml

@@ -1,24 +1,4 @@
 steps:
-  - command: .buildkite/scripts/lifecycle/pre_build.sh
-    label: Pre-Build
-    timeout_in_minutes: 10
-    id: pre_build
-    agents:
-      machineType: n2-standard-2
-
-  - wait
-
-  - command: .buildkite/scripts/steps/store_cache.sh
-    label: Store Cache for build
-    timeout_in_minutes: 10
-    id: store_cache
-    soft_fail: true
-    depends_on:
-      - terrazzo-initial-pipeline-upload
-    agents:
-      machineType: n2-standard-2
-      diskSizeGb: 95
-
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution
     agents:
@@ -123,7 +103,38 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - wait
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
+    agents:
+      machineType: n2-standard-2
+      diskSizeGb: 115
+    timeout_in_minutes: 10
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      JEST_CONFIGS_DEPS: 'build,quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      FTR_CONFIGS_DEPS: 'build,quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
+    agents:
+      machineType: c3d-standard-8
+    key: build_scout_tests
+    timeout_in_minutes: 20
+    depends_on:
+      - build
+    env:
+      SCOUT_CONFIGS_DEPS: 'quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types,build_scout_tests'
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
@@ -146,6 +157,13 @@ steps:
       spotZones: us-central1-f,us-central1-c,us-central1-a
       diskSizeGb: 105
     key: build_api_docs
+    depends_on:
+      - build
+      - check_types
+      - linting
+      - linting_with_types
+      - quick_checks
+      - check_oas_snapshot
     timeout_in_minutes: 90
     retry:
       automatic:

.buildkite/scripts/pipelines/pull_request/pipeline.sh

@@ -2,11 +2,10 @@
 
 set -euo pipefail
 
-if [[ "${GITHUB_PR_LABELS:-}" == *"ci:beta-faster-pr-build"* ]]; then
-  # Quietly upload the cache-warmup step before everything else,
-  # in order to run it sooner than others, and have a warm cache by the time other steps run.
-  (buildkite-agent pipeline upload .buildkite/pipelines/pull_request/store_moon_cache.yml > /dev/null \
-   && echo "Uploaded cache-warmup step" >&2) || echo "Failed to upload cache-warmup step" >&2
-fi
+# Quietly upload the cache-warmup step before everything else,
+# in order to run it sooner than others, and have a warm cache by the time other steps run.
+(buildkite-agent pipeline upload .buildkite/pipelines/pull_request/store_moon_cache.yml > /dev/null \
+ && echo "Uploaded cache-warmup step" >&2) || echo "Failed to upload cache-warmup step" >&2
+
 
 ts-node .buildkite/scripts/pipelines/pull_request/pipeline.ts

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -59,20 +59,13 @@ const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new R
       return;
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:beta-faster-pr-build')) {
-      await runPreBuild();
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base_merged_phases.yml', false));
-    } else {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/pick_test_groups.yml'));
-    }
+    await runPreBuild();
+    pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
     if (prHasFIPSLabel()) {
       pipeline.push(getPipeline('.buildkite/pipelines/fips/verify_fips_enabled.yml'));
     }
 
-    pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
-
     if (await doAnyChangesMatch([/^src\/platform\/packages\/private\/kbn-handlebars/])) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
     }

src/platform/packages/shared/kbn-scout/src/cli/manifests.ts

@@ -46,30 +46,32 @@ async function updateScoutConfigManifests(
   const updatedConfigPaths: string[] = [];
 
   // Update manifests for files that are outdated
-  for (const config of testConfigs.all) {
-    expectedManifestPaths.push(config.manifest.path);
-    const configDirSHA1 = await getGitSHA1ForPath(path.dirname(config.path));
-
-    if (onlyOutdated && config.manifest.exists && config.manifest.sha1 === configDirSHA1) {
-      log.debug(` ✅ ${config.module.name} / ${config.category} / ${config.type}`);
-      continue;
-    }
-
-    if (config.manifest.exists) {
-      if (config.manifest.sha1 !== configDirSHA1) {
-        log.info(
-          `Manifest file is outdated for Scout test config at ${config.path} ` +
-            `(expected parent directory git object hash '${config.manifest.sha1}' but got '${configDirSHA1}')`
-        );
+  await Promise.all(
+    testConfigs.all.map(async (config) => {
+      expectedManifestPaths.push(config.manifest.path);
+      const configDirSHA1 = await getGitSHA1ForPath(path.dirname(config.path));
+
+      if (onlyOutdated && config.manifest.exists && config.manifest.sha1 === configDirSHA1) {
+        log.debug(` ✅ ${config.module.name} / ${config.category} / ${config.type}`);
+        return;
       }
-    } else {
-      log.info(`No manifest file found for Scout test config at ${config.path}`);
-    }
 
-    log.info(`Generating manifest for test config at '${config.path}'`);
-    await generateScoutConfigManifest(config.path, log);
-    updatedConfigPaths.push(config.path);
-  }
+      if (config.manifest.exists) {
+        if (config.manifest.sha1 !== configDirSHA1) {
+          log.info(
+            `Manifest file is outdated for Scout test config at ${config.path} ` +
+              `(expected parent directory git object hash '${config.manifest.sha1}' but got '${configDirSHA1}')`
+          );
+        }
+      } else {
+        log.info(`No manifest file found for Scout test config at ${config.path}`);
+      }
+
+      log.info(`Generating manifest for test config at '${config.path}'`);
+      await generateScoutConfigManifest(config.path, log);
+      updatedConfigPaths.push(config.path);
+    })
+  );
 
   if (removeDangling) {
     // Remove any manifest files that no longer have a corresponding test config