diff --git a/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts b/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts index 184ea80f9881a..dd4cabb30bd65 100644 --- a/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts @@ -49,6 +49,13 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) { return; } + const SCOUT_CONFIGS_DEPS = + process.env.SCOUT_CONFIGS_DEPS !== undefined + ? process.env.SCOUT_CONFIGS_DEPS.split(',') + .map((t) => t.trim()) + .filter(Boolean) + : ['build_scout_tests']; + const scoutCiRunGroups = modulesWithTests.map((module) => { // Check if any config in this module uses parallel workers const usesParallelWorkers = module.configs.some((config) => config.usesParallelWorkers); @@ -67,7 +74,7 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) { { group: 'Scout Configs', key: 'scout-configs', - depends_on: ['build_scout_tests'], + depends_on: SCOUT_CONFIGS_DEPS, steps: scoutCiRunGroups.map( ({ label, key, group, agents }): BuildkiteStep => ({ label, diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 5746c76ae25ea..be6fb28875762 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,24 +1,4 @@ steps: - - command: .buildkite/scripts/lifecycle/pre_build.sh - label: Pre-Build - timeout_in_minutes: 10 - id: pre_build - agents: - machineType: n2-standard-2 - - - wait - - - command: .buildkite/scripts/steps/store_cache.sh - label: Store Cache for build - timeout_in_minutes: 10 - id: store_cache - soft_fail: true - depends_on: - - terrazzo-initial-pipeline-upload - agents: - machineType: n2-standard-2 - diskSizeGb: 95 - - command: .buildkite/scripts/steps/build_kibana.sh label: Build Kibana Distribution agents: @@ -123,7 +103,38 @@ steps: - exit_status: '-1' limit: 3 - - wait + - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh + label: 'Pick Test Group Run Order' + agents: + machineType: n2-standard-2 + diskSizeGb: 115 + timeout_in_minutes: 10 + env: + JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' + JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh' + JEST_CONFIGS_DEPS: 'build,quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types' + FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh' + FTR_CONFIGS_DEPS: 'build,quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types' + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh + label: 'Scout Test Run Builder' + agents: + machineType: c3d-standard-8 + key: build_scout_tests + timeout_in_minutes: 20 + depends_on: + - build + env: + SCOUT_CONFIGS_DEPS: 'quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types,build_scout_tests' + SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh' + retry: + automatic: + - exit_status: '*' + limit: 1 - command: .buildkite/scripts/steps/ci_stats_ready.sh label: Mark CI Stats as ready @@ -146,6 +157,13 @@ steps: spotZones: us-central1-f,us-central1-c,us-central1-a diskSizeGb: 105 key: build_api_docs + depends_on: + - build + - check_types + - linting + - linting_with_types + - quick_checks + - check_oas_snapshot timeout_in_minutes: 90 retry: automatic: diff --git a/.buildkite/pipelines/pull_request/base_merged_phases.yml b/.buildkite/pipelines/pull_request/base_merged_phases.yml deleted file mode 100644 index fcb37b46e54f9..0000000000000 --- a/.buildkite/pipelines/pull_request/base_merged_phases.yml +++ /dev/null @@ -1,143 +0,0 @@ -steps: - - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution - agents: - machineType: c3d-standard-16-lssd - zones: northamerica-northeast2-b,us-central1-c,us-central1-a - key: build - if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" - timeout_in_minutes: 90 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/quick_checks.sh - label: 'Quick Checks' - agents: - machineType: n2-highcpu-8 - preemptible: true - diskSizeGb: 115 - key: quick_checks - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/checks.sh - label: 'Checks' - key: checks - agents: - machineType: n2-standard-2 - preemptible: true - diskSizeGb: 115 - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/lint.sh - label: 'Linting' - agents: - machineType: n2-standard-16 - preemptible: true - diskSizeGb: 115 - key: linting - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/lint_with_types.sh - label: 'Linting (with types)' - agents: - machineType: n2-standard-32 - preemptible: true - diskSizeGb: 115 - key: linting_with_types - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/checks/capture_oas_snapshot.sh - label: 'Check OAS Snapshot' - agents: - machineType: n2-standard-4 - preemptible: true - key: check_oas_snapshot - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/typecheck/check_types.sh - label: 'Check Types' - agents: - machineType: c4-standard-4 - diskType: hyperdisk-balanced - preemptible: true - spotZones: northamerica-northeast2-b,us-central1-c,us-central1-a - diskSizeGb: 115 - key: check_types - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh - label: 'Pick Test Group Run Order' - agents: - machineType: n2-standard-2 - diskSizeGb: 115 - timeout_in_minutes: 10 - env: - JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' - JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh' - JEST_CONFIGS_DEPS: 'check_types' - FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh' - FTR_CONFIGS_DEPS: 'build,check_types' - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/ci_stats_ready.sh - label: Mark CI Stats as ready - agents: - machineType: n2-standard-2 - diskSizeGb: 115 - timeout_in_minutes: 10 - depends_on: - - build - - build_api_docs - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh - label: 'Build API Docs' - agents: - machineType: n2-highmem-4 - preemptible: true - diskSizeGb: 115 - key: build_api_docs - depends_on: - - build - - check_types - - linting - - linting_with_types - - quick_checks - - check_oas_snapshot - timeout_in_minutes: 90 - retry: - automatic: - - exit_status: '-1' - limit: 3 diff --git a/.buildkite/pipelines/pull_request/pick_test_groups.yml b/.buildkite/pipelines/pull_request/pick_test_groups.yml deleted file mode 100644 index fef4d115945b5..0000000000000 --- a/.buildkite/pipelines/pull_request/pick_test_groups.yml +++ /dev/null @@ -1,15 +0,0 @@ -steps: - - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh - label: 'Pick Test Group Run Order' - agents: - machineType: n2-standard-2 - diskSizeGb: 115 - timeout_in_minutes: 10 - env: - JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' - JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh' - FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh' - retry: - automatic: - - exit_status: '*' - limit: 1 diff --git a/.buildkite/pipelines/pull_request/scout_tests.yml b/.buildkite/pipelines/pull_request/scout_tests.yml deleted file mode 100644 index 26991518edfe0..0000000000000 --- a/.buildkite/pipelines/pull_request/scout_tests.yml +++ /dev/null @@ -1,21 +0,0 @@ -steps: - - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh - label: 'Scout Test Run Builder' - agents: - machineType: n2-standard-4 - key: build_scout_tests - timeout_in_minutes: 10 - depends_on: - - build - - quick_checks - - checks - - linting - - linting_with_types - - check_types - - check_oas_snapshot - env: - SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh' - retry: - automatic: - - exit_status: '*' - limit: 1 diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.sh b/.buildkite/scripts/pipelines/pull_request/pipeline.sh index 8baf5e67037cb..79f1635f33551 100755 --- a/.buildkite/scripts/pipelines/pull_request/pipeline.sh +++ b/.buildkite/scripts/pipelines/pull_request/pipeline.sh @@ -2,11 +2,10 @@ set -euo pipefail -if [[ "${GITHUB_PR_LABELS:-}" == *"ci:beta-faster-pr-build"* ]]; then - # Quietly upload the cache-warmup step before everything else, - # in order to run it sooner than others, and have a warm cache by the time other steps run. - (buildkite-agent pipeline upload .buildkite/pipelines/pull_request/store_moon_cache.yml > /dev/null \ - && echo "Uploaded cache-warmup step" >&2) || echo "Failed to upload cache-warmup step" >&2 -fi +# Quietly upload the cache-warmup step before everything else, +# in order to run it sooner than others, and have a warm cache by the time other steps run. +(buildkite-agent pipeline upload .buildkite/pipelines/pull_request/store_moon_cache.yml > /dev/null \ + && echo "Uploaded cache-warmup step" >&2) || echo "Failed to upload cache-warmup step" >&2 + ts-node .buildkite/scripts/pipelines/pull_request/pipeline.ts diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts index 3b216bd3d2595..21fcd84a4f952 100644 --- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts +++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts @@ -59,20 +59,13 @@ const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new R return; } - if (GITHUB_PR_LABELS.includes('ci:beta-faster-pr-build')) { - await runPreBuild(); - pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base_merged_phases.yml', false)); - } else { - pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false)); - pipeline.push(getPipeline('.buildkite/pipelines/pull_request/pick_test_groups.yml')); - } + await runPreBuild(); + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false)); if (prHasFIPSLabel()) { pipeline.push(getPipeline('.buildkite/pipelines/fips/verify_fips_enabled.yml')); } - pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml')); - if (await doAnyChangesMatch([/^src\/platform\/packages\/private\/kbn-handlebars/])) { pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml')); }