diff --git a/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts new file mode 100644 index 0000000000000..0464894d802d8 --- /dev/null +++ b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +import { readConfig } from 'jest-config'; +import { SearchSource } from 'jest'; +import Runtime from 'jest-runtime'; +import { resolve } from 'path'; +import { getKibanaDir, runBatchedPromises } from '#pipeline-utils'; + +export async function getTestsFromJestConfig(configPath: string): Promise { + try { + const emptyArgv = { + $0: '', + _: [], + }; + const config = await readConfig(emptyArgv, configPath); + const searchSource = new SearchSource( + await Runtime.createContext(config.projectConfig, { + maxWorkers: 1, + watchman: false, + watch: false, + console: { + ...console, + warn() { + // ignore haste-map warnings + }, + }, + }) + ); + + const results = await searchSource.getTestPaths(config.globalConfig, config.projectConfig); + return results.tests.map((t) => t.path); + } catch (error) { + console.error( + `Error while resolving test files from config: ${configPath} - validate your config.` + ); + throw error; + } +} + +export async function filterEmptyJestConfigs( + jestUnitConfigsWithEmpties: string[], + maxParallelism = 1 +): Promise { + const promiseThunks = jestUnitConfigsWithEmpties.map((configPath) => async () => { + const kibanaRelativePath = resolve(getKibanaDir(), configPath); + const testFiles = await getTestsFromJestConfig(kibanaRelativePath); + return testFiles?.length > 0 ? [configPath] : []; + }); + const nonEmptyConfigPaths = await runBatchedPromises(promiseThunks, maxParallelism); + // flat-mapping works better type-wise than filtering an Array + return nonEmptyConfigPaths.flat(); +} diff --git a/.buildkite/pipeline-utils/ci-stats/index.ts b/.buildkite/pipeline-utils/ci-stats/index.ts index aef42573ef474..d71ddabbc927f 100644 --- a/.buildkite/pipeline-utils/ci-stats/index.ts +++ b/.buildkite/pipeline-utils/ci-stats/index.ts @@ -12,3 +12,4 @@ export * from './on_complete'; export * from './on_metrics_viable'; export * from './on_start'; export * from './pick_test_group_run_order'; +export * from './get_tests_from_config'; diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 7b2d0b1bc8b5d..ade16fe40ad0c 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -8,6 +8,7 @@ */ import * as Fs from 'fs'; +import os from 'os'; import * as globby from 'globby'; import minimatch from 'minimatch'; @@ -21,207 +22,18 @@ import { CiStatsClient } from './client'; import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json'; import { serverless, stateful } from '../../ftr_configs_manifests.json'; -import { collectEnvFromLabels, expandAgentQueue } from '#pipeline-utils'; +import { filterEmptyJestConfigs } from './get_tests_from_config'; +import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils'; const ALL_FTR_MANIFEST_REL_PATHS = serverless.concat(stateful); type RunGroup = TestGroupRunOrderResponse['types'][0]; - -const getRequiredEnv = (name: string) => { - const value = process.env[name]; - if (typeof value !== 'string' || !value) { - throw new Error(`Missing required environment variable "${name}"`); - } - return value; -}; - -function getRunGroups(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup[] { - const types = allTypes.filter((t) => t.type === typeName); - if (!types.length) { - throw new Error(`missing test group run order for group [${typeName}]`); - } - - const uniqueTooLongMin = [ - ...new Set( - types.map((t) => t.tooLongMin).filter((value): value is number => typeof value === 'number') - ), - ]; - const tooLongThresholdLabel = - uniqueTooLongMin.length > 0 - ? `configured warning threshold${ - uniqueTooLongMin.length === 1 ? ` of ${uniqueTooLongMin[0]} minutes` : '' - }` - : 'maximum amount of time desired for a single CI job'; - - const misses = types.flatMap((t) => t.namesWithoutDurations); - if (misses.length > 0) { - bk.setAnnotation( - `test-group-missing-durations:${typeName}`, - 'warning', - [ - misses.length === 1 - ? `The following "${typeName}" config doesn't have a recorded time in ci-stats so the automatically-determined test groups might be a little unbalanced.` - : `The following "${typeName}" configs don't have recorded times in ci-stats so the automatically-determined test groups might be a little unbalanced.`, - misses.length === 1 - ? `If this is a new config then this warning can be ignored as times will be reported soon.` - : `If these are new configs then this warning can be ignored as times will be reported soon.`, - misses.length === 1 - ? `The other possibility is that there aren't any tests in this config, so times are never reported.` - : `The other possibility is that there aren't any tests in these configs, so times are never reported.`, - 'Empty test configs should be removed', - '', - ...misses.map((n) => ` - ${n}`), - ].join('\n') - ); - } - - const tooLongs = types.flatMap((t) => t.tooLong ?? []); - if (tooLongs.length > 0) { - bk.setAnnotation( - `test-group-too-long:${typeName}`, - 'warning', - [ - tooLongs.length === 1 - ? `The following "${typeName}" config has a duration that exceeds the ${tooLongThresholdLabel}. ` + - `This is not an error, and if you don't own this config then you can ignore this warning. ` + - `If you own this config please split it up ASAP and ask Operations if you have questions about how to do that.` - : `The following "${typeName}" configs have durations that exceed the ${tooLongThresholdLabel}. ` + - `This is not an error, and if you don't own any of these configs then you can ignore this warning.` + - `If you own any of these configs please split them up ASAP and ask Operations if you have questions about how to do that.`, - '', - ...tooLongs.map(({ config, durationMin }) => ` - ${config}: ${durationMin} minutes`), - ].join('\n') - ); - } - - return types; -} - -function getRunGroup(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup { - const groups = getRunGroups(bk, allTypes, typeName); - if (groups.length !== 1) { - throw new Error(`expected to find exactly 1 "${typeName}" run group`); - } - return groups[0]; -} - -function getTrackedBranch(): string { - let pkg; - try { - pkg = JSON.parse(Fs.readFileSync('package.json', 'utf8')); - } catch (_) { - const error = _ instanceof Error ? _ : new Error(`${_} thrown`); - throw new Error(`unable to read kibana's package.json file: ${error.message}`); - } - - const branch = pkg.branch; - if (typeof branch !== 'string') { - throw new Error('missing `branch` field from package.json file'); - } - - return branch; -} - -function isObj(x: unknown): x is Record { - return typeof x === 'object' && x !== null; -} - interface FtrConfigsManifest { defaultQueue?: string; disabled?: string[]; enabled?: Array; } -function getEnabledFtrConfigs(patterns?: string[], solutions?: string[]) { - const configs: { - enabled: Array; - defaultQueue: string | undefined; - } = { enabled: [], defaultQueue: undefined }; - const uniqueQueues = new Set(); - - const mappedSolutions = solutions?.map((s) => (s === 'observability' ? 'oblt' : s)); - for (const manifestRelPath of ALL_FTR_MANIFEST_REL_PATHS) { - if ( - mappedSolutions && - !( - mappedSolutions.some((s) => manifestRelPath.includes(`ftr_${s}_`)) || - // When applying the solution filter, still allow platform tests - manifestRelPath.includes('ftr_platform_') || - manifestRelPath.includes('ftr_base_') - ) - ) { - continue; - } - try { - const ymlData = loadYaml(Fs.readFileSync(manifestRelPath, 'utf8')); - if (!isObj(ymlData)) { - throw new Error('expected yaml file to parse to an object'); - } - const manifest = ymlData as FtrConfigsManifest; - - configs.enabled.push(...(manifest?.enabled ?? [])); - if (manifest.defaultQueue) { - uniqueQueues.add(manifest.defaultQueue); - } - } catch (_) { - const error = _ instanceof Error ? _ : new Error(`${_} thrown`); - throw new Error(`unable to parse ${manifestRelPath} file: ${error.message}`); - } - } - - try { - if (configs.enabled.length === 0) { - throw new Error('expected yaml files to have at least 1 "enabled" key'); - } - if (uniqueQueues.size !== 1) { - throw Error( - `FTR manifest yml files should define the same 'defaultQueue', but found different ones: ${[ - ...uniqueQueues, - ].join(' ')}` - ); - } - configs.defaultQueue = uniqueQueues.values().next().value; - - if ( - !Array.isArray(configs.enabled) || - !configs.enabled.every( - (p): p is string | { [configPath: string]: { queue: string } } => - typeof p === 'string' || - (isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string')) - ) - ) { - throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n - - {configPath}: - queue: {queueName}`); - } - if (typeof configs.defaultQueue !== 'string') { - throw new Error('expected yaml file to have a string "defaultQueue" key'); - } - - const defaultQueue = configs.defaultQueue; - const ftrConfigsByQueue = new Map(); - for (const enabled of configs.enabled) { - const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0]; - const queue = isObj(enabled) ? enabled[path].queue : defaultQueue; - - if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) { - continue; - } - - const group = ftrConfigsByQueue.get(queue); - if (group) { - group.push(path); - } else { - ftrConfigsByQueue.set(queue, [path]); - } - } - return { defaultQueue, ftrConfigsByQueue }; - } catch (_) { - const error = _ instanceof Error ? _ : new Error(`${_} thrown`); - throw new Error(`unable to collect enabled FTR configs: ${error.message}`); - } -} - export async function pickTestGroupRunOrder() { const bk = new BuildkiteClient(); const ciStats = new CiStatsClient(); @@ -352,13 +164,17 @@ export async function pickTestGroupRunOrder() { ); }; - const jestUnitConfigs = LIMIT_CONFIG_TYPE.includes('unit') + const jestUnitConfigsWithEmpties = LIMIT_CONFIG_TYPE.includes('unit') ? globby.sync(getJestConfigGlobs(['**/jest.config.js', '!**/__fixtures__/**']), { cwd: process.cwd(), absolute: false, ignore: [...DISABLED_JEST_CONFIGS, '**/node_modules/**'], }) : []; + const jestUnitConfigs = await filterEmptyJestConfigs( + jestUnitConfigsWithEmpties, + os.availableParallelism() + ); const jestIntegrationConfigs = LIMIT_CONFIG_TYPE.includes('integration') ? globby.sync(getJestConfigGlobs(['**/jest.integration.config.js', '!**/__fixtures__/**']), { @@ -608,6 +424,7 @@ export async function pickTestGroupRunOrder() { : [], ].flat(); +<<<<<<< HEAD // Register cancelable child keys before uploading so a concurrent gate failure // can discover and short-circuit these jobs immediately. if (unit.count > 0) { @@ -624,4 +441,173 @@ export async function pickTestGroupRunOrder() { // upload the step definitions to Buildkite bk.uploadSteps(steps); +======= +function getRunGroups(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup[] { + const types = allTypes.filter((t) => t.type === typeName); + if (!types.length) { + throw new Error(`missing test group run order for group [${typeName}]`); + } + + const misses = types.flatMap((t) => t.namesWithoutDurations); + if (misses.length > 0) { + bk.setAnnotation( + `test-group-missing-durations:${typeName}`, + 'warning', + [ + misses.length === 1 + ? `The following "${typeName}" config doesn't have a recorded time in ci-stats so the automatically-determined test groups might be a little unbalanced.` + : `The following "${typeName}" configs don't have recorded times in ci-stats so the automatically-determined test groups might be a little unbalanced.`, + misses.length === 1 + ? `If this is a new config then this warning can be ignored as times will be reported soon.` + : `If these are new configs then this warning can be ignored as times will be reported soon.`, + misses.length === 1 + ? `The other possibility is that there aren't any tests in this config, so times are never reported.` + : `The other possibility is that there aren't any tests in these configs, so times are never reported.`, + 'Empty test configs should be removed', + '', + ...misses.map((n) => ` - ${n}`), + ].join('\n') + ); + } + + const tooLongs = types.flatMap((t) => t.tooLong ?? []); + if (tooLongs.length > 0) { + bk.setAnnotation( + `test-group-too-long:${typeName}`, + 'warning', + [ + tooLongs.length === 1 + ? `The following "${typeName}" config has a duration that exceeds the maximum amount of time desired for a single CI job. ` + + `This is not an error, and if you don't own this config then you can ignore this warning. ` + + `If you own this config please split it up ASAP and ask Operations if you have questions about how to do that.` + : `The following "${typeName}" configs have durations that exceed the maximum amount of time desired for a single CI job. ` + + `This is not an error, and if you don't own any of these configs then you can ignore this warning.` + + `If you own any of these configs please split them up ASAP and ask Operations if you have questions about how to do that.`, + '', + ...tooLongs.map(({ config, durationMin }) => ` - ${config}: ${durationMin} minutes`), + ].join('\n') + ); + } + + return types; +} + +function getRunGroup(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup { + const groups = getRunGroups(bk, allTypes, typeName); + if (groups.length !== 1) { + throw new Error(`expected to find exactly 1 "${typeName}" run group`); + } + return groups[0]; +} + +function getTrackedBranch(): string { + let pkg; + try { + pkg = JSON.parse(Fs.readFileSync('package.json', 'utf8')); + } catch (_) { + const error = _ instanceof Error ? _ : new Error(`${_} thrown`); + throw new Error(`unable to read kibana's package.json file: ${error.message}`); + } + + const branch = pkg.branch; + if (typeof branch !== 'string') { + throw new Error('missing `branch` field from package.json file'); + } + + return branch; +} + +function isObj(x: unknown): x is Record { + return typeof x === 'object' && x !== null; +} + +function getEnabledFtrConfigs(patterns?: string[], solutions?: string[]) { + const configs: { + enabled: Array; + defaultQueue: string | undefined; + } = { enabled: [], defaultQueue: undefined }; + const uniqueQueues = new Set(); + + const mappedSolutions = solutions?.map((s) => (s === 'observability' ? 'oblt' : s)); + for (const manifestRelPath of ALL_FTR_MANIFEST_REL_PATHS) { + if ( + mappedSolutions && + !( + mappedSolutions.some((s) => manifestRelPath.includes(`ftr_${s}_`)) || + // When applying the solution filter, still allow platform tests + manifestRelPath.includes('ftr_platform_') || + manifestRelPath.includes('ftr_base_') + ) + ) { + continue; + } + try { + const ymlData = loadYaml(Fs.readFileSync(manifestRelPath, 'utf8')); + if (!isObj(ymlData)) { + throw new Error('expected yaml file to parse to an object'); + } + const manifest = ymlData as FtrConfigsManifest; + + configs.enabled.push(...(manifest?.enabled ?? [])); + if (manifest.defaultQueue) { + uniqueQueues.add(manifest.defaultQueue); + } + } catch (_) { + const error = _ instanceof Error ? _ : new Error(`${_} thrown`); + throw new Error(`unable to parse ${manifestRelPath} file: ${error.message}`); + } + } + + try { + if (configs.enabled.length === 0) { + throw new Error('expected yaml files to have at least 1 "enabled" key'); + } + if (uniqueQueues.size !== 1) { + throw Error( + `FTR manifest yml files should define the same 'defaultQueue', but found different ones: ${[ + ...uniqueQueues, + ].join(' ')}` + ); + } + configs.defaultQueue = uniqueQueues.values().next().value; + + if ( + !Array.isArray(configs.enabled) || + !configs.enabled.every( + (p): p is string | { [configPath: string]: { queue: string } } => + typeof p === 'string' || + (isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string')) + ) + ) { + throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n + - {configPath}: + queue: {queueName}`); + } + if (typeof configs.defaultQueue !== 'string') { + throw new Error('expected yaml file to have a string "defaultQueue" key'); + } + + const defaultQueue = configs.defaultQueue; + const ftrConfigsByQueue = new Map(); + for (const enabled of configs.enabled) { + const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0]; + const queue = isObj(enabled) ? enabled[path].queue : defaultQueue; + + if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) { + continue; + } + + const group = ftrConfigsByQueue.get(queue); + if (group) { + group.push(path); + } else { + ftrConfigsByQueue.set(queue, [path]); + } + } + return { defaultQueue, ftrConfigsByQueue }; + } catch (_) { + const error = _ instanceof Error ? _ : new Error(`${_} thrown`); + throw new Error(`unable to collect enabled FTR configs: ${error.message}`); + } +>>>>>>> 48890453f76f ([CI] Filter empty jest tests before grouping (#242440)) } diff --git a/.buildkite/pipeline-utils/scout/index.ts b/.buildkite/pipeline-utils/scout/index.ts index 7ab608f2dcde6..16f8f46fd0843 100644 --- a/.buildkite/pipeline-utils/scout/index.ts +++ b/.buildkite/pipeline-utils/scout/index.ts @@ -8,6 +8,9 @@ */ export * from './pick_scout_test_group_run_order'; +<<<<<<< HEAD export * from './paths'; export * from './test_tracks'; export * from './test_distribution_strategies'; +======= +>>>>>>> 48890453f76f ([CI] Filter empty jest tests before grouping (#242440)) diff --git a/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts b/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts index 29bacaf49123d..85266cb812627 100644 --- a/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/scout/pick_scout_test_group_run_order.ts @@ -8,6 +8,7 @@ */ import Fs from 'fs'; +<<<<<<< HEAD import { expandAgentQueue } from '../agent_images'; import { BuildkiteClient, type BuildkiteStep } from '../buildkite'; import { collectEnvFromLabels } from '../pr_labels'; @@ -25,6 +26,19 @@ export interface ModuleDiscoveryInfo { serverRunFlags: string[]; usesParallelWorkers: boolean; }[]; +======= +import { BuildkiteClient, type BuildkiteStep } from '../buildkite'; +import { collectEnvFromLabels } from '../pr_labels'; +import { expandAgentQueue } from '../agent_images'; +import { getRequiredEnv } from '#pipeline-utils'; + +interface ScoutTestDiscoveryConfig { + group: string; + path: string; + usesParallelWorkers: boolean; + configs: string[]; + type: 'plugin' | 'package'; +>>>>>>> 48890453f76f ([CI] Filter empty jest tests before grouping (#242440)) } // Collect environment variables to pass through to test execution steps @@ -41,15 +55,26 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) { throw new Error(`Scout configs file not found at ${scoutConfigsPath}`); } +<<<<<<< HEAD const modulesWithTests = JSON.parse( Fs.readFileSync(scoutConfigsPath, 'utf-8') ) as ModuleDiscoveryInfo[]; if (modulesWithTests.length === 0) { +======= + const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8')) as Record< + string, + ScoutTestDiscoveryConfig + >; + const pluginsOrPackagesWithScoutTests: string[] = Object.keys(rawScoutConfigs); + + if (pluginsOrPackagesWithScoutTests.length === 0) { +>>>>>>> 48890453f76f ([CI] Filter empty jest tests before grouping (#242440)) // no scout configs found, nothing to need to upload steps return; } +<<<<<<< HEAD const SCOUT_CONFIGS_DEPS = process.env.SCOUT_CONFIGS_DEPS !== undefined ? process.env.SCOUT_CONFIGS_DEPS.split(',') @@ -109,4 +134,43 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) { // upload the step definitions to Buildkite bk.uploadSteps(steps); +======= + const scoutCiRunGroups = pluginsOrPackagesWithScoutTests.map((name) => ({ + label: `Scout: [ ${rawScoutConfigs[name].group} / ${name} ] ${rawScoutConfigs[name].type}`, + key: name, + agents: expandAgentQueue(rawScoutConfigs[name].usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'), + group: rawScoutConfigs[name].group, + })); + + // upload the step definitions to Buildkite + bk.uploadSteps( + [ + { + group: 'Scout Configs', + key: 'scout-configs', + depends_on: ['build_scout_tests'], + steps: scoutCiRunGroups.map( + ({ label, key, group, agents }): BuildkiteStep => ({ + label, + command: getRequiredEnv('SCOUT_CONFIGS_SCRIPT'), + timeout_in_minutes: 60, + agents, + env: { + SCOUT_CONFIG_GROUP_KEY: key, + SCOUT_CONFIG_GROUP_TYPE: group, + ...envFromlabels, + ...scoutExtraEnv, + }, + retry: { + automatic: [ + { exit_status: '10', limit: 1 }, + { exit_status: '*', limit: 3 }, + ], + }, + }) + ), + }, + ].flat() + ); +>>>>>>> 48890453f76f ([CI] Filter empty jest tests before grouping (#242440)) } diff --git a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts index b44bbf27e1781..b0d47df56ce9b 100644 --- a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts +++ b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts @@ -19,6 +19,8 @@ import { getAllFtrConfigsAndManifests } from './ftr_configs_manifest'; const THIS_PATH = 'src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts'; +const IGNORED_FOLDERS = ['.buildkite/']; + const IGNORED_PATHS = [ THIS_PATH, 'src/platform/packages/shared/kbn-test/src/jest/run_check_jest_configs_cli.ts', @@ -68,6 +70,10 @@ export async function runCheckFtrConfigsCli() { return false; } + if (IGNORED_FOLDERS.some((folder) => file.startsWith(Path.resolve(REPO_ROOT, folder)))) { + return false; + } + // playwright config files if (file.match(/\/*playwright*.config.ts$/)) { return false;