diff --git a/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.test.ts b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.test.ts new file mode 100644 index 0000000000000..e127fc8d8b23e --- /dev/null +++ b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import path from 'path'; +import * as globby from 'globby'; + +const REPO_ROOT = path.resolve(__dirname, '../../..'); + +/** + * Extensions covered by the glob in filterEmptyJestConfigs. + * If a jest config uses a testMatch/testRegex that doesn't end in one of these, + * the fast glob could miss test files and silently skip the config in CI. + */ +const COVERED_EXTENSIONS = /\.(test|spec)\.(ts|tsx|js|jsx|mjs)$/; + +describe('filterEmptyJestConfigs glob coverage', () => { + const allConfigs = globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], { + cwd: REPO_ROOT, + absolute: true, + ignore: ['**/node_modules/**'], + }); + + it('found jest configs to validate', () => { + expect(allConfigs.length).toBeGreaterThan(100); + }); + + it('every jest config testMatch/testRegex is covered by the glob patterns', () => { + const uncovered: string[] = []; + + for (const configPath of allConfigs) { + let config: Record; + try { + config = require(configPath); + } catch { + continue; + } + + const testMatch = config.testMatch as string[] | undefined; + const testRegex = config.testRegex as string | string[] | undefined; + + if (testMatch) { + for (const pattern of testMatch) { + // Extract the file extension portion from the glob pattern + const extMatch = pattern.match(/\*\.([\w|{},]+)$/); + if (extMatch) { + const extensions = extMatch[1].replace(/[{}]/g, '').split(','); + for (const ext of extensions) { + const testFilename = `example.test.${ext}`; + if (!COVERED_EXTENSIONS.test(testFilename)) { + const specFilename = `example.spec.${ext}`; + if (!COVERED_EXTENSIONS.test(specFilename)) { + uncovered.push( + `${path.relative( + REPO_ROOT, + configPath + )}: testMatch extension ".${ext}" not covered` + ); + } + } + } + } + } + } + + if (testRegex) { + const regexes = Array.isArray(testRegex) ? testRegex : [testRegex]; + for (const regex of regexes) { + // Verify the regex would match files ending in .test.ts or .spec.ts etc. + const sampleFiles = [ + 'foo.test.ts', + 'foo.test.tsx', + 'foo.test.js', + 'foo.test.jsx', + 'foo.test.mjs', + 'foo.spec.ts', + ]; + const re = new RegExp(regex); + const anyMatch = sampleFiles.some((f) => re.test(f)); + if (!anyMatch) { + uncovered.push( + `${path.relative( + REPO_ROOT, + configPath + )}: testRegex "${regex}" doesn't match standard test file names` + ); + } + } + } + } + + if (uncovered.length > 0) { + fail( + `The following jest configs use patterns not covered by filterEmptyJestConfigs glob.\n` + + `Update TEST_FILE_PATTERNS in get_tests_from_config.ts to cover them:\n\n` + + uncovered.map((u) => ` - ${u}`).join('\n') + ); + } + }); +}); diff --git a/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts index 0464894d802d8..6c615b9ad01f8 100644 --- a/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts +++ b/.buildkite/pipeline-utils/ci-stats/get_tests_from_config.ts @@ -6,53 +6,46 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -import { readConfig } from 'jest-config'; -import { SearchSource } from 'jest'; -import Runtime from 'jest-runtime'; -import { resolve } from 'path'; -import { getKibanaDir, runBatchedPromises } from '#pipeline-utils'; +import { dirname, resolve } from 'path'; +import * as globby from 'globby'; +import { getKibanaDir } from '#pipeline-utils'; -export async function getTestsFromJestConfig(configPath: string): Promise { - try { - const emptyArgv = { - $0: '', - _: [], - }; - const config = await readConfig(emptyArgv, configPath); - const searchSource = new SearchSource( - await Runtime.createContext(config.projectConfig, { - maxWorkers: 1, - watchman: false, - watch: false, - console: { - ...console, - warn() { - // ignore haste-map warnings - }, - }, - }) - ); +const TEST_FILE_PATTERNS = ['**/*.test.{ts,tsx,js,jsx,mjs}', '**/*.spec.{ts,tsx,js,jsx,mjs}']; - const results = await searchSource.getTestPaths(config.globalConfig, config.projectConfig); - return results.tests.map((t) => t.path); - } catch (error) { - console.error( - `Error while resolving test files from config: ${configPath} - validate your config.` - ); - throw error; +// Loaded lazily because getKibanaDir() isn't available at module-init time. +let ignorePatterns: string[]; +function getIgnorePatterns(): string[] { + if (!ignorePatterns) { + // Integration test patterns loaded from the Jest integration preset so this + // stays in sync automatically if that preset ever changes. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const integrationPreset = require(resolve( + getKibanaDir(), + 'src/platform/packages/shared/kbn-test/jest_integration_node/jest-preset.js' + )); + ignorePatterns = ['**/node_modules/**', ...integrationPreset.testMatch]; } + return ignorePatterns; } -export async function filterEmptyJestConfigs( - jestUnitConfigsWithEmpties: string[], - maxParallelism = 1 -): Promise { - const promiseThunks = jestUnitConfigsWithEmpties.map((configPath) => async () => { - const kibanaRelativePath = resolve(getKibanaDir(), configPath); - const testFiles = await getTestsFromJestConfig(kibanaRelativePath); - return testFiles?.length > 0 ? [configPath] : []; +/** + * Fast check for whether a jest config's directory contains any test files. + * Uses a simple glob instead of Jest's full resolver (readConfig + Runtime.createContext + * + SearchSource.getTestPaths) which is ~20x slower across 1000+ configs. + */ +function hasTestFiles(configAbsPath: string): boolean { + const dir = dirname(configAbsPath); + const matches = globby.sync(TEST_FILE_PATTERNS, { + cwd: dir, + ignore: getIgnorePatterns(), + onlyFiles: true, }); - const nonEmptyConfigPaths = await runBatchedPromises(promiseThunks, maxParallelism); - // flat-mapping works better type-wise than filtering an Array - return nonEmptyConfigPaths.flat(); + return matches.length > 0; +} + +export function filterEmptyJestConfigs(jestUnitConfigsWithEmpties: string[]): string[] { + const kibanaDir = getKibanaDir(); + return jestUnitConfigsWithEmpties.filter((configPath) => + hasTestFiles(resolve(kibanaDir, configPath)) + ); } diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 9cdaf50218982..6a932881aacf7 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -8,7 +8,6 @@ */ import * as Fs from 'fs'; -import os from 'os'; import * as globby from 'globby'; import minimatch from 'minimatch'; @@ -219,10 +218,7 @@ export async function pickTestGroupRunOrder() { ignore: [...DISABLED_JEST_CONFIGS, '**/node_modules/**'], }) : []; - const jestUnitConfigsFiltered = await filterEmptyJestConfigs( - jestUnitConfigsWithEmpties, - os.availableParallelism() - ); + const jestUnitConfigsFiltered = filterEmptyJestConfigs(jestUnitConfigsWithEmpties); // Expand sharded unit configs (e.g. cases/jest.config.js) into shard-annotated entries let jestUnitConfigs = expandShardedJestConfigs(jestUnitConfigsFiltered);