From 203f5228f95ff0c6856a51710a08cee99c88e2cf Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 23 Mar 2026 15:58:31 +0100 Subject: [PATCH 01/12] adopt affected-packages, hide the selective jest/jest-integration behind the label: ci:use-selective-testing --- .../affected-packages/README.md | 2 - .../pipeline-utils/affected-packages/const.ts | 32 ++++++++++++ .../pipeline-utils/affected-packages/index.ts | 24 ++++++--- .../affected-packages/list_affected | 1 - .../affected-packages/module_lookup.ts | 3 +- .../affected-packages/strategy_git.ts | 7 +-- .../affected-packages/strategy_moon.ts | 2 +- .../pipeline-utils/affected-packages/utils.ts | 11 +++++ .../ci-stats/pick_test_group_run_order.ts | 49 +++++++++++++++++-- 9 files changed, 108 insertions(+), 23 deletions(-) create mode 100644 .buildkite/pipeline-utils/affected-packages/const.ts diff --git a/.buildkite/pipeline-utils/affected-packages/README.md b/.buildkite/pipeline-utils/affected-packages/README.md index 9f9d33bc77ddc..0255754e54e1e 100644 --- a/.buildkite/pipeline-utils/affected-packages/README.md +++ b/.buildkite/pipeline-utils/affected-packages/README.md @@ -69,7 +69,6 @@ const affectedPackages = await getAffectedPackages( { strategy: 'git', // default, can also be 'moon' includeDownstream: true, - logging: false, ignorePatterns: ['**/*.md', 'docs/**'], ignoreUncategorizedChanges: false, } @@ -109,7 +108,6 @@ const filteredFiles = filterFilesByPackages( |------------------------------------|---------------------------------|--------------|----------------------| | `AFFECTED_STRATEGY` | `git`, `moon` | `git` | `git` | | `AFFECTED_DOWNSTREAM` | `true`, `false` | `false` | `true` | -| `AFFECTED_LOGGING` | `true`, `false` | `false` | `true` | | `AFFECTED_IGNORE` | comma-separated globs | — | — | | `AFFECTED_IGNORE_UNCATEGORIZED_CHANGES` | `true`, `false` | `false` | `false` | | `GITHUB_PR_MERGE_BASE` | any git ref | `origin/main`| — | diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts new file mode 100644 index 0000000000000..f04dab6512dd1 --- /dev/null +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export const UNCATEGORIZED_MODULE_ID = '[uncategorized]'; + +export const SELECTIVE_TESTS_LABEL = 'ci:use-selective-testing'; + +// TODO: find reasonable set of critical files for unit tests +export const CRITICAL_FILES_JEST_UNIT_TESTS = [ + 'scripts/jest.js', + 'scripts/jest_all.js', + 'package.json', + 'yarn.lock', + 'tsconfig.json', + 'src/platform/packages/shared/kbn-test/**/*', +]; + +// TODO: find reasonable set of critical files for integration tests +export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ + 'scripts/jest_integration.js', + 'scripts/jest_all.js', + 'package.json', + 'yarn.lock', + 'tsconfig.json', + 'src/platform/packages/shared/kbn-test/**/*', +]; diff --git a/.buildkite/pipeline-utils/affected-packages/index.ts b/.buildkite/pipeline-utils/affected-packages/index.ts index 181fb28ae42e1..83e553e7cd2e4 100644 --- a/.buildkite/pipeline-utils/affected-packages/index.ts +++ b/.buildkite/pipeline-utils/affected-packages/index.ts @@ -11,13 +11,15 @@ import { findModuleForPath } from './module_lookup'; import { getAffectedModulesGit } from './strategy_git'; import { getAffectedProjectsMoon } from './strategy_moon'; +export * from './const'; +export * from './utils'; + export interface AffectedPackagesConfig { - strategy: 'git' | 'moon'; - includeDownstream: boolean; - logging: boolean; + strategy?: 'git' | 'moon'; + includeDownstream?: boolean; /** Glob patterns for changed files to exclude before module resolution (git strategy only). */ - ignorePatterns: string[]; - ignoreUncategorizedChanges: boolean; + ignorePatterns?: string[]; + ignoreUncategorizedChanges?: boolean; } /** @@ -26,12 +28,19 @@ export interface AffectedPackagesConfig { */ export async function getAffectedPackages( mergeBase: string | undefined, - config: AffectedPackagesConfig = getConfigFromEnv() + configArgs: AffectedPackagesConfig = getConfigFromEnv() ): Promise> { if (!mergeBase) { throw new Error('No merge base found'); } + const config = { + strategy: configArgs.strategy ?? 'git', + includeDownstream: configArgs.includeDownstream ?? false, + ignorePatterns: configArgs.ignorePatterns ?? [], + ignoreUncategorizedChanges: configArgs.ignoreUncategorizedChanges ?? false, + }; + try { const affectedPackages = config.strategy === 'git' @@ -82,7 +91,6 @@ function getConfigFromEnv(): AffectedPackagesConfig { } const strategy = rawStrategy; const includeDownstream = process.env.AFFECTED_DOWNSTREAM !== 'false'; - const logging = process.env.AFFECTED_LOGGING !== 'false'; const ignorePatterns = (process.env.AFFECTED_IGNORE || '') .split(',') .map((p) => p.trim()) @@ -90,5 +98,5 @@ function getConfigFromEnv(): AffectedPackagesConfig { const ignoreUncategorizedChanges = process.env.AFFECTED_IGNORE_UNCATEGORIZED_CHANGES !== 'false'; - return { strategy, includeDownstream, logging, ignorePatterns, ignoreUncategorizedChanges }; + return { strategy, includeDownstream, ignorePatterns, ignoreUncategorizedChanges }; } diff --git a/.buildkite/pipeline-utils/affected-packages/list_affected b/.buildkite/pipeline-utils/affected-packages/list_affected index 74c610a423bfd..4e7b0ba9bb181 100755 --- a/.buildkite/pipeline-utils/affected-packages/list_affected +++ b/.buildkite/pipeline-utils/affected-packages/list_affected @@ -85,7 +85,6 @@ async function main() { const affectedPackages = await getAffectedPackages(mergeBase, { strategy, includeDownstream, - logging: false, ignorePatterns, ignoreUncategorizedChanges, }); diff --git a/.buildkite/pipeline-utils/affected-packages/module_lookup.ts b/.buildkite/pipeline-utils/affected-packages/module_lookup.ts index 5f5a5ad523984..5225e0c3a5ed9 100644 --- a/.buildkite/pipeline-utils/affected-packages/module_lookup.ts +++ b/.buildkite/pipeline-utils/affected-packages/module_lookup.ts @@ -12,8 +12,7 @@ import * as fs from 'fs'; import { execSync } from 'child_process'; import * as JSON5 from 'json5'; import { getKibanaDir } from '../utils'; - -export const UNCATEGORIZED_MODULE_ID = '[uncategorized]'; +import { UNCATEGORIZED_MODULE_ID } from './const'; export interface ModuleLookup { /** diff --git a/.buildkite/pipeline-utils/affected-packages/strategy_git.ts b/.buildkite/pipeline-utils/affected-packages/strategy_git.ts index a89644ae08171..d7f5c4a7510bb 100644 --- a/.buildkite/pipeline-utils/affected-packages/strategy_git.ts +++ b/.buildkite/pipeline-utils/affected-packages/strategy_git.ts @@ -9,11 +9,8 @@ import { execSync } from 'child_process'; import { getKibanaDir } from '../utils'; -import { - findModuleForPath, - buildModuleDownstreamGraph, - UNCATEGORIZED_MODULE_ID, -} from './module_lookup'; +import { findModuleForPath, buildModuleDownstreamGraph } from './module_lookup'; +import { UNCATEGORIZED_MODULE_ID } from './const'; import { filterIgnoredFiles } from './utils'; const isCI = !!process.env.CI?.match(/^(1|true)$/i); diff --git a/.buildkite/pipeline-utils/affected-packages/strategy_moon.ts b/.buildkite/pipeline-utils/affected-packages/strategy_moon.ts index 28b7a601f5748..448bf9da6c1d9 100644 --- a/.buildkite/pipeline-utils/affected-packages/strategy_moon.ts +++ b/.buildkite/pipeline-utils/affected-packages/strategy_moon.ts @@ -22,7 +22,7 @@ export function getAffectedProjectsMoon( const output = execSync(command, { cwd: REPO_ROOT, encoding: 'utf8', - maxBuffer: 10 * 1024 * 1024, // 10MB buffer + maxBuffer: 30 * 1024 * 1024, // 30MB buffer env: { ...process.env, MOON_BASE: mergeBase, diff --git a/.buildkite/pipeline-utils/affected-packages/utils.ts b/.buildkite/pipeline-utils/affected-packages/utils.ts index e8b128410548a..807f687dc77e6 100644 --- a/.buildkite/pipeline-utils/affected-packages/utils.ts +++ b/.buildkite/pipeline-utils/affected-packages/utils.ts @@ -19,3 +19,14 @@ export function filterIgnoredFiles(files: string[], patterns: string[]): string[ } return files.filter((file) => !patterns.some((p) => minimatch(file, p, { dot: true }))); } + +/** + * Returns true when any pattern matches any file in the list + */ +export function touchedCriticalFiles(files: string[], criticalFiles: string[]): boolean { + return ( + files.some((file) => + criticalFiles.some((criticalFile) => minimatch(file, criticalFile, { dot: true })) + ) && false // temporary disable + ); +} diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 6de2f7696eb30..83b14ecf66ea3 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -24,10 +24,19 @@ import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json'; import SHARDED_JEST_CONFIGS from '../../sharded_jest_configs.json'; import { serverless, stateful } from '../../ftr_configs_manifests.json'; import { filterEmptyJestConfigs } from './get_tests_from_config'; +import { + getAffectedPackages, + filterFilesByPackages, + SELECTIVE_TESTS_LABEL, + CRITICAL_FILES_JEST_UNIT_TESTS, + touchedCriticalFiles, +} from '../affected-packages'; import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils'; -const SHARD_ANNOTATION_SEP = '||shard='; +// TODO: this is always false on on-merge, when switching to enable this by default, check if this is a PR +const USE_SELECTIVE_TESTING = process.env.GITHUB_LABELS?.includes(SELECTIVE_TESTS_LABEL); +const SHARD_ANNOTATION_SEP = '||shard='; /** * Expands configs that appear in the shard map into N shard-annotated entries. * For example, if `fleet/jest.integration.config.js` has 2 shards, it becomes: @@ -221,7 +230,39 @@ export async function pickTestGroupRunOrder() { // Expand sharded integration configs into shard-annotated entries const jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw); - if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) { + // Apply affected package filtering + const affectedPackages = await getAffectedPackages(process.env.GITHUB_PR_MERGE_BASE, { + strategy: 'git', + includeDownstream: true, + ignorePatterns: [], // might want to exclude metadata/text changes in the future + ignoreUncategorizedChanges: true, + }).catch((error) => { + console.error('Error getting affected packages', error); + return null; + }); + + const shouldFilterByAffected = + USE_SELECTIVE_TESTING && affectedPackages && affectedPackages.size > 0; + + const filteredJestUnitConfigs = + shouldFilterByAffected && !touchedCriticalFiles(jestUnitConfigs, CRITICAL_FILES_JEST_UNIT_TESTS) + ? filterFilesByPackages(jestUnitConfigs, affectedPackages) + : jestUnitConfigs; + console.warn( + `Filtering Jest unit tests for affected packages: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` + ); + const filteredJestIntegrationConfigs = shouldFilterByAffected + ? filterFilesByPackages(jestIntegrationConfigs, affectedPackages) + : jestIntegrationConfigs; + console.warn( + `Filtering Jest integration tests for affected packages: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` + ); + + if ( + !ftrConfigsByQueue.size && + !filteredJestUnitConfigs.length && + !filteredJestIntegrationConfigs.length + ) { throw new Error('unable to find any unit, integration, or FTR configs'); } @@ -288,7 +329,7 @@ export async function pickTestGroupRunOrder() { overheadMin: 0.2, warmupMin: 4, concurrency: 3, - names: jestUnitConfigs, + names: filteredJestUnitConfigs, }, { type: INTEGRATION_TYPE, @@ -297,7 +338,7 @@ export async function pickTestGroupRunOrder() { overheadMin: 0.2, warmupMin: 2, concurrency: 1, - names: jestIntegrationConfigs, + names: filteredJestIntegrationConfigs, }, ...Array.from(ftrConfigsByQueue).map(([queue, names]) => ({ type: FUNCTIONAL_TYPE, From ec9869512cdd9291792d41e2a14bde01a4cf8f68 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 25 Mar 2026 15:54:45 +0100 Subject: [PATCH 02/12] no affected package -> no unit tests --- .../ci-stats/pick_test_group_run_order.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 48bca392788f8..0ce69dfa456a6 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -245,21 +245,24 @@ export async function pickTestGroupRunOrder() { return null; }); - const shouldFilterByAffected = - USE_SELECTIVE_TESTING && affectedPackages && affectedPackages.size > 0; + const shouldFilterByAffected = USE_SELECTIVE_TESTING && affectedPackages; + + if (USE_SELECTIVE_TESTING) { + console.warn('Filtering Jest unit/integration tests for affected packages:', affectedPackages); + } const filteredJestUnitConfigs = shouldFilterByAffected && !touchedCriticalFiles(jestUnitConfigs, CRITICAL_FILES_JEST_UNIT_TESTS) ? filterFilesByPackages(jestUnitConfigs, affectedPackages) : jestUnitConfigs; console.warn( - `Filtering Jest unit tests for affected packages: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` + `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` ); const filteredJestIntegrationConfigs = shouldFilterByAffected ? filterFilesByPackages(jestIntegrationConfigs, affectedPackages) : jestIntegrationConfigs; console.warn( - `Filtering Jest integration tests for affected packages: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` + `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` ); if ( From 711f1b9c28cd6bb7808bc615335b0dde32c239fd Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 12:40:46 +0200 Subject: [PATCH 03/12] co-locate selective testing and related filtering code --- .../pipeline-utils/affected-packages/index.ts | 1 + .../affected-packages/strategy_git.ts | 9 ++- .../pipeline-utils/affected-packages/utils.ts | 6 +- .../ci-stats/pick_test_group_run_order.ts | 60 +++++++++++-------- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/.buildkite/pipeline-utils/affected-packages/index.ts b/.buildkite/pipeline-utils/affected-packages/index.ts index 83e553e7cd2e4..d9b32da5b3211 100644 --- a/.buildkite/pipeline-utils/affected-packages/index.ts +++ b/.buildkite/pipeline-utils/affected-packages/index.ts @@ -13,6 +13,7 @@ import { getAffectedProjectsMoon } from './strategy_moon'; export * from './const'; export * from './utils'; +export { listChangedFiles } from './strategy_git'; export interface AffectedPackagesConfig { strategy?: 'git' | 'moon'; diff --git a/.buildkite/pipeline-utils/affected-packages/strategy_git.ts b/.buildkite/pipeline-utils/affected-packages/strategy_git.ts index d7f5c4a7510bb..a3cba0691bdcb 100644 --- a/.buildkite/pipeline-utils/affected-packages/strategy_git.ts +++ b/.buildkite/pipeline-utils/affected-packages/strategy_git.ts @@ -47,7 +47,14 @@ export function getAffectedModulesGit({ return includeDownstream ? getDownstreamDependents(directlyAffected) : directlyAffected; } -function listChangedFiles({ mergeBase, commit }: { mergeBase: string; commit: string }): string[] { +/** Paths changed from `git merge-base mergeBase HEAD` to `commit` (plus local untracked when not CI). */ +export function listChangedFiles({ + mergeBase, + commit, +}: { + mergeBase: string; + commit: string; +}): string[] { const execOptions = { cwd: getKibanaDir(), encoding: 'utf8' as const, diff --git a/.buildkite/pipeline-utils/affected-packages/utils.ts b/.buildkite/pipeline-utils/affected-packages/utils.ts index 807f687dc77e6..5bc3323e69652 100644 --- a/.buildkite/pipeline-utils/affected-packages/utils.ts +++ b/.buildkite/pipeline-utils/affected-packages/utils.ts @@ -24,9 +24,7 @@ export function filterIgnoredFiles(files: string[], patterns: string[]): string[ * Returns true when any pattern matches any file in the list */ export function touchedCriticalFiles(files: string[], criticalFiles: string[]): boolean { - return ( - files.some((file) => - criticalFiles.some((criticalFile) => minimatch(file, criticalFile, { dot: true })) - ) && false // temporary disable + return files.some((file) => + criticalFiles.some((criticalFile) => minimatch(file, criticalFile, { dot: true })) ); } diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 0ce69dfa456a6..69f30a89b7508 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -26,6 +26,7 @@ import { serverless, stateful } from '../../ftr_configs_manifests.json'; import { filterEmptyJestConfigs } from './get_tests_from_config'; import { getAffectedPackages, + listChangedFiles, filterFilesByPackages, SELECTIVE_TESTS_LABEL, CRITICAL_FILES_JEST_UNIT_TESTS, @@ -234,36 +235,43 @@ export async function pickTestGroupRunOrder() { // Expand sharded integration configs into shard-annotated entries const jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw); - // Apply affected package filtering - const affectedPackages = await getAffectedPackages(process.env.GITHUB_PR_MERGE_BASE, { - strategy: 'git', - includeDownstream: true, - ignorePatterns: [], // might want to exclude metadata/text changes in the future - ignoreUncategorizedChanges: true, - }).catch((error) => { - console.error('Error getting affected packages', error); - return null; - }); + let filteredJestUnitConfigs = jestUnitConfigs; + let filteredJestIntegrationConfigs = jestIntegrationConfigs; + + if (USE_SELECTIVE_TESTING && process.env.GITHUB_PR_MERGE_BASE) { + const mergeBase = process.env.GITHUB_PR_MERGE_BASE; + const affectedPackages = await getAffectedPackages(mergeBase, { + strategy: 'git', + includeDownstream: true, + ignorePatterns: [], // might want to exclude metadata/text changes in the future + ignoreUncategorizedChanges: true, + }).catch((error) => { + console.error('Error getting affected packages', error); + return null; + }); - const shouldFilterByAffected = USE_SELECTIVE_TESTING && affectedPackages; + const shouldFilterByAffected = Boolean(affectedPackages); + const prChangedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' }); - if (USE_SELECTIVE_TESTING) { console.warn('Filtering Jest unit/integration tests for affected packages:', affectedPackages); - } - const filteredJestUnitConfigs = - shouldFilterByAffected && !touchedCriticalFiles(jestUnitConfigs, CRITICAL_FILES_JEST_UNIT_TESTS) - ? filterFilesByPackages(jestUnitConfigs, affectedPackages) - : jestUnitConfigs; - console.warn( - `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` - ); - const filteredJestIntegrationConfigs = shouldFilterByAffected - ? filterFilesByPackages(jestIntegrationConfigs, affectedPackages) - : jestIntegrationConfigs; - console.warn( - `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` - ); + if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS)) { + if (shouldFilterByAffected) { + filteredJestIntegrationConfigs = filterFilesByPackages( + jestIntegrationConfigs, + affectedPackages + ); + filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages); + } + } + + console.warn( + `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` + ); + console.warn( + `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` + ); + } if ( !ftrConfigsByQueue.size && From ebb43dd4d08db7c3eeab913de466ffe4b5541248 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 13:11:34 +0200 Subject: [PATCH 04/12] debug why selective testing is not engaged --- .../pipeline-utils/ci-stats/pick_test_group_run_order.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 69f30a89b7508..ec0279a34af39 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -238,6 +238,11 @@ export async function pickTestGroupRunOrder() { let filteredJestUnitConfigs = jestUnitConfigs; let filteredJestIntegrationConfigs = jestIntegrationConfigs; + console.log({ + USE_SELECTIVE_TESTING, + GITHUB_PR_MERGE_BASE: process.env.GITHUB_PR_MERGE_BASE, + }); + if (USE_SELECTIVE_TESTING && process.env.GITHUB_PR_MERGE_BASE) { const mergeBase = process.env.GITHUB_PR_MERGE_BASE; const affectedPackages = await getAffectedPackages(mergeBase, { From aea034f67ee89075cc6c710d522b0174580f648f Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 13:33:52 +0200 Subject: [PATCH 05/12] fix label usage --- .../pipeline-utils/ci-stats/pick_test_group_run_order.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index ec0279a34af39..c6da788c6a043 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -35,7 +35,7 @@ import { import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils'; // TODO: this is always false on on-merge, when switching to enable this by default, check if this is a PR -const USE_SELECTIVE_TESTING = process.env.GITHUB_LABELS?.includes(SELECTIVE_TESTS_LABEL); +const USE_SELECTIVE_TESTING = process.env.GITHUB_PR_LABELS?.includes(SELECTIVE_TESTS_LABEL); const SHARD_ANNOTATION_SEP = '||shard='; /** @@ -238,11 +238,6 @@ export async function pickTestGroupRunOrder() { let filteredJestUnitConfigs = jestUnitConfigs; let filteredJestIntegrationConfigs = jestIntegrationConfigs; - console.log({ - USE_SELECTIVE_TESTING, - GITHUB_PR_MERGE_BASE: process.env.GITHUB_PR_MERGE_BASE, - }); - if (USE_SELECTIVE_TESTING && process.env.GITHUB_PR_MERGE_BASE) { const mergeBase = process.env.GITHUB_PR_MERGE_BASE; const affectedPackages = await getAffectedPackages(mergeBase, { From 01feadf687b9c2a1ea1b1ad8e33e06764d13da43 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 14:15:56 +0200 Subject: [PATCH 06/12] prettier --- .../ci-stats/pick_test_group_run_order.ts | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index c6da788c6a043..25114d6667bf6 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -31,6 +31,7 @@ import { SELECTIVE_TESTS_LABEL, CRITICAL_FILES_JEST_UNIT_TESTS, touchedCriticalFiles, + CRITICAL_FILES_JEST_INTEGRATION_TESTS, } from '../affected-packages'; import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils'; @@ -255,22 +256,28 @@ export async function pickTestGroupRunOrder() { console.warn('Filtering Jest unit/integration tests for affected packages:', affectedPackages); - if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS)) { - if (shouldFilterByAffected) { - filteredJestIntegrationConfigs = filterFilesByPackages( - jestIntegrationConfigs, - affectedPackages - ); - filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages); - } + if ( + shouldFilterByAffected && + !touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS) + ) { + filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages); + console.warn( + `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` + ); } - console.warn( - `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` - ); - console.warn( - `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` - ); + if ( + shouldFilterByAffected && + !touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_INTEGRATION_TESTS) + ) { + filteredJestIntegrationConfigs = filterFilesByPackages( + jestIntegrationConfigs, + affectedPackages + ); + console.warn( + `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` + ); + } } if ( From a950e7ea2927ae2cd797ab271326c7ead0b0af68 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 14:28:07 +0200 Subject: [PATCH 07/12] add affected-modules to sensitive files --- .buildkite/pipeline-utils/affected-packages/const.ts | 2 ++ .../pipeline-utils/affected-packages/module_lookup.test.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index f04dab6512dd1..af86a686d1cf7 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -19,6 +19,7 @@ export const CRITICAL_FILES_JEST_UNIT_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', + '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; // TODO: find reasonable set of critical files for integration tests @@ -29,4 +30,5 @@ export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', + '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; diff --git a/.buildkite/pipeline-utils/affected-packages/module_lookup.test.ts b/.buildkite/pipeline-utils/affected-packages/module_lookup.test.ts index 01705bd4478e7..b3234f68106ae 100644 --- a/.buildkite/pipeline-utils/affected-packages/module_lookup.test.ts +++ b/.buildkite/pipeline-utils/affected-packages/module_lookup.test.ts @@ -24,10 +24,10 @@ import { getModuleDependencies, buildModuleDownstreamGraph, resetModuleLookupCache, - UNCATEGORIZED_MODULE_ID, } from './module_lookup'; import { getAffectedModulesGit } from './strategy_git'; import { filterIgnoredFiles } from './utils'; +import { UNCATEGORIZED_MODULE_ID } from './const'; function git(cwd: string, command: string): string { return execSync(`git ${command}`, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }); From 5f4334aa54f879c99177c58dcb6122bcdcec0a32 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 30 Mar 2026 15:40:08 +0200 Subject: [PATCH 08/12] extract affected filtering application to a function for readability --- .../ci-stats/pick_test_group_run_order.ts | 111 ++++++++++-------- 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 25114d6667bf6..8f47b3ff1bdfe 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -224,7 +224,7 @@ export async function pickTestGroupRunOrder() { os.availableParallelism() ); // Expand sharded unit configs (e.g. cases/jest.config.js) into shard-annotated entries - const jestUnitConfigs = expandShardedJestConfigs(jestUnitConfigsFiltered); + let jestUnitConfigs = expandShardedJestConfigs(jestUnitConfigsFiltered); const jestIntegrationConfigsRaw = LIMIT_CONFIG_TYPE.includes('integration') ? globby.sync(getJestConfigGlobs(['**/jest.integration.config.js', '!**/__fixtures__/**']), { @@ -234,57 +234,16 @@ export async function pickTestGroupRunOrder() { }) : []; // Expand sharded integration configs into shard-annotated entries - const jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw); - - let filteredJestUnitConfigs = jestUnitConfigs; - let filteredJestIntegrationConfigs = jestIntegrationConfigs; + let jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw); if (USE_SELECTIVE_TESTING && process.env.GITHUB_PR_MERGE_BASE) { - const mergeBase = process.env.GITHUB_PR_MERGE_BASE; - const affectedPackages = await getAffectedPackages(mergeBase, { - strategy: 'git', - includeDownstream: true, - ignorePatterns: [], // might want to exclude metadata/text changes in the future - ignoreUncategorizedChanges: true, - }).catch((error) => { - console.error('Error getting affected packages', error); - return null; - }); - - const shouldFilterByAffected = Boolean(affectedPackages); - const prChangedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' }); - - console.warn('Filtering Jest unit/integration tests for affected packages:', affectedPackages); - - if ( - shouldFilterByAffected && - !touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS) - ) { - filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages); - console.warn( - `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` - ); - } - - if ( - shouldFilterByAffected && - !touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_INTEGRATION_TESTS) - ) { - filteredJestIntegrationConfigs = filterFilesByPackages( - jestIntegrationConfigs, - affectedPackages - ); - console.warn( - `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` - ); - } + const { filteredJestUnitConfigs, filteredJestIntegrationConfigs } = + await filterConfigsByAffectedPackages(jestUnitConfigs, jestIntegrationConfigs); + jestUnitConfigs = filteredJestUnitConfigs; + jestIntegrationConfigs = filteredJestIntegrationConfigs; } - if ( - !ftrConfigsByQueue.size && - !filteredJestUnitConfigs.length && - !filteredJestIntegrationConfigs.length - ) { + if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) { throw new Error('unable to find any unit, integration, or FTR configs'); } @@ -352,7 +311,7 @@ export async function pickTestGroupRunOrder() { overheadMin: 0.2, warmupMin: 4, concurrency: 3, - names: filteredJestUnitConfigs, + names: jestUnitConfigs, }, { type: INTEGRATION_TYPE, @@ -362,7 +321,7 @@ export async function pickTestGroupRunOrder() { overheadMin: 0.2, warmupMin: 2, concurrency: 1, - names: filteredJestIntegrationConfigs, + names: jestIntegrationConfigs, }, ...Array.from(ftrConfigsByQueue).map(([queue, names]) => ({ type: FUNCTIONAL_TYPE, @@ -725,3 +684,55 @@ function getEnabledFtrConfigs(patterns?: string[], solutions?: string[]) { throw new Error(`unable to collect enabled FTR configs: ${error.message}`); } } + +async function filterConfigsByAffectedPackages( + jestUnitConfigs: string[], + jestIntegrationConfigs: string[] +) { + const mergeBase = process.env.GITHUB_PR_MERGE_BASE!; + const affectedPackages = await getAffectedPackages(mergeBase, { + strategy: 'git', + includeDownstream: true, + ignorePatterns: [], // might want to exclude metadata/text changes in the future + ignoreUncategorizedChanges: true, + }).catch((error) => { + console.error('Error getting affected packages', error); + return null; + }); + + const shouldFilterByAffected = Boolean(affectedPackages); + if (!shouldFilterByAffected) { + console.log('Not filtering Jest unit/integration tests because no affected packages found'); + return { + filteredJestUnitConfigs: jestUnitConfigs, + filteredJestIntegrationConfigs: jestIntegrationConfigs, + }; + } + + const prChangedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' }); + console.log('Filtering Jest unit/integration tests for affected packages:', affectedPackages); + + let filteredJestUnitConfigs = jestUnitConfigs; + let filteredJestIntegrationConfigs = jestIntegrationConfigs; + if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS)) { + filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages); + console.log( + `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}` + ); + } else { + console.log('Not filtering Jest unit tests because critical files changed'); + } + + if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_INTEGRATION_TESTS)) { + filteredJestIntegrationConfigs = filterFilesByPackages( + jestIntegrationConfigs, + affectedPackages + ); + console.log( + `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}` + ); + } else { + console.log('Not filtering Jest integration tests because critical files changed'); + } + return { filteredJestUnitConfigs, filteredJestIntegrationConfigs }; +} From 2a00ec700180f9755d28057853e64b5bcc3c9a40 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 1 Apr 2026 11:42:47 +0200 Subject: [PATCH 09/12] add test commit to test affected graph --- .buildkite/pipeline-utils/affected-packages/const.ts | 4 ++-- src/platform/packages/private/kbn-index-editor/index.ts | 2 ++ src/platform/packages/private/kbn-item-buffer/index.ts | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index af86a686d1cf7..6ff2f5a241ef9 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -19,7 +19,7 @@ export const CRITICAL_FILES_JEST_UNIT_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', - '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + // '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; // TODO: find reasonable set of critical files for integration tests @@ -30,5 +30,5 @@ export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', - '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + // '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; diff --git a/src/platform/packages/private/kbn-index-editor/index.ts b/src/platform/packages/private/kbn-index-editor/index.ts index 5d906b662ff5d..8c75d6da810d5 100644 --- a/src/platform/packages/private/kbn-index-editor/index.ts +++ b/src/platform/packages/private/kbn-index-editor/index.ts @@ -12,3 +12,5 @@ export { ACTION_EDIT_LOOKUP_INDEX, registerIndexEditorActions } from './src/ui_a export { registerIndexEditorAnalyticsEvents } from './src/telemetry/events_registration'; export type { EditLookupIndexContentContext, EditLookupIndexFlyoutDeps } from './src/types'; + +/** More test commit */ diff --git a/src/platform/packages/private/kbn-item-buffer/index.ts b/src/platform/packages/private/kbn-item-buffer/index.ts index 92b8be8e72fe6..0d24ad739766a 100644 --- a/src/platform/packages/private/kbn-item-buffer/index.ts +++ b/src/platform/packages/private/kbn-item-buffer/index.ts @@ -7,5 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +/** Test commit */ + export { ItemBuffer, TimedItemBuffer } from './src'; export type { ItemBufferParams, TimedItemBufferParams } from './src'; From e50d3309af7925b55935138cfdb68c1c0a74f146 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 1 Apr 2026 16:09:37 +0200 Subject: [PATCH 10/12] Revert "add test commit to test affected graph" This reverts commit 2a00ec700180f9755d28057853e64b5bcc3c9a40. --- .buildkite/pipeline-utils/affected-packages/const.ts | 4 ++-- src/platform/packages/private/kbn-index-editor/index.ts | 2 -- src/platform/packages/private/kbn-item-buffer/index.ts | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index 6ff2f5a241ef9..af86a686d1cf7 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -19,7 +19,7 @@ export const CRITICAL_FILES_JEST_UNIT_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', - // '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; // TODO: find reasonable set of critical files for integration tests @@ -30,5 +30,5 @@ export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ 'yarn.lock', 'tsconfig.json', 'src/platform/packages/shared/kbn-test/**/*', - // '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', ]; diff --git a/src/platform/packages/private/kbn-index-editor/index.ts b/src/platform/packages/private/kbn-index-editor/index.ts index 8c75d6da810d5..5d906b662ff5d 100644 --- a/src/platform/packages/private/kbn-index-editor/index.ts +++ b/src/platform/packages/private/kbn-index-editor/index.ts @@ -12,5 +12,3 @@ export { ACTION_EDIT_LOOKUP_INDEX, registerIndexEditorActions } from './src/ui_a export { registerIndexEditorAnalyticsEvents } from './src/telemetry/events_registration'; export type { EditLookupIndexContentContext, EditLookupIndexFlyoutDeps } from './src/types'; - -/** More test commit */ diff --git a/src/platform/packages/private/kbn-item-buffer/index.ts b/src/platform/packages/private/kbn-item-buffer/index.ts index 0d24ad739766a..92b8be8e72fe6 100644 --- a/src/platform/packages/private/kbn-item-buffer/index.ts +++ b/src/platform/packages/private/kbn-item-buffer/index.ts @@ -7,7 +7,5 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -/** Test commit */ - export { ItemBuffer, TimedItemBuffer } from './src'; export type { ItemBufferParams, TimedItemBufferParams } from './src'; From 6c440e9415dae3c2b08e2aacde099ac5f02fd092 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 1 Apr 2026 16:40:53 +0200 Subject: [PATCH 11/12] Add some more sensitive paths for affected filtering --- .../pipeline-utils/affected-packages/const.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index af86a686d1cf7..83198d229a9ce 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -11,24 +11,40 @@ export const UNCATEGORIZED_MODULE_ID = '[uncategorized]'; export const SELECTIVE_TESTS_LABEL = 'ci:use-selective-testing'; -// TODO: find reasonable set of critical files for unit tests +// Changes here skip affected-package filtering for Jest (full run). +// Keep narrow: global test harness, transforms, CI selection. export const CRITICAL_FILES_JEST_UNIT_TESTS = [ 'scripts/jest.js', 'scripts/jest_all.js', 'package.json', 'yarn.lock', 'tsconfig.json', + '.node-version', + '.nvmrc', + 'src/setup_node_env/**/*', + 'packages/kbn-babel-preset/**/*', + 'src/platform/packages/shared/kbn-repo-info/**/*', 'src/platform/packages/shared/kbn-test/**/*', + 'src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/**/*', + 'src/platform/packages/shared/react/kibana_mount/test_helpers/react_mount_serializer.ts', '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}', ]; -// TODO: find reasonable set of critical files for integration tests export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ 'scripts/jest_integration.js', 'scripts/jest_all.js', 'package.json', 'yarn.lock', 'tsconfig.json', + '.node-version', + '.nvmrc', + 'src/setup_node_env/**/*', + 'packages/kbn-babel-preset/**/*', + 'src/platform/packages/shared/kbn-repo-info/**/*', 'src/platform/packages/shared/kbn-test/**/*', + 'src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/**/*', + 'src/platform/packages/shared/react/kibana_mount/test_helpers/react_mount_serializer.ts', '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}', ]; From 4611ee800f18e3b9e184921dfbf2a0c01eb0d191 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 1 Apr 2026 20:37:17 +0200 Subject: [PATCH 12/12] add more ignore paths --- .buildkite/pipeline-utils/affected-packages/const.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index 83198d229a9ce..532395190e2ce 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -27,6 +27,7 @@ export const CRITICAL_FILES_JEST_UNIT_TESTS = [ 'src/platform/packages/shared/kbn-test/**/*', 'src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/**/*', 'src/platform/packages/shared/react/kibana_mount/test_helpers/react_mount_serializer.ts', + 'src/platform/packages/private/kbn-jest-serializers/**/*', '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}', ];