[9.3] [CI] Affected testing for Jest/Jest integration (#259137)

.buildkite/pipeline-utils/affected-packages/README.md

@@ -69,7 +69,6 @@ const affectedPackages = await getAffectedPackages(
   {
     strategy: 'git',       // default, can also be 'moon'
     includeDownstream: true,
-    logging: false,
     ignorePatterns: ['**/*.md', 'docs/**'],
     ignoreUncategorizedChanges: false,
   }
@@ -109,7 +108,6 @@ const filteredFiles = filterFilesByPackages(
 |------------------------------------|---------------------------------|--------------|----------------------|
 | `AFFECTED_STRATEGY`                | `git`, `moon`                   | `git`        | `git`                |
 | `AFFECTED_DOWNSTREAM`              | `true`, `false`                 | `false`      | `true`               |
-| `AFFECTED_LOGGING`                 | `true`, `false`                 | `false`      | `true`               |
 | `AFFECTED_IGNORE`                  | comma-separated globs           | —            | —                    |
 | `AFFECTED_IGNORE_UNCATEGORIZED_CHANGES` | `true`, `false`            | `false`      | `false`              |
 | `GITHUB_PR_MERGE_BASE`             | any git ref                     | `origin/main`| —                    |

.buildkite/pipeline-utils/affected-packages/const.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export const UNCATEGORIZED_MODULE_ID = '[uncategorized]';
+
+export const SELECTIVE_TESTS_LABEL = 'ci:use-selective-testing';
+
+// Changes here skip affected-package filtering for Jest (full run).
+// Keep narrow: global test harness, transforms, CI selection.
+export const CRITICAL_FILES_JEST_UNIT_TESTS = [
+  'scripts/jest.js',
+  'scripts/jest_all.js',
+  'package.json',
+  'yarn.lock',
+  'tsconfig.json',
+  '.node-version',
+  '.nvmrc',
+  'src/setup_node_env/**/*',
+  'packages/kbn-babel-preset/**/*',
+  'src/platform/packages/shared/kbn-repo-info/**/*',
+  'src/platform/packages/shared/kbn-test/**/*',
+  'src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/**/*',
+  'src/platform/packages/shared/react/kibana_mount/test_helpers/react_mount_serializer.ts',
+  'src/platform/packages/private/kbn-jest-serializers/**/*',
+  '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}',
+  '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}',
+];
+
+export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [
+  'scripts/jest_integration.js',
+  'scripts/jest_all.js',
+  'package.json',
+  'yarn.lock',
+  'tsconfig.json',
+  '.node-version',
+  '.nvmrc',
+  'src/setup_node_env/**/*',
+  'packages/kbn-babel-preset/**/*',
+  'src/platform/packages/shared/kbn-repo-info/**/*',
+  'src/platform/packages/shared/kbn-test/**/*',
+  'src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/**/*',
+  'src/platform/packages/shared/react/kibana_mount/test_helpers/react_mount_serializer.ts',
+  '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}',
+  '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}',
+];

.buildkite/pipeline-utils/affected-packages/index.ts

@@ -11,13 +11,16 @@ import { findModuleForPath } from './module_lookup';
 import { getAffectedModulesGit } from './strategy_git';
 import { getAffectedProjectsMoon } from './strategy_moon';
 
+export * from './const';
+export * from './utils';
+export { listChangedFiles } from './strategy_git';
+
 export interface AffectedPackagesConfig {
-  strategy: 'git' | 'moon';
-  includeDownstream: boolean;
-  logging: boolean;
+  strategy?: 'git' | 'moon';
+  includeDownstream?: boolean;
   /** Glob patterns for changed files to exclude before module resolution (git strategy only). */
-  ignorePatterns: string[];
-  ignoreUncategorizedChanges: boolean;
+  ignorePatterns?: string[];
+  ignoreUncategorizedChanges?: boolean;
 }
 
 /**
@@ -26,12 +29,19 @@ export interface AffectedPackagesConfig {
  */
 export async function getAffectedPackages(
   mergeBase: string | undefined,
-  config: AffectedPackagesConfig = getConfigFromEnv()
+  configArgs: AffectedPackagesConfig = getConfigFromEnv()
 ): Promise<Set<string>> {
   if (!mergeBase) {
     throw new Error('No merge base found');
   }
 
+  const config = {
+    strategy: configArgs.strategy ?? 'git',
+    includeDownstream: configArgs.includeDownstream ?? false,
+    ignorePatterns: configArgs.ignorePatterns ?? [],
+    ignoreUncategorizedChanges: configArgs.ignoreUncategorizedChanges ?? false,
+  };
+
   try {
     const affectedPackages =
       config.strategy === 'git'
@@ -82,13 +92,12 @@ function getConfigFromEnv(): AffectedPackagesConfig {
   }
   const strategy = rawStrategy;
   const includeDownstream = process.env.AFFECTED_DOWNSTREAM !== 'false';
-  const logging = process.env.AFFECTED_LOGGING !== 'false';
   const ignorePatterns = (process.env.AFFECTED_IGNORE || '')
     .split(',')
     .map((p) => p.trim())
     .filter(Boolean);
 
   const ignoreUncategorizedChanges = process.env.AFFECTED_IGNORE_UNCATEGORIZED_CHANGES !== 'false';
 
-  return { strategy, includeDownstream, logging, ignorePatterns, ignoreUncategorizedChanges };
+  return { strategy, includeDownstream, ignorePatterns, ignoreUncategorizedChanges };
 }

.buildkite/pipeline-utils/affected-packages/list_affected

@@ -85,7 +85,6 @@ async function main() {
     const affectedPackages = await getAffectedPackages(mergeBase, {
       strategy,
       includeDownstream,
-      logging: false,
       ignorePatterns,
       ignoreUncategorizedChanges,
     });

.buildkite/pipeline-utils/affected-packages/module_lookup.test.ts

@@ -24,10 +24,10 @@ import {
   getModuleDependencies,
   buildModuleDownstreamGraph,
   resetModuleLookupCache,
-  UNCATEGORIZED_MODULE_ID,
 } from './module_lookup';
 import { getAffectedModulesGit } from './strategy_git';
 import { filterIgnoredFiles } from './utils';
+import { UNCATEGORIZED_MODULE_ID } from './const';
 
 function git(cwd: string, command: string): string {
   return execSync(`git ${command}`, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] });

.buildkite/pipeline-utils/affected-packages/module_lookup.ts

@@ -12,8 +12,7 @@ import * as fs from 'fs';
 import { execSync } from 'child_process';
 import * as JSON5 from 'json5';
 import { getKibanaDir } from '../utils';
-
-export const UNCATEGORIZED_MODULE_ID = '[uncategorized]';
+import { UNCATEGORIZED_MODULE_ID } from './const';
 
 export interface ModuleLookup {
   /**

.buildkite/pipeline-utils/affected-packages/strategy_git.ts

@@ -9,11 +9,8 @@
 
 import { execSync } from 'child_process';
 import { getKibanaDir } from '../utils';
-import {
-  findModuleForPath,
-  buildModuleDownstreamGraph,
-  UNCATEGORIZED_MODULE_ID,
-} from './module_lookup';
+import { findModuleForPath, buildModuleDownstreamGraph } from './module_lookup';
+import { UNCATEGORIZED_MODULE_ID } from './const';
 import { filterIgnoredFiles } from './utils';
 
 const isCI = !!process.env.CI?.match(/^(1|true)$/i);
@@ -50,7 +47,14 @@ export function getAffectedModulesGit({
   return includeDownstream ? getDownstreamDependents(directlyAffected) : directlyAffected;
 }
 
-function listChangedFiles({ mergeBase, commit }: { mergeBase: string; commit: string }): string[] {
+/** Paths changed from `git merge-base mergeBase HEAD` to `commit` (plus local untracked when not CI). */
+export function listChangedFiles({
+  mergeBase,
+  commit,
+}: {
+  mergeBase: string;
+  commit: string;
+}): string[] {
   const execOptions = {
     cwd: getKibanaDir(),
     encoding: 'utf8' as const,

.buildkite/pipeline-utils/affected-packages/strategy_moon.ts

@@ -22,7 +22,7 @@ export function getAffectedProjectsMoon(
   const output = execSync(command, {
     cwd: REPO_ROOT,
     encoding: 'utf8',
-    maxBuffer: 10 * 1024 * 1024, // 10MB buffer
+    maxBuffer: 30 * 1024 * 1024, // 30MB buffer
     env: {
       ...process.env,
       MOON_BASE: mergeBase,

.buildkite/pipeline-utils/affected-packages/utils.ts

@@ -19,3 +19,12 @@ export function filterIgnoredFiles(files: string[], patterns: string[]): string[
   }
   return files.filter((file) => !patterns.some((p) => minimatch(file, p, { dot: true })));
 }
+
+/**
+ * Returns true when any pattern matches any file in the list
+ */
+export function touchedCriticalFiles(files: string[], criticalFiles: string[]): boolean {
+  return files.some((file) =>
+    criticalFiles.some((criticalFile) => minimatch(file, criticalFile, { dot: true }))
+  );
+}

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -24,10 +24,21 @@ import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
 import SHARDED_JEST_CONFIGS from '../../sharded_jest_configs.json';
 import { serverless, stateful } from '../../ftr_configs_manifests.json';
 import { filterEmptyJestConfigs } from './get_tests_from_config';
+import {
+  getAffectedPackages,
+  listChangedFiles,
+  filterFilesByPackages,
+  SELECTIVE_TESTS_LABEL,
+  CRITICAL_FILES_JEST_UNIT_TESTS,
+  touchedCriticalFiles,
+  CRITICAL_FILES_JEST_INTEGRATION_TESTS,
+} from '../affected-packages';
 import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils';
 
-const SHARD_ANNOTATION_SEP = '||shard=';
+// TODO: this is always false on on-merge, when switching to enable this by default, check if this is a PR
+const USE_SELECTIVE_TESTING = process.env.GITHUB_PR_LABELS?.includes(SELECTIVE_TESTS_LABEL);
 
+const SHARD_ANNOTATION_SEP = '||shard=';
 /**
  * Expands configs that appear in the shard map into N shard-annotated entries.
  * For example, if `fleet/jest.integration.config.js` has 2 shards, it becomes:
@@ -213,7 +224,7 @@ export async function pickTestGroupRunOrder() {
     os.availableParallelism()
   );
   // Expand sharded unit configs (e.g. cases/jest.config.js) into shard-annotated entries
-  const jestUnitConfigs = expandShardedJestConfigs(jestUnitConfigsFiltered);
+  let jestUnitConfigs = expandShardedJestConfigs(jestUnitConfigsFiltered);
 
   const jestIntegrationConfigsRaw = LIMIT_CONFIG_TYPE.includes('integration')
     ? globby.sync(getJestConfigGlobs(['**/jest.integration.config.js', '!**/__fixtures__/**']), {
@@ -223,7 +234,14 @@ export async function pickTestGroupRunOrder() {
       })
     : [];
   // Expand sharded integration configs into shard-annotated entries
-  const jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw);
+  let jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw);
+
+  if (USE_SELECTIVE_TESTING && process.env.GITHUB_PR_MERGE_BASE) {
+    const { filteredJestUnitConfigs, filteredJestIntegrationConfigs } =
+      await filterConfigsByAffectedPackages(jestUnitConfigs, jestIntegrationConfigs);
+    jestUnitConfigs = filteredJestUnitConfigs;
+    jestIntegrationConfigs = filteredJestIntegrationConfigs;
+  }
 
   if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) {
     throw new Error('unable to find any unit, integration, or FTR configs');
@@ -666,3 +684,55 @@ function getEnabledFtrConfigs(patterns?: string[], solutions?: string[]) {
     throw new Error(`unable to collect enabled FTR configs: ${error.message}`);
   }
 }
+
+async function filterConfigsByAffectedPackages(
+  jestUnitConfigs: string[],
+  jestIntegrationConfigs: string[]
+) {
+  const mergeBase = process.env.GITHUB_PR_MERGE_BASE!;
+  const affectedPackages = await getAffectedPackages(mergeBase, {
+    strategy: 'git',
+    includeDownstream: true,
+    ignorePatterns: [], // might want to exclude metadata/text changes in the future
+    ignoreUncategorizedChanges: true,
+  }).catch((error) => {
+    console.error('Error getting affected packages', error);
+    return null;
+  });
+
+  const shouldFilterByAffected = Boolean(affectedPackages);
+  if (!shouldFilterByAffected) {
+    console.log('Not filtering Jest unit/integration tests because no affected packages found');
+    return {
+      filteredJestUnitConfigs: jestUnitConfigs,
+      filteredJestIntegrationConfigs: jestIntegrationConfigs,
+    };
+  }
+
+  const prChangedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' });
+  console.log('Filtering Jest unit/integration tests for affected packages:', affectedPackages);
+
+  let filteredJestUnitConfigs = jestUnitConfigs;
+  let filteredJestIntegrationConfigs = jestIntegrationConfigs;
+  if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_UNIT_TESTS)) {
+    filteredJestUnitConfigs = filterFilesByPackages(jestUnitConfigs, affectedPackages);
+    console.log(
+      `Filtering Jest unit tests: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}`
+    );
+  } else {
+    console.log('Not filtering Jest unit tests because critical files changed');
+  }
+
+  if (!touchedCriticalFiles(prChangedFiles, CRITICAL_FILES_JEST_INTEGRATION_TESTS)) {
+    filteredJestIntegrationConfigs = filterFilesByPackages(
+      jestIntegrationConfigs,
+      affectedPackages
+    );
+    console.log(
+      `Filtering Jest integration tests: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}`
+    );
+  } else {
+    console.log('Not filtering Jest integration tests because critical files changed');
+  }
+  return { filteredJestUnitConfigs, filteredJestIntegrationConfigs };
+}