diff --git a/.buildkite/pipeline-utils/affected-packages/const.ts b/.buildkite/pipeline-utils/affected-packages/const.ts index 532395190e2ce..5d2b34599978f 100644 --- a/.buildkite/pipeline-utils/affected-packages/const.ts +++ b/.buildkite/pipeline-utils/affected-packages/const.ts @@ -49,3 +49,20 @@ export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [ '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}', ]; + +export const CRITICAL_FILES_SCOUT = [ + 'package.json', + 'yarn.lock', + 'tsconfig.json', + '.node-version', + '.nvmrc', + 'src/setup_node_env/**/*', + 'packages/kbn-babel-preset/**/*', + 'src/platform/packages/shared/kbn-repo-info/**/*', + 'src/platform/packages/shared/kbn-scout/**/*', + 'src/platform/packages/private/kbn-scout-reporting/**/*', + 'scripts/scout.js', + '.buildkite/scripts/steps/test/scout/**/*', + '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}', + '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}', +]; diff --git a/.buildkite/pipeline-utils/index.ts b/.buildkite/pipeline-utils/index.ts index f03137139c022..417faebe60e7c 100644 --- a/.buildkite/pipeline-utils/index.ts +++ b/.buildkite/pipeline-utils/index.ts @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +export * from './affected-packages'; export * from './agent_images'; export * from './buildkite'; export * as CiStats from './ci-stats'; diff --git a/.buildkite/scripts/steps/test/scout/resolve_selective_testing.ts b/.buildkite/scripts/steps/test/scout/resolve_selective_testing.ts new file mode 100644 index 0000000000000..63fcf9500190e --- /dev/null +++ b/.buildkite/scripts/steps/test/scout/resolve_selective_testing.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { + getAffectedPackages, + listChangedFiles, + touchedCriticalFiles, + CRITICAL_FILES_SCOUT, +} from '#pipeline-utils'; + +const mergeBase = process.env.AFFECTED_MERGE_BASE; +const outPath = process.env.AFFECTED_MODULES_FILE; + +if (!mergeBase) { + console.error('AFFECTED_MERGE_BASE environment variable is required'); + process.exit(1); +} + +if (!outPath) { + console.error('AFFECTED_MODULES_FILE environment variable is required'); + process.exit(1); +} + +(async () => { + // List changed files once; reuse for both affected-packages and critical-files check. + const changedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' }); + + // Write affected modules JSON (replaces the list_affected binary call). + const affectedPackages = await getAffectedPackages(mergeBase, { + strategy: 'git', + includeDownstream: true, + ignoreUncategorizedChanges: true, + }); + const resolvedOutPath = path.resolve(outPath); + fs.mkdirSync(path.dirname(resolvedOutPath), { recursive: true }); + fs.writeFileSync(resolvedOutPath, JSON.stringify(Array.from(affectedPackages).sort(), null, 2)); + + // Top-level check: if critical Scout files were touched, the all Scout tests should run. + const criticalTouched = touchedCriticalFiles(changedFiles, CRITICAL_FILES_SCOUT); + if (criticalTouched) { + console.warn( + 'Critical Scout files changed — selective testing will be skipped (full suite run)' + ); + } + + // Output true/false to stdout for the shell script to capture. + process.stdout.write(criticalTouched ? 'true' : 'false'); +})(); diff --git a/.buildkite/scripts/steps/test/scout/test_run_builder.sh b/.buildkite/scripts/steps/test/scout/test_run_builder.sh index 421df64a9edd2..8555d9c8e28d0 100755 --- a/.buildkite/scripts/steps/test/scout/test_run_builder.sh +++ b/.buildkite/scripts/steps/test/scout/test_run_builder.sh @@ -55,24 +55,38 @@ else SCOUT_DISCOVERY_TARGET="local-stateful-only" fi - AFFECTED_MODULES_FILE="" - if [[ -n "${GITHUB_PR_MERGE_BASE:-}" ]] && [[ "${SELECTIVE_TESTING_ENABLED:-}" == "true" ]]; then - mkdir -p .scout - AFFECTED_MODULES_FILE=".scout/affected_modules.json" - .buildkite/pipeline-utils/affected-packages/list_affected \ - --strategy git --deep --merge-base "$GITHUB_PR_MERGE_BASE" --json \ - > "$AFFECTED_MODULES_FILE" + # PR builds: GITHUB_PR_MERGE_BASE is computed by set_git_merge_base() in util.sh. + # On-merge builds: falls back to HEAD~1 (parent of the merge commit). + if [[ -n "${GITHUB_PR_MERGE_BASE:-}" ]]; then + echo "Merge base (PR): ${GITHUB_PR_MERGE_BASE}" + else + echo "GITHUB_PR_MERGE_BASE not set — using HEAD~1 as merge base (on-merge build)" fi - - echo "--- Discover Playwright Configs and upload to Buildkite artifacts${AFFECTED_MODULES_FILE:+ (selective testing)}" - AFFECTED_FLAG=() - if [[ -n "$AFFECTED_MODULES_FILE" ]]; then - AFFECTED_FLAG=(--affected-modules "$AFFECTED_MODULES_FILE") + export AFFECTED_MERGE_BASE="${GITHUB_PR_MERGE_BASE:-HEAD~1}" + + mkdir -p .scout + export AFFECTED_MODULES_FILE=".scout/affected_modules.json" + + # Computes affected modules (writes AFFECTED_MODULES_FILE) and checks whether + # any critical Scout files were touched. + SCOUT_CRITICAL_FILES_TOUCHED=$(ts-node "$(dirname "${0}")/resolve_selective_testing.ts") + echo "Critical Scout files touched: ${SCOUT_CRITICAL_FILES_TOUCHED}" + + echo "--- Discover Playwright Configs and upload to Buildkite artifacts (affected modules detected)" + SELECTIVE_SCOUT_DISCOVERY_FLAG=() + if [[ "${SELECTIVE_TESTING_ENABLED:-}" == "true" ]] \ + && ! is_pr_with_label "scout:run-all-tests" \ + && [[ "$SCOUT_CRITICAL_FILES_TOUCHED" != "true" ]]; then + SELECTIVE_SCOUT_DISCOVERY_FLAG=(--selective-testing) + echo "Selective testing: enabled (--selective-testing flag will be passed to discover-playwright-configs)" + else + echo "Selective testing: disabled — reason: SELECTIVE_TESTING_ENABLED=${SELECTIVE_TESTING_ENABLED:-false}, scout:run-all-tests label=$(is_pr_with_label "scout:run-all-tests" && echo yes || echo no), critical files touched=${SCOUT_CRITICAL_FILES_TOUCHED}" fi node scripts/scout discover-playwright-configs \ --include-custom-servers \ --target "$SCOUT_DISCOVERY_TARGET" \ - "${AFFECTED_FLAG[@]}" \ + --affected-modules "$AFFECTED_MODULES_FILE" \ + "${SELECTIVE_SCOUT_DISCOVERY_FLAG[@]}" \ --save cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json buildkite-agent artifact upload "scout_playwright_configs.json" diff --git a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts index 1a317b4fde67b..f9f492d48ac9d 100644 --- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts +++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts @@ -10,6 +10,7 @@ import type { FlagsReader } from '@kbn/dev-cli-runner'; import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry'; import type { ToolingLog } from '@kbn/tooling-log'; +import { findPackageForPath } from '@kbn/repo-packages'; import fs from 'fs'; import { filterModulesByScoutCiConfig, @@ -43,6 +44,10 @@ jest.mock('fs', () => { }; }); +jest.mock('@kbn/repo-packages', () => ({ + findPackageForPath: jest.fn(), +})); + jest.mock('@kbn/repo-info', () => ({ REPO_ROOT: '/mock/repo/root', })); @@ -139,6 +144,9 @@ describe('runDiscoverPlaywrightConfigs', () => { flagsReader.enum.mockReturnValue('all'); flagsReader.boolean.mockReturnValue(false); flagsReader.arrayOfStrings.mockReturnValue([]); + flagsReader.string.mockImplementation(() => ''); + + (findPackageForPath as jest.Mock).mockReset(); (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules); (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]); @@ -313,6 +321,97 @@ describe('runDiscoverPlaywrightConfigs', () => { expect(foundMessage![0]).toContain('1 package(s)'); // packageA }); + it('fails when --selective-testing is set without --affected-modules', () => { + flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing'); + flagsReader.string.mockReturnValue(''); + + expect(() => runDiscoverPlaywrightConfigs(flagsReader, log)).toThrow( + '--selective-testing requires --affected-modules' + ); + }); + + it('with --selective-testing and --affected-modules, limits discovery to affected modules only', () => { + flagsReader.enum.mockReturnValue('mki'); + flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing'); + flagsReader.string.mockImplementation((name: string) => + name === 'affected-modules' ? '/mock/affected_modules.json' : '' + ); + + (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => { + if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' }; + if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' }; + if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' }; + return undefined; + }); + + (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => { + if (readPath === '/mock/affected_modules.json') { + return JSON.stringify(['@kbn/pluginA']); + } + if (typeof readPath === 'string' && readPath.endsWith('package.json')) { + return JSON.stringify({ name: 'kibana', version: '1.0.0' }); + } + if (typeof readPath === 'string' && readPath.endsWith('.yml')) { + return 'mock yaml content'; + } + return ''; + }); + + runDiscoverPlaywrightConfigs(flagsReader, log); + + const infoCalls = log.info.mock.calls; + const foundMessage = infoCalls.find((call) => + call[0].includes('Found Playwright config files') + ); + expect(foundMessage).toBeDefined(); + expect(foundMessage![0]).toContain('1 plugin(s)'); + expect(foundMessage![0]).toContain('0 package(s)'); + + expect( + infoCalls.some((call) => + String(call[0]).includes('Selective testing: Scout discovery limited') + ) + ).toBe(true); + }); + + it('with --affected-modules but without --selective-testing, still discovers all modules matching the target', () => { + flagsReader.enum.mockReturnValue('mki'); + flagsReader.boolean.mockReturnValue(false); + flagsReader.string.mockImplementation((name: string) => + name === 'affected-modules' ? '/mock/affected_modules_non_draft.json' : '' + ); + + (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => { + if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' }; + if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' }; + if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' }; + return undefined; + }); + + (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => { + if (readPath === '/mock/affected_modules_non_draft.json') { + return JSON.stringify(['@kbn/pluginA']); + } + if (typeof readPath === 'string' && readPath.endsWith('package.json')) { + return JSON.stringify({ name: 'kibana', version: '1.0.0' }); + } + if (typeof readPath === 'string' && readPath.endsWith('.yml')) { + return 'mock yaml content'; + } + return ''; + }); + + runDiscoverPlaywrightConfigs(flagsReader, log); + + const infoCalls = log.info.mock.calls; + const foundMessage = infoCalls.find((call) => + call[0].includes('Found Playwright config files') + ); + expect(foundMessage).toBeDefined(); + expect(foundMessage![0]).toContain('2 plugin(s)'); + expect(foundMessage![0]).toContain('1 package(s)'); + }); + it('filters configs based on target tags for "mki" target (@cloud-serverless-*)', () => { flagsReader.enum.mockReturnValue('mki'); flagsReader.boolean.mockReturnValue(false); @@ -631,7 +730,7 @@ describe('runDiscoverPlaywrightConfigs', () => { const infoCalls = log.info.mock.calls; expect(infoCalls.length).toBeGreaterThan(0); - expect(infoCalls[0][0]).toContain('Found Playwright config files'); + expect(infoCalls.some((call) => call[0].includes('Found Playwright config files'))).toBe(true); // Check that config logs include tags const configLogs = infoCalls.filter((call) => call[0].startsWith('- ')); @@ -705,7 +804,7 @@ describe('runDiscoverPlaywrightConfigs', () => { expect(Array.isArray(savedData)).toBe(true); expect(log.info).toHaveBeenCalledWith( - expect.stringContaining('Scout configs were filtered for CI. Saved') + expect.stringContaining('Scout configs saved for CI (full suite)') ); }); diff --git a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts index 94a795284c0ef..1bd8b35619a73 100644 --- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts +++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { createFailError } from '@kbn/dev-cli-errors'; import type { Command, FlagsReader } from '@kbn/dev-cli-runner'; import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info'; import { testableModules } from '@kbn/scout-reporting/src/registry'; @@ -219,7 +220,8 @@ const splitStreamsTestsByServerRunFlags = ( const handleNonFlattenedOutput = ( filteredModules: ModuleDiscoveryInfo[], flagsReader: FlagsReader, - log: ToolingLog + log: ToolingLog, + selectiveTesting: boolean ): void => { if (flagsReader.boolean('save')) { const filteredForCiModules = filterModulesByScoutCiConfig(log, filteredModules); @@ -230,8 +232,9 @@ const handleNonFlattenedOutput = ( const { plugins: savedPluginCount, packages: savedPackageCount } = countModulesByType(splitModules); + const runScope = selectiveTesting ? 'selective' : 'full suite'; log.info( - `Scout configs were filtered for CI. Saved ${savedPluginCount} plugin(s) and ${savedPackageCount} package(s) to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'` + `Scout configs saved for CI (${runScope}): ${savedPluginCount} plugin(s) and ${savedPackageCount} package(s) written to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'` ); return; } @@ -250,18 +253,43 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool const targetTags = getTestTagsForTarget(target); const flatten = flagsReader.boolean('flatten'); const includeCustomServers = flagsReader.boolean('include-custom-servers'); + const selectiveTesting = flagsReader.boolean('selective-testing'); const affectedModulesPath = flagsReader.string('affected-modules'); + if (selectiveTesting && !affectedModulesPath) { + throw createFailError( + '--selective-testing requires --affected-modules (JSON array of @kbn/ IDs from list_affected).' + ); + } + // Build initial module discovery info const modulesWithTests = buildModuleDiscoveryInfo(); - // Mark affected status when selective testing is enabled (all modules kept, isAffected set) + // --affected-modules: keep every Scout module that passes target/CI filters; set isAffected + // per module so CI step labels can use an "affected " prefix where the PR touched that @kbn/ ID. const modulesAfterAffectedMark = affectedModulesPath ? markModulesAffectedStatus(modulesWithTests, affectedModulesPath, log) : modulesWithTests; + // --selective-testing: narrow to affected module groups only. + const limitDiscoveryToAffectedModules = selectiveTesting; + + const modulesForTargetTags = limitDiscoveryToAffectedModules + ? modulesAfterAffectedMark.filter((m) => m.isAffected === true) + : modulesAfterAffectedMark; + + if (limitDiscoveryToAffectedModules) { + log.info( + `Selective testing: Scout discovery limited to affected modules (${modulesForTargetTags.length} of ${modulesAfterAffectedMark.length})` + ); + } else { + log.info( + `Full suite run: all ${modulesAfterAffectedMark.length} discovered module(s) will be included (selective testing is disabled)` + ); + } + // Filter modules by target tags and compute server run flags - const filteredModulesByTags = filterModulesByTargetTags(modulesAfterAffectedMark, targetTags); + const filteredModulesByTags = filterModulesByTargetTags(modulesForTargetTags, targetTags); const filteredModules = filterModulesByCustomServerPaths( filteredModulesByTags, includeCustomServers @@ -273,7 +301,12 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool if (flatten) { handleFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log); } else { - handleNonFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log); + handleNonFlattenedOutput( + filteredModulesWithExcludedConfigs, + flagsReader, + log, + selectiveTesting + ); } }; @@ -294,6 +327,11 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool * Output formats: * - Standard: Lists modules grouped by plugin/package with their configs and tags * - Flattened: Groups configs by deployment mode (stateful/serverless), group, and run mode + * + * Affected modules: + * - With --affected-modules, all modules are still considered; isAffected flags drive the + * "affected " Buildkite step prefix. With --selective-testing, only affected module groups + * are emitted; those steps keep the same prefix. */ export const discoverPlaywrightConfigsCmd: Command = { name: 'discover-playwright-configs', @@ -311,9 +349,13 @@ export const discoverPlaywrightConfigsCmd: Command = { - 'local-stateful-only': @local-stateful-* tags only - 'mki': @cloud-serverless-* tags - 'ech': @cloud-stateful-* tags - --affected-modules Path to a JSON file containing affected @kbn/ module IDs - (produced by list_affected). All modules run; affected ones - get "affected" prefix in Buildkite step (selective testing). + --affected-modules Path to a JSON file of affected @kbn/ module IDs (list_affected). + All Scout modules still go through discovery; each module is marked + isAffected so CI can prefix steps with "affected " when the PR + touches that module. Combine with --selective-testing to emit only + affected module groups. + --selective-testing Requires --affected-modules. + Limits output / Scout CI steps to affected modules; labels unchanged. --include-custom-servers Include configs under 'test/scout_*' paths for custom server setups --validate Validate that all discovered modules are registered in Scout CI config --save Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}' @@ -348,18 +390,22 @@ export const discoverPlaywrightConfigsCmd: Command = { # Save flattened configs for Cloud test execution node scripts/scout discover-playwright-configs --flatten --save - # Selective testing: only configs for affected modules + # Affected-module labels on every Scout group (full CI matrix) node scripts/scout discover-playwright-configs --affected-modules .scout/affected_modules.json --save + + # Only affected module groups (selective testing for PRs) + node scripts/scout discover-playwright-configs --affected-modules .scout/affected_modules.json --selective-testing --save `, flags: { string: ['target', 'affected-modules'], - boolean: ['save', 'validate', 'flatten', 'include-custom-servers'], + boolean: ['save', 'validate', 'flatten', 'include-custom-servers', 'selective-testing'], default: { target: 'all', save: false, validate: false, flatten: false, 'include-custom-servers': false, + 'selective-testing': false, }, }, run: ({ flagsReader, log }) => { diff --git a/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.test.ts b/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.test.ts index f5f5bd165e645..c61ce0aaa5f01 100644 --- a/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.test.ts +++ b/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.test.ts @@ -203,7 +203,7 @@ describe('affected_modules', () => { expect(result.every((m) => m.isAffected === false)).toBe(true); }); - it('should log affected and rest counts', () => { + it('should log affected and unaffected counts', () => { (fs.readFileSync as jest.Mock).mockReturnValue( JSON.stringify(['@kbn/security-solution-plugin']) ); @@ -211,7 +211,7 @@ describe('affected_modules', () => { markModulesAffectedStatus(modules, '/affected.json', mockLog); expect(mockLog.info).toHaveBeenCalledWith( - expect.stringContaining('1 affected module(s), 2 rest') + expect.stringContaining('1 affected, 2 unaffected') ); }); }); diff --git a/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.ts b/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.ts index f9127d40c830f..f99ebabb38eb3 100644 --- a/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.ts +++ b/src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.ts @@ -48,8 +48,8 @@ export const readAffectedModules = (filePath: string, log: ToolingLog): Set isAffected: true @@ -91,9 +91,9 @@ export const markModulesAffectedStatus = ( }); log.info( - `Selective testing: ${affectedCount} affected module(s), ${ + `Affected modules: ${affectedCount} affected, ${ marked.length - affectedCount - } rest, ${unmappedCount} unmapped` + } unaffected, ${unmappedCount} unmapped` ); return marked; diff --git a/src/platform/packages/shared/kbn-scout/src/tests_discovery/types.ts b/src/platform/packages/shared/kbn-scout/src/tests_discovery/types.ts index 5d67fca74d6a7..83d7e085b0caa 100644 --- a/src/platform/packages/shared/kbn-scout/src/tests_discovery/types.ts +++ b/src/platform/packages/shared/kbn-scout/src/tests_discovery/types.ts @@ -20,7 +20,7 @@ export interface ModuleDiscoveryInfo { name: string; group: string; type: 'plugin' | 'package'; - /** When selective testing is enabled, true if this module's @kbn/ ID is in the affected set */ + /** Set when using --affected-modules: true if this module's @kbn/ ID is in the affected set */ isAffected?: boolean; configs: { path: string;