[9.3] Scout: enable Selective testing in Pull Requests (#261510)

.buildkite/pipeline-utils/affected-packages/const.ts

@@ -49,3 +49,20 @@ export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [
   '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}',
   '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}',
 ];
+
+export const CRITICAL_FILES_SCOUT = [
+  'package.json',
+  'yarn.lock',
+  'tsconfig.json',
+  '.node-version',
+  '.nvmrc',
+  'src/setup_node_env/**/*',
+  'packages/kbn-babel-preset/**/*',
+  'src/platform/packages/shared/kbn-repo-info/**/*',
+  'src/platform/packages/shared/kbn-scout/**/*',
+  'src/platform/packages/private/kbn-scout-reporting/**/*',
+  'scripts/scout.js',
+  '.buildkite/scripts/steps/test/scout/**/*',
+  '.buildkite/pipeline-utils/affected-packages/**/*.{ts,js,sh}',
+  '.buildkite/pipeline-utils/ci-stats/**/*.{ts,js}',
+];

.buildkite/pipeline-utils/index.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+export * from './affected-packages';
 export * from './agent_images';
 export * from './buildkite';
 export * as CiStats from './ci-stats';

.buildkite/scripts/steps/test/scout/resolve_selective_testing.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  getAffectedPackages,
+  listChangedFiles,
+  touchedCriticalFiles,
+  CRITICAL_FILES_SCOUT,
+} from '#pipeline-utils';
+
+const mergeBase = process.env.AFFECTED_MERGE_BASE;
+const outPath = process.env.AFFECTED_MODULES_FILE;
+
+if (!mergeBase) {
+  console.error('AFFECTED_MERGE_BASE environment variable is required');
+  process.exit(1);
+}
+
+if (!outPath) {
+  console.error('AFFECTED_MODULES_FILE environment variable is required');
+  process.exit(1);
+}
+
+(async () => {
+  // List changed files once; reuse for both affected-packages and critical-files check.
+  const changedFiles = listChangedFiles({ mergeBase, commit: 'HEAD' });
+
+  // Write affected modules JSON (replaces the list_affected binary call).
+  const affectedPackages = await getAffectedPackages(mergeBase, {
+    strategy: 'git',
+    includeDownstream: true,
+    ignoreUncategorizedChanges: true,
+  });
+  const resolvedOutPath = path.resolve(outPath);
+  fs.mkdirSync(path.dirname(resolvedOutPath), { recursive: true });
+  fs.writeFileSync(resolvedOutPath, JSON.stringify(Array.from(affectedPackages).sort(), null, 2));
+
+  // Top-level check: if critical Scout files were touched, the all Scout tests should run.
+  const criticalTouched = touchedCriticalFiles(changedFiles, CRITICAL_FILES_SCOUT);
+  if (criticalTouched) {
+    console.warn(
+      'Critical Scout files changed — selective testing will be skipped (full suite run)'
+    );
+  }
+
+  // Output true/false to stdout for the shell script to capture.
+  process.stdout.write(criticalTouched ? 'true' : 'false');
+})();

.buildkite/scripts/steps/test/scout/test_run_builder.sh

@@ -55,24 +55,38 @@ else
     SCOUT_DISCOVERY_TARGET="local-stateful-only"
   fi
 
-  AFFECTED_MODULES_FILE=""
-  if [[ -n "${GITHUB_PR_MERGE_BASE:-}" ]] && [[ "${SELECTIVE_TESTING_ENABLED:-}" == "true" ]]; then
-    mkdir -p .scout
-    AFFECTED_MODULES_FILE=".scout/affected_modules.json"
-    .buildkite/pipeline-utils/affected-packages/list_affected \
-      --strategy git --deep --merge-base "$GITHUB_PR_MERGE_BASE" --json \
-      > "$AFFECTED_MODULES_FILE"
+  # PR builds: GITHUB_PR_MERGE_BASE is computed by set_git_merge_base() in util.sh.
+  # On-merge builds: falls back to HEAD~1 (parent of the merge commit).
+  if [[ -n "${GITHUB_PR_MERGE_BASE:-}" ]]; then
+    echo "Merge base (PR): ${GITHUB_PR_MERGE_BASE}"
+  else
+    echo "GITHUB_PR_MERGE_BASE not set — using HEAD~1 as merge base (on-merge build)"
   fi
-
-  echo "--- Discover Playwright Configs and upload to Buildkite artifacts${AFFECTED_MODULES_FILE:+ (selective testing)}"
-  AFFECTED_FLAG=()
-  if [[ -n "$AFFECTED_MODULES_FILE" ]]; then
-    AFFECTED_FLAG=(--affected-modules "$AFFECTED_MODULES_FILE")
+  export AFFECTED_MERGE_BASE="${GITHUB_PR_MERGE_BASE:-HEAD~1}"
+
+  mkdir -p .scout
+  export AFFECTED_MODULES_FILE=".scout/affected_modules.json"
+
+  # Computes affected modules (writes AFFECTED_MODULES_FILE) and checks whether
+  # any critical Scout files were touched.
+  SCOUT_CRITICAL_FILES_TOUCHED=$(ts-node "$(dirname "${0}")/resolve_selective_testing.ts")
+  echo "Critical Scout files touched: ${SCOUT_CRITICAL_FILES_TOUCHED}"
+
+  echo "--- Discover Playwright Configs and upload to Buildkite artifacts (affected modules detected)"
+  SELECTIVE_SCOUT_DISCOVERY_FLAG=()
+  if [[ "${SELECTIVE_TESTING_ENABLED:-}" == "true" ]] \
+    && ! is_pr_with_label "scout:run-all-tests" \
+    && [[ "$SCOUT_CRITICAL_FILES_TOUCHED" != "true" ]]; then
+    SELECTIVE_SCOUT_DISCOVERY_FLAG=(--selective-testing)
+    echo "Selective testing: enabled (--selective-testing flag will be passed to discover-playwright-configs)"
+  else
+    echo "Selective testing: disabled — reason: SELECTIVE_TESTING_ENABLED=${SELECTIVE_TESTING_ENABLED:-false}, scout:run-all-tests label=$(is_pr_with_label "scout:run-all-tests" && echo yes || echo no), critical files touched=${SCOUT_CRITICAL_FILES_TOUCHED}"
   fi
   node scripts/scout discover-playwright-configs \
     --include-custom-servers \
     --target "$SCOUT_DISCOVERY_TARGET" \
-    "${AFFECTED_FLAG[@]}" \
+    --affected-modules "$AFFECTED_MODULES_FILE" \
+    "${SELECTIVE_SCOUT_DISCOVERY_FLAG[@]}" \
     --save
   cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
   buildkite-agent artifact upload "scout_playwright_configs.json"

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts

@@ -10,6 +10,7 @@
 import type { FlagsReader } from '@kbn/dev-cli-runner';
 import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { findPackageForPath } from '@kbn/repo-packages';
 import fs from 'fs';
 import {
   filterModulesByScoutCiConfig,
@@ -43,6 +44,10 @@ jest.mock('fs', () => {
   };
 });
 
+jest.mock('@kbn/repo-packages', () => ({
+  findPackageForPath: jest.fn(),
+}));
+
 jest.mock('@kbn/repo-info', () => ({
   REPO_ROOT: '/mock/repo/root',
 }));
@@ -139,6 +144,9 @@ describe('runDiscoverPlaywrightConfigs', () => {
     flagsReader.enum.mockReturnValue('all');
     flagsReader.boolean.mockReturnValue(false);
     flagsReader.arrayOfStrings.mockReturnValue([]);
+    flagsReader.string.mockImplementation(() => '');
+
+    (findPackageForPath as jest.Mock).mockReset();
 
     (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules);
     (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]);
@@ -313,6 +321,97 @@ describe('runDiscoverPlaywrightConfigs', () => {
     expect(foundMessage![0]).toContain('1 package(s)'); // packageA
   });
 
+  it('fails when --selective-testing is set without --affected-modules', () => {
+    flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing');
+    flagsReader.string.mockReturnValue('');
+
+    expect(() => runDiscoverPlaywrightConfigs(flagsReader, log)).toThrow(
+      '--selective-testing requires --affected-modules'
+    );
+  });
+
+  it('with --selective-testing and --affected-modules, limits discovery to affected modules only', () => {
+    flagsReader.enum.mockReturnValue('mki');
+    flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing');
+    flagsReader.string.mockImplementation((name: string) =>
+      name === 'affected-modules' ? '/mock/affected_modules.json' : ''
+    );
+
+    (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => {
+      if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' };
+      if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' };
+      if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' };
+      return undefined;
+    });
+
+    (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => {
+      if (readPath === '/mock/affected_modules.json') {
+        return JSON.stringify(['@kbn/pluginA']);
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('package.json')) {
+        return JSON.stringify({ name: 'kibana', version: '1.0.0' });
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('.yml')) {
+        return 'mock yaml content';
+      }
+      return '';
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const foundMessage = infoCalls.find((call) =>
+      call[0].includes('Found Playwright config files')
+    );
+    expect(foundMessage).toBeDefined();
+    expect(foundMessage![0]).toContain('1 plugin(s)');
+    expect(foundMessage![0]).toContain('0 package(s)');
+
+    expect(
+      infoCalls.some((call) =>
+        String(call[0]).includes('Selective testing: Scout discovery limited')
+      )
+    ).toBe(true);
+  });
+
+  it('with --affected-modules but without --selective-testing, still discovers all modules matching the target', () => {
+    flagsReader.enum.mockReturnValue('mki');
+    flagsReader.boolean.mockReturnValue(false);
+    flagsReader.string.mockImplementation((name: string) =>
+      name === 'affected-modules' ? '/mock/affected_modules_non_draft.json' : ''
+    );
+
+    (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => {
+      if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' };
+      if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' };
+      if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' };
+      return undefined;
+    });
+
+    (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => {
+      if (readPath === '/mock/affected_modules_non_draft.json') {
+        return JSON.stringify(['@kbn/pluginA']);
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('package.json')) {
+        return JSON.stringify({ name: 'kibana', version: '1.0.0' });
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('.yml')) {
+        return 'mock yaml content';
+      }
+      return '';
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const foundMessage = infoCalls.find((call) =>
+      call[0].includes('Found Playwright config files')
+    );
+    expect(foundMessage).toBeDefined();
+    expect(foundMessage![0]).toContain('2 plugin(s)');
+    expect(foundMessage![0]).toContain('1 package(s)');
+  });
+
   it('filters configs based on target tags for "mki" target (@cloud-serverless-*)', () => {
     flagsReader.enum.mockReturnValue('mki');
     flagsReader.boolean.mockReturnValue(false);
@@ -631,7 +730,7 @@ describe('runDiscoverPlaywrightConfigs', () => {
 
     const infoCalls = log.info.mock.calls;
     expect(infoCalls.length).toBeGreaterThan(0);
-    expect(infoCalls[0][0]).toContain('Found Playwright config files');
+    expect(infoCalls.some((call) => call[0].includes('Found Playwright config files'))).toBe(true);
 
     // Check that config logs include tags
     const configLogs = infoCalls.filter((call) => call[0].startsWith('- '));
@@ -705,7 +804,7 @@ describe('runDiscoverPlaywrightConfigs', () => {
     expect(Array.isArray(savedData)).toBe(true);
 
     expect(log.info).toHaveBeenCalledWith(
-      expect.stringContaining('Scout configs were filtered for CI. Saved')
+      expect.stringContaining('Scout configs saved for CI (full suite)')
     );
   });