[scout] enable selective testing for pull requests

.buildkite/scripts/steps/test/scout/test_run_builder.sh

@@ -68,10 +68,16 @@ else
   if [[ -n "$AFFECTED_MODULES_FILE" ]]; then
     AFFECTED_FLAG=(--affected-modules "$AFFECTED_MODULES_FILE")
   fi
+  # Draft PRs only: --selective-testing is rejected by discover-playwright-configs unless GITHUB_PR_DRAFT=true.
+  SELECTIVE_SCOUT_DISCOVERY_FLAG=()
+  if [[ "${GITHUB_PR_DRAFT:-false}" == "true" ]] && [[ -n "$AFFECTED_MODULES_FILE" ]]; then
+    SELECTIVE_SCOUT_DISCOVERY_FLAG=(--selective-testing)
+  fi
   node scripts/scout discover-playwright-configs \
     --include-custom-servers \
     --target "$SCOUT_DISCOVERY_TARGET" \
     "${AFFECTED_FLAG[@]}" \
+    "${SELECTIVE_SCOUT_DISCOVERY_FLAG[@]}" \
     --save
   cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
   buildkite-agent artifact upload "scout_playwright_configs.json"

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts

@@ -10,6 +10,7 @@
 import type { FlagsReader } from '@kbn/dev-cli-runner';
 import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { findPackageForPath } from '@kbn/repo-packages';
 import fs from 'fs';
 import {
   filterModulesByScoutCiConfig,
@@ -43,6 +44,10 @@ jest.mock('fs', () => {
   };
 });
 
+jest.mock('@kbn/repo-packages', () => ({
+  findPackageForPath: jest.fn(),
+}));
+
 jest.mock('@kbn/repo-info', () => ({
   REPO_ROOT: '/mock/repo/root',
 }));
@@ -139,6 +144,11 @@ describe('runDiscoverPlaywrightConfigs', () => {
     flagsReader.enum.mockReturnValue('all');
     flagsReader.boolean.mockReturnValue(false);
     flagsReader.arrayOfStrings.mockReturnValue([]);
+    flagsReader.string.mockImplementation(() => '');
+
+    delete process.env.GITHUB_PR_DRAFT;
+
+    (findPackageForPath as jest.Mock).mockReset();
 
     (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules);
     (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]);
@@ -313,6 +323,112 @@ describe('runDiscoverPlaywrightConfigs', () => {
     expect(foundMessage![0]).toContain('1 package(s)'); // packageA
   });
 
+  it('fails when --selective-testing is set without --affected-modules', () => {
+    flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing');
+    flagsReader.string.mockReturnValue('');
+
+    expect(() => runDiscoverPlaywrightConfigs(flagsReader, log)).toThrow(
+      '--selective-testing requires --affected-modules'
+    );
+  });
+
+  it('fails when --selective-testing is used on a non-draft PR (GITHUB_PR_DRAFT unset)', () => {
+    delete process.env.GITHUB_PR_DRAFT;
+    flagsReader.enum.mockReturnValue('mki');
+    flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing');
+    flagsReader.string.mockImplementation((name: string) =>
+      name === 'affected-modules' ? '/mock/affected_modules.json' : ''
+    );
+
+    expect(() => runDiscoverPlaywrightConfigs(flagsReader, log)).toThrow(
+      '--selective-testing is only valid for draft pull requests'
+    );
+  });
+
+  it('with --selective-testing and --affected-modules on a draft PR, limits discovery to affected modules only', () => {
+    process.env.GITHUB_PR_DRAFT = 'true';
+
+    flagsReader.enum.mockReturnValue('mki');
+    flagsReader.boolean.mockImplementation((flag: string) => flag === 'selective-testing');
+    flagsReader.string.mockImplementation((name: string) =>
+      name === 'affected-modules' ? '/mock/affected_modules.json' : ''
+    );
+
+    (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => {
+      if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' };
+      if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' };
+      if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' };
+      return undefined;
+    });
+
+    (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => {
+      if (readPath === '/mock/affected_modules.json') {
+        return JSON.stringify(['@kbn/pluginA']);
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('package.json')) {
+        return JSON.stringify({ name: 'kibana', version: '1.0.0' });
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('.yml')) {
+        return 'mock yaml content';
+      }
+      return '';
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const foundMessage = infoCalls.find((call) =>
+      call[0].includes('Found Playwright config files')
+    );
+    expect(foundMessage).toBeDefined();
+    expect(foundMessage![0]).toContain('1 plugin(s)');
+    expect(foundMessage![0]).toContain('0 package(s)');
+
+    expect(
+      infoCalls.some((call) =>
+        String(call[0]).includes('Selective testing: Scout discovery limited')
+      )
+    ).toBe(true);
+  });
+
+  it('with --affected-modules but without --selective-testing, still discovers all modules matching the target', () => {
+    flagsReader.enum.mockReturnValue('mki');
+    flagsReader.boolean.mockReturnValue(false);
+    flagsReader.string.mockImplementation((name: string) =>
+      name === 'affected-modules' ? '/mock/affected_modules_non_draft.json' : ''
+    );
+
+    (findPackageForPath as jest.Mock).mockImplementation((_root: string, absPath: string) => {
+      if (absPath.includes('/pluginA/')) return { id: '@kbn/pluginA' };
+      if (absPath.includes('/pluginB/')) return { id: '@kbn/pluginB' };
+      if (absPath.includes('/packageA/')) return { id: '@kbn/packageA' };
+      return undefined;
+    });
+
+    (fs.readFileSync as jest.Mock).mockImplementation((readPath: string) => {
+      if (readPath === '/mock/affected_modules_non_draft.json') {
+        return JSON.stringify(['@kbn/pluginA']);
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('package.json')) {
+        return JSON.stringify({ name: 'kibana', version: '1.0.0' });
+      }
+      if (typeof readPath === 'string' && readPath.endsWith('.yml')) {
+        return 'mock yaml content';
+      }
+      return '';
+    });
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const foundMessage = infoCalls.find((call) =>
+      call[0].includes('Found Playwright config files')
+    );
+    expect(foundMessage).toBeDefined();
+    expect(foundMessage![0]).toContain('2 plugin(s)');
+    expect(foundMessage![0]).toContain('1 package(s)');
+  });
+
   it('filters configs based on target tags for "mki" target (@cloud-serverless-*)', () => {
     flagsReader.enum.mockReturnValue('mki');
     flagsReader.boolean.mockReturnValue(false);

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { createFailError } from '@kbn/dev-cli-errors';
 import type { Command, FlagsReader } from '@kbn/dev-cli-runner';
 import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
 import { testableModules } from '@kbn/scout-reporting/src/registry';
@@ -250,18 +251,45 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool
   const targetTags = getTestTagsForTarget(target);
   const flatten = flagsReader.boolean('flatten');
   const includeCustomServers = flagsReader.boolean('include-custom-servers');
+  const selectiveTesting = flagsReader.boolean('selective-testing');
   const affectedModulesPath = flagsReader.string('affected-modules');
 
+  if (selectiveTesting && !affectedModulesPath) {
+    throw createFailError(
+      '--selective-testing requires --affected-modules (JSON array of @kbn/ IDs from list_affected).'
+    );
+  }
+
+  if (selectiveTesting && process.env.GITHUB_PR_DRAFT !== 'true') {
+    throw createFailError(
+      '--selective-testing is only valid for draft pull requests (GITHUB_PR_DRAFT=true from the PR build trigger).'
+    );
+  }
+
   // Build initial module discovery info
   const modulesWithTests = buildModuleDiscoveryInfo();
 
-  // Mark affected status when selective testing is enabled (all modules kept, isAffected set)
+  // --affected-modules only: keep every Scout module that passes target/CI filters; set isAffected
+  // per module so CI step labels can use an "affected " prefix where the PR touched that @kbn/ ID.
   const modulesAfterAffectedMark = affectedModulesPath
     ? markModulesAffectedStatus(modulesWithTests, affectedModulesPath, log)
     : modulesWithTests;
 
+  // Draft PRs only (validated above): narrow to affected module groups; steps keep the "affected " prefix.
+  const limitDiscoveryToAffectedModules = selectiveTesting;
+
+  const modulesForTargetTags = limitDiscoveryToAffectedModules
+    ? modulesAfterAffectedMark.filter((m) => m.isAffected === true)
+    : modulesAfterAffectedMark;
+
+  if (limitDiscoveryToAffectedModules) {
+    log.info(
+      `Selective testing: Scout discovery limited to affected modules (${modulesForTargetTags.length} of ${modulesAfterAffectedMark.length})`
+    );
+  }
+
   // Filter modules by target tags and compute server run flags
-  const filteredModulesByTags = filterModulesByTargetTags(modulesAfterAffectedMark, targetTags);
+  const filteredModulesByTags = filterModulesByTargetTags(modulesForTargetTags, targetTags);
   const filteredModules = filterModulesByCustomServerPaths(
     filteredModulesByTags,
     includeCustomServers
@@ -294,6 +322,11 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool
  * Output formats:
  * - Standard: Lists modules grouped by plugin/package with their configs and tags
  * - Flattened: Groups configs by deployment mode (stateful/serverless), group, and run mode
+ *
+ * Affected modules:
+ * - With --affected-modules, all modules are still considered; isAffected flags drive the
+ *   "affected " Buildkite step prefix. With --selective-testing (draft PRs only: GITHUB_PR_DRAFT=true),
+ *   only affected module groups are emitted; those steps keep the same prefix.
  */
 export const discoverPlaywrightConfigsCmd: Command<void> = {
   name: 'discover-playwright-configs',
@@ -311,9 +344,13 @@ export const discoverPlaywrightConfigsCmd: Command<void> = {
                               - 'local-stateful-only': @local-stateful-* tags only
                               - 'mki': @cloud-serverless-* tags
                               - 'ech': @cloud-stateful-* tags
-    --affected-modules <file>  Path to a JSON file containing affected @kbn/ module IDs
-                              (produced by list_affected). All modules run; affected ones
-                              get "affected" prefix in Buildkite step (selective testing).
+    --affected-modules <file>  Path to a JSON file of affected @kbn/ module IDs (list_affected).
+                              All Scout modules still go through discovery; each module is marked
+                              isAffected so CI can prefix steps with "affected " when the PR
+                              touches that module. Combine with --selective-testing to emit only
+                              affected module groups.
+    --selective-testing       Draft PRs only (GITHUB_PR_DRAFT=true). Requires --affected-modules.
+                              Limits output / Scout CI steps to affected modules; labels unchanged.
     --include-custom-servers  Include configs under 'test/scout_*' paths for custom server setups
     --validate                Validate that all discovered modules are registered in Scout CI config
     --save                    Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'
@@ -348,18 +385,22 @@ export const discoverPlaywrightConfigsCmd: Command<void> = {
     # Save flattened configs for Cloud test execution
     node scripts/scout discover-playwright-configs --flatten --save
 
-    # Selective testing: only configs for affected modules
+    # Affected-module labels on every Scout group (full CI matrix)
     node scripts/scout discover-playwright-configs --affected-modules .scout/affected_modules.json --save
+
+    # Only affected module groups (draft PR; requires GITHUB_PR_DRAFT=true, e.g. CI)
+    GITHUB_PR_DRAFT=true node scripts/scout discover-playwright-configs --affected-modules .scout/affected_modules.json --selective-testing --save
   `,
   flags: {
     string: ['target', 'affected-modules'],
-    boolean: ['save', 'validate', 'flatten', 'include-custom-servers'],
+    boolean: ['save', 'validate', 'flatten', 'include-custom-servers', 'selective-testing'],
     default: {
       target: 'all',
       save: false,
       validate: false,
       flatten: false,
       'include-custom-servers': false,
+      'selective-testing': false,
     },
   },
   run: ({ flagsReader, log }) => {

src/platform/packages/shared/kbn-scout/src/tests_discovery/affected_modules.ts

@@ -48,8 +48,8 @@ export const readAffectedModules = (filePath: string, log: ToolingLog): Set<stri
 };
 
 /**
- * Mark modules with isAffected based on the affected modules set.
- * All modules are returned; none are filtered out.
+ * Mark modules with isAffected based on the affected modules set (see --affected-modules).
+ * All modules are returned; use --selective-testing in discover-playwright-configs to drop non-affected.
  *
  * Behavior:
  * - Module maps to an affected @kbn/ ID -> isAffected: true

src/platform/packages/shared/kbn-scout/src/tests_discovery/types.ts

@@ -20,7 +20,7 @@ export interface ModuleDiscoveryInfo {
   name: string;
   group: string;
   type: 'plugin' | 'package';
-  /** When selective testing is enabled, true if this module's @kbn/ ID is in the affected set */
+  /** Set when using --affected-modules: true if this module's @kbn/ ID is in the affected set */
   isAffected?: boolean;
   configs: {
     path: string;