[8.19] [scout] discover tests with custom server configs (#251297)

.buildkite/pipelines/on_merge.yml

@@ -209,7 +209,7 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+  - command: .buildkite/scripts/steps/test/scout/test_run_builder.sh
     label: 'Scout Test Run Builder'
     agents:
       image: family/kibana-ubuntu-2404
@@ -223,7 +223,7 @@ steps:
       - build
     env:
       SCOUT_CONFIGS_DEPS: 'build_scout_tests,quick_checks,checks,linting,linting_with_types,check_oas_snapshot,check_types'
-      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout/configs.sh'
     retry:
       automatic:
         - exit_status: '*'

.buildkite/scout_ci_config.yml

@@ -21,3 +21,7 @@ packages:
   disabled:
     - kbn-scout # Internal scout tests are run in advance to validate Scout integrity (see .buildkite/scripts/steps/test/scout_test_run_builder.sh)
     - kbn-scout-release-testing # Release tests will run separately as part of the release process
+
+# Define test configs to be excluded from automatic discovery & execution in CI environment (process.env.CI=true)
+excluded_configs:
+  - x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts

.buildkite/scripts/steps/test/scout_test_run_builder.sh

@@ -10,8 +10,8 @@ node scripts/scout run-playwright-test-check
 echo '--- Update Scout Test Config Manifests'
 node scripts/scout.js update-test-config-manifests
 
-echo '--- Discover Playwright Configs and upload to Buildkite artifacts (only stateful tests)'
-node scripts/scout discover-playwright-configs --target ech --save
+echo '--- Discover Playwright Configs and upload to Buildkite artifacts'
+node scripts/scout discover-playwright-configs --target ech --include-custom-servers --save
 cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
 buildkite-agent artifact upload "scout_playwright_configs.json"
 

src/platform/packages/private/kbn-scout-info/src/paths.ts

@@ -7,8 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import path from 'node:path';
 import { REPO_ROOT } from '@kbn/repo-info';
+import path from 'node:path';
 
 export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout');
 
@@ -33,7 +33,7 @@ export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
 );
 
 export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB =
-  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout';
+  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout{_*,}';
 
 export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX = new RegExp(
   `(?:src|x-pack)` +

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts

@@ -11,8 +11,11 @@ import type { FlagsReader } from '@kbn/dev-cli-runner';
 import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry';
 import type { ToolingLog } from '@kbn/tooling-log';
 import fs from 'fs';
-import { filterModulesByScoutCiConfig } from '../tests_discovery/search_configs';
 import { getServerRunFlagsFromTags } from '../tests_discovery/tag_utils';
+import {
+  filterModulesByScoutCiConfig,
+  getScoutCiExcludedConfigs,
+} from '../tests_discovery/search_configs';
 import type { ModuleDiscoveryInfo } from './config_discovery';
 import { runDiscoverPlaywrightConfigs } from './config_discovery';
 
@@ -51,6 +54,7 @@ jest.mock('@kbn/scout-info', () => ({
 
 jest.mock('../tests_discovery/search_configs', () => ({
   filterModulesByScoutCiConfig: jest.fn(),
+  getScoutCiExcludedConfigs: jest.fn(),
 }));
 
 jest.mock('@kbn/scout-reporting/src/registry', () => {
@@ -132,6 +136,7 @@ describe('runDiscoverPlaywrightConfigs', () => {
     flagsReader.arrayOfStrings.mockReturnValue([]);
 
     (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules);
+    (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]);
 
     // Default mock modules
     mockTestableModules.modules = [
@@ -326,6 +331,149 @@ describe('runDiscoverPlaywrightConfigs', () => {
     expect(foundMessage![0]).toContain('1 package(s)'); // packageA
   });
 
+  it('includes custom-server configs alongside defaults when "include-custom-servers" is true', () => {
+    flagsReader.enum.mockReturnValue('all');
+    flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers');
+
+    mockTestableModules.modules = [
+      {
+        name: 'pluginCustom',
+        group: 'groupCustom',
+        type: 'plugin' as const,
+        visibility: 'private' as const,
+        root: 'x-pack/platform/plugins/private/pluginCustom',
+        configs: [
+          {
+            path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'custom123',
+              tests: [
+                {
+                  id: 'customTest1',
+                  title: 'Custom Test 1',
+                  expectedStatus: 'passed',
+                  location: { file: 'custom.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+          {
+            path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'normal456',
+              tests: [
+                {
+                  id: 'normalTest1',
+                  title: 'Normal Test 1',
+                  expectedStatus: 'passed',
+                  location: { file: 'normal.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ];
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const configLogs = infoCalls.filter((call) => call[0].startsWith('- '));
+    expect(configLogs.length).toBeGreaterThan(0);
+    expect(configLogs.some((call) => call[0].includes('/test/scout_custom/'))).toBe(true);
+    expect(
+      configLogs.some((call) => call[0].includes('/pluginCustom/config.playwright.config.ts'))
+    ).toBe(true);
+  });
+
+  it('excludes configs listed in scout_ci_config.yml when running in CI', () => {
+    const originalCi = process.env.CI;
+    process.env.CI = 'true';
+
+    flagsReader.enum.mockReturnValue('all');
+    flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers');
+
+    const excludedConfigPath =
+      'x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts';
+
+    (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([excludedConfigPath]);
+
+    mockTestableModules.modules = [
+      {
+        name: 'pluginCspm',
+        group: 'security',
+        type: 'plugin' as const,
+        visibility: 'private' as const,
+        root: 'x-pack/solutions/security/plugins/cloud_security_posture',
+        configs: [
+          {
+            path: excludedConfigPath,
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: excludedConfigPath,
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'exclude123',
+              tests: [
+                {
+                  id: 'excludedTest',
+                  title: 'Excluded Test',
+                  expectedStatus: 'passed',
+                  location: { file: 'excluded.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+          {
+            path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'include456',
+              tests: [
+                {
+                  id: 'includedTest',
+                  title: 'Included Test',
+                  expectedStatus: 'passed',
+                  location: { file: 'included.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ];
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const configLogs = infoCalls.filter((call) => call[0].startsWith('- '));
+    expect(configLogs.some((call) => call[0].includes(excludedConfigPath))).toBe(false);
+    expect(
+      configLogs.some((call) => call[0].includes('/test/scout/ui/config.playwright.config.ts'))
+    ).toBe(true);
+
+    process.env.CI = originalCi;
+  });
+
   it('filters config tags to only include cross tags', () => {
     flagsReader.enum.mockReturnValue('ech'); // ESS_ONLY = ['@local-stateful-classic']
     flagsReader.boolean.mockReturnValue(false);

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -12,7 +12,10 @@ import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
 import { testableModules } from '@kbn/scout-reporting/src/registry';
 import type { ToolingLog } from '@kbn/tooling-log';
 import { saveFlattenedConfigGroups, saveModuleDiscoveryInfo } from '../tests_discovery/file_utils';
-import { filterModulesByScoutCiConfig } from '../tests_discovery/search_configs';
+import {
+  filterModulesByScoutCiConfig,
+  getScoutCiExcludedConfigs,
+} from '../tests_discovery/search_configs';
 import {
   collectUniqueTags,
   getServerRunFlagsFromTags,
@@ -82,6 +85,45 @@ const filterModulesByTargetTags = (
     .filter((module) => module.configs.length > 0);
 };
 
+const CUSTOM_SERVERS_PATH_PATTERN = /\/test\/scout_[^/]+/;
+
+const filterModulesByCustomServerPaths = (
+  modules: ModuleDiscoveryInfo[],
+  includeCustomServers: boolean
+): ModuleDiscoveryInfo[] => {
+  if (includeCustomServers) {
+    return modules;
+  }
+
+  return modules
+    .map((module) => ({
+      ...module,
+      configs: module.configs.filter((config) => {
+        const isCustomServerConfig = CUSTOM_SERVERS_PATH_PATTERN.test(config.path);
+        return !isCustomServerConfig;
+      }),
+    }))
+    .filter((module) => module.configs.length > 0);
+};
+
+const filterModulesByExcludedConfigPaths = (
+  modules: ModuleDiscoveryInfo[],
+  excludedConfigPaths: string[]
+): ModuleDiscoveryInfo[] => {
+  if (excludedConfigPaths.length === 0) {
+    return modules;
+  }
+
+  const excludedSet = new Set(excludedConfigPaths);
+
+  return modules
+    .map((module) => ({
+      ...module,
+      configs: module.configs.filter((config) => !excludedSet.has(config.path)),
+    }))
+    .filter((module) => module.configs.length > 0);
+};
+
 // Logs discovered modules in non-flattened format
 const logDiscoveredModules = (modules: ModuleDiscoveryInfo[], log: ToolingLog): void => {
   const { plugins: pluginCount, packages: packageCount } = countModulesByType(modules);
@@ -163,16 +205,24 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool
   const target = (flagsReader.enum('target', TARGET_TYPES) || 'all') as TargetType;
   const targetTags = getTestTagsForTarget(target);
   const flatten = flagsReader.boolean('flatten');
+  const includeCustomServers = flagsReader.boolean('include-custom-servers');
 
   // Build initial module discovery info
   const modulesWithTests = buildModuleDiscoveryInfo();
   // Filter modules by target tags and compute server run flags
-  const filteredModules = filterModulesByTargetTags(modulesWithTests, targetTags);
+  const filteredModulesByTags = filterModulesByTargetTags(modulesWithTests, targetTags);
+  const filteredModules = filterModulesByCustomServerPaths(
+    filteredModulesByTags,
+    includeCustomServers
+  );
+  const filteredModulesWithExcludedConfigs = process.env.CI
+    ? filterModulesByExcludedConfigPaths(filteredModules, getScoutCiExcludedConfigs())
+    : filteredModules;
   // Handle output based on flatten flag
   if (flatten) {
-    handleFlattenedOutput(filteredModules, flagsReader, log);
+    handleFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log);
   } else {
-    handleNonFlattenedOutput(filteredModules, flagsReader, log);
+    handleNonFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log);
   }
 };
 
@@ -202,14 +252,15 @@ export const discoverPlaywrightConfigsCmd: Command<void> = {
   validate against CI configuration, or save filtered results to a file.
 
   Options:
-    --target <target>  Filter configs by deployment target:
-                       - 'all': deployment-agnostic tags (default)
-                       - 'mki': serverless-only tags
-                       - 'ech': stateful-only tags
-    --validate         Validate that all discovered modules are registered in Scout CI config
-    --save             Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'
-    --flatten          Output configs in flattened format grouped by mode, group, and scout command
-                       (useful for Cloud test execution)
+    --target <target>         Filter configs by deployment target:
+                              - 'all': deployment-agnostic tags (default)
+                              - 'mki': serverless-only tags
+                              - 'ech': stateful-only tags
+    --include-custom-servers  Include configs under 'test/scout_*' paths for custom server setups
+    --validate                Validate that all discovered modules are registered in Scout CI config
+    --save                    Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'
+    --flatten                 Output configs in flattened format grouped by mode, group, and scout command
+                              (useful for Cloud test execution)
 
   Examples:
     # Discover all deployment-agnostic configs
@@ -218,6 +269,9 @@ export const discoverPlaywrightConfigsCmd: Command<void> = {
     # Discover serverless-only configs
     node scripts/scout discover-playwright-configs --target mki
 
+    # Discover local custom-server configs only
+    node scripts/scout discover-playwright-configs --include-custom-servers
+
     # Validate discovered configs against CI configuration
     node scripts/scout discover-playwright-configs --validate
 
@@ -229,8 +283,14 @@ export const discoverPlaywrightConfigsCmd: Command<void> = {
   `,
   flags: {
     string: ['target'],
-    boolean: ['save', 'validate', 'flatten'],
-    default: { target: 'all', save: false, validate: false, flatten: false },
+    boolean: ['save', 'validate', 'flatten', 'include-custom-servers'],
+    default: {
+      target: 'all',
+      save: false,
+      validate: false,
+      flatten: false,
+      'include-custom-servers': false,
+    },
   },
   run: ({ flagsReader, log }) => {
     runDiscoverPlaywrightConfigs(flagsReader, log);