[scout] discover tests with custom server configs

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -82,22 +82,6 @@ steps:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/test/scout_uiam_tests.sh
-        label: 'Scout UIAM Tests'
-        agents:
-          image: family/kibana-ubuntu-2404
-          imageProject: elastic-images-prod
-          provider: gcp
-          machineType: n2-standard-4
-          diskSizeGb: 115
-        timeout_in_minutes: 15
-        depends_on:
-          - build
-        retry:
-          automatic:
-            - exit_status: '*'
-              limit: 1
-
       - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
         label: 'Serverless Entity Analytics - Security Solution Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"

.buildkite/pipelines/on_merge.yml

@@ -217,22 +217,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/scout_uiam_tests.sh
-    label: 'Scout UIAM Tests'
-    agents:
-      image: family/kibana-ubuntu-2404
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      diskSizeGb: 105
-    timeout_in_minutes: 15
-    depends_on:
-      - build
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
     label: 'Serverless Entity Analytics - Security Cypress Tests'
     agents:

.buildkite/scout_ci_config.yml

@@ -3,7 +3,10 @@ plugins:
   enabled:
     - apm
     - console
+    - cloud_security_posture
     - discover_enhanced
+    - entity_store
+    - fleet
     - index_management
     - infra
     - maps
@@ -13,15 +16,14 @@ plugins:
     - profiling
     - search_getting_started
     - search_homepage
+    - security
     - security_solution
     - slo
     - spaces
     - streams
     - streams_app
-    - workflows_extensions
     - transform
-    - fleet
-    - entity_store
+    - workflows_extensions
   disabled:
 
 packages:
@@ -30,3 +32,7 @@ packages:
   disabled:
     - kbn-scout # Internal scout tests are run in advance to validate Scout integrity (see .buildkite/scripts/steps/test/scout_test_run_builder.sh)
     - kbn-scout-release-testing # Release tests will run separately as part of the release process
+
+# Define test configs to be excluded from automatic discovery & execution in CI environment (process.env.CI=true)
+excluded_configs:
+  - x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -73,9 +73,6 @@ const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new R
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_tests.yml'));
 
-    // A temporary pipeline for UIAM tests that should be removed when the main Scout pipeline supports custom configs.
-    pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_uiam_tests.yml'));
-
     if (await doAnyChangesMatch([/^src\/platform\/packages\/private\/kbn-handlebars/])) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
     }

.buildkite/scripts/steps/test/scout_test_run_builder.sh

@@ -11,7 +11,7 @@ echo '--- Update Scout Test Config Manifests'
 node scripts/scout.js update-test-config-manifests
 
 echo '--- Discover Playwright Configs and upload to Buildkite artifacts'
-node scripts/scout discover-playwright-configs --save
+node scripts/scout discover-playwright-configs --include-custom-servers --save
 cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json
 buildkite-agent artifact upload "scout_playwright_configs.json"
 

src/platform/packages/private/kbn-scout-info/src/paths.ts

@@ -7,8 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import path from 'node:path';
 import { REPO_ROOT } from '@kbn/repo-info';
+import path from 'node:path';
 
 export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout');
 
@@ -33,7 +33,7 @@ export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
 );
 
 export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB =
-  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout';
+  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout{_*,}';
 
 export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX = new RegExp(
   `(?:src|x-pack)` +

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts

@@ -11,7 +11,10 @@ import type { FlagsReader } from '@kbn/dev-cli-runner';
 import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry';
 import type { ToolingLog } from '@kbn/tooling-log';
 import fs from 'fs';
-import { filterModulesByScoutCiConfig } from '../tests_discovery/search_configs';
+import {
+  filterModulesByScoutCiConfig,
+  getScoutCiExcludedConfigs,
+} from '../tests_discovery/search_configs';
 import type { ModuleDiscoveryInfo } from './config_discovery';
 import { runDiscoverPlaywrightConfigs } from './config_discovery';
 
@@ -49,6 +52,7 @@ jest.mock('@kbn/scout-info', () => ({
 
 jest.mock('../tests_discovery/search_configs', () => ({
   filterModulesByScoutCiConfig: jest.fn(),
+  getScoutCiExcludedConfigs: jest.fn(),
 }));
 
 jest.mock('@kbn/scout-reporting/src/registry', () => {
@@ -126,6 +130,7 @@ describe('runDiscoverPlaywrightConfigs', () => {
     flagsReader.arrayOfStrings.mockReturnValue([]);
 
     (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules);
+    (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]);
 
     // Default mock modules
     mockTestableModules.modules = [
@@ -320,6 +325,149 @@ describe('runDiscoverPlaywrightConfigs', () => {
     expect(foundMessage![0]).toContain('1 package(s)'); // packageA
   });
 
+  it('includes custom-server configs alongside defaults when "include-custom-servers" is true', () => {
+    flagsReader.enum.mockReturnValue('all');
+    flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers');
+
+    mockTestableModules.modules = [
+      {
+        name: 'pluginCustom',
+        group: 'groupCustom',
+        type: 'plugin' as const,
+        visibility: 'private' as const,
+        root: 'x-pack/platform/plugins/private/pluginCustom',
+        configs: [
+          {
+            path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'custom123',
+              tests: [
+                {
+                  id: 'customTest1',
+                  title: 'Custom Test 1',
+                  expectedStatus: 'passed',
+                  location: { file: 'custom.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+          {
+            path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'normal456',
+              tests: [
+                {
+                  id: 'normalTest1',
+                  title: 'Normal Test 1',
+                  expectedStatus: 'passed',
+                  location: { file: 'normal.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ];
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const configLogs = infoCalls.filter((call) => call[0].startsWith('- '));
+    expect(configLogs.length).toBeGreaterThan(0);
+    expect(configLogs.some((call) => call[0].includes('/test/scout_custom/'))).toBe(true);
+    expect(
+      configLogs.some((call) => call[0].includes('/pluginCustom/config.playwright.config.ts'))
+    ).toBe(true);
+  });
+
+  it('excludes configs listed in scout_ci_config.yml when running in CI', () => {
+    const originalCi = process.env.CI;
+    process.env.CI = 'true';
+
+    flagsReader.enum.mockReturnValue('all');
+    flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers');
+
+    const excludedConfigPath =
+      'x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts';
+
+    (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([excludedConfigPath]);
+
+    mockTestableModules.modules = [
+      {
+        name: 'pluginCspm',
+        group: 'security',
+        type: 'plugin' as const,
+        visibility: 'private' as const,
+        root: 'x-pack/solutions/security/plugins/cloud_security_posture',
+        configs: [
+          {
+            path: excludedConfigPath,
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: excludedConfigPath,
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'exclude123',
+              tests: [
+                {
+                  id: 'excludedTest',
+                  title: 'Excluded Test',
+                  expectedStatus: 'passed',
+                  location: { file: 'excluded.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+          {
+            path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts',
+            category: 'ui',
+            type: 'playwright',
+            manifest: {
+              path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts',
+              exists: true,
+              lastModified: '2024-01-01T00:00:00Z',
+              sha1: 'include456',
+              tests: [
+                {
+                  id: 'includedTest',
+                  title: 'Included Test',
+                  expectedStatus: 'passed',
+                  location: { file: 'included.spec.ts', line: 1, column: 1 },
+                  tags: ['@ess'],
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ];
+
+    runDiscoverPlaywrightConfigs(flagsReader, log);
+
+    const infoCalls = log.info.mock.calls;
+    const configLogs = infoCalls.filter((call) => call[0].startsWith('- '));
+    expect(configLogs.some((call) => call[0].includes(excludedConfigPath))).toBe(false);
+    expect(
+      configLogs.some((call) => call[0].includes('/test/scout/ui/config.playwright.config.ts'))
+    ).toBe(true);
+
+    process.env.CI = originalCi;
+  });
+
   it('filters config tags to only include cross tags', () => {
     flagsReader.enum.mockReturnValue('ech'); // ESS_ONLY = ['@ess']
     flagsReader.boolean.mockReturnValue(false);