[9.0] [scout] extend config-discovery with CI validator (#214403)

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -83,5 +83,7 @@ steps:
       PING_SLACK_TEAM: "@appex-qa-team"
     retry:
       automatic:
+        - exit_status: 10
+          limit: 0
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/scout_tests.yml

@@ -8,5 +8,8 @@ steps:
       SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
     retry:
       automatic:
+        # No retry when Scout CI config validation fails
+        - exit_status: 10
+          limit: 0
         - exit_status: '*'
           limit: 1

.buildkite/scout_ci_config.yml

@@ -0,0 +1,8 @@
+# Define which plugins should be run or skipped in Scout CI pipeline
+ui_tests:
+  enabled:
+    - apm
+    - discover_enhanced
+    - maps
+    - observability_onboarding
+  disabled:

.buildkite/scripts/steps/checks/check_scout_config.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check for unregistered Scout Playwright configs
+node scripts/scout discover-playwright-configs --validate 

.buildkite/scripts/steps/checks/quick_checks.txt

@@ -20,3 +20,4 @@
 .buildkite/scripts/steps/checks/styled_components_mapping.sh
 .buildkite/scripts/steps/checks/dependencies_missing_owner.sh
 .buildkite/scripts/steps/checks/validate_pipelines.sh
+.buildkite/scripts/steps/checks/check_scout_config.sh

.github/CODEOWNERS

@@ -1593,8 +1593,10 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /.ci/.storybook @elastic/kibana-operations
 
 # QA - Appex QA
+.buildkite/scout_ci_config.yml @elastic/appex-qa
 /x-pack/platform/plugins/shared/maps/ui_tests @elastic/appex-qa # temporarily
 /x-pack/platform/plugins/private/discover_enhanced/ui_tests/ @elastic/appex-qa # temporarily
+/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/discover_cdp_perf.spec.ts @elastic/kibana-data-discovery # test tracks bundle size limits
 /x-pack/test/functional/fixtures/package_registry_config.yml @elastic/appex-qa # No usages found
 /x-pack/test/functional/fixtures/kbn_archiver/packaging.json @elastic/appex-qa # No usages found
 /x-pack/test/functional/es_archives/filebeat @elastic/appex-qa

src/platform/packages/shared/kbn-scout/README.md

@@ -9,6 +9,7 @@
 3. Key Components
 4. How to Use
 5. Contributing
+6. Running tests on CI
 
 ### Overview
 
@@ -306,3 +307,20 @@ export const scoutTestFixtures = mergeTests(
   - `Fixtures` can combine browser interactions with API requests, but they should be used wisely, especially with the `test` scope: a new instance of the fixture is created for **every test block**. If a fixture performs expensive operations (API setup, data ingestion), excessive usage can **slow down** the test suite runtime. Consider using `worker` scope when appropriate to reuse instances across tests within a worker.
 - **Add Unit Tests:** Include tests for new logic where applicable, ensuring it works as expected.
 - **Playwright documentation:** [Official best practices](https://playwright.dev/docs/best-practices)
+
+
+### Running tests on CI
+Scout is still in active development, which means frequent code changes may sometimes cause test failures. To maintain stability, we currently do not run Scout tests for every PR and encourage teams to limit the number of tests they add for now.
+
+If a test is difficult to stabilize within a reasonable timeframe, we reserve the right to disable it or even all tests for particular plugin.
+
+To manage Scout test execution, we use the `.buildkite/scout_ci_config.yml` file, where Kibana plugins with Scout tests are registered. If you're unsure about the stability of your tests, please add your plugin under the `disabled` section.
+
+You can check whether your plugin is already registered by running:
+```bash
+node scripts/scout discover-playwright-configs --validate
+```
+On CI we run Scout tests only for `enabled` plugins:
+
+For PRs, Scout tests run only if there are changes to registered plugins or Scout-related packages.
+On merge commits, Scout tests run in a non-blocking mode.

src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts

@@ -13,6 +13,7 @@ import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info';
 import path from 'path';
 import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config';
 import { measurePerformance } from '../common';
+import { validateWithScoutCiConfig } from '../config/discovery';
 
 /**
  * Discover Playwright configuration files with Scout tests
@@ -24,18 +25,19 @@ export const discoverPlaywrightConfigs: Command<void> = {
 
   Common usage:
     node scripts/scout discover-playwright-configs --searchPaths <search_paths>
-    node scripts/scout discover-playwright-configs --save
+    node scripts/scout discover-playwright-configs --validate // validate if all plugins are registered in the Scout CI config
+    node scripts/scout discover-playwright-configs --save // save the discovered Playwright config files to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'
     node scripts/scout discover-playwright-configs
   `,
   flags: {
     string: ['searchPaths'],
-    boolean: ['save'],
-    default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false },
+    boolean: ['save', 'validate'],
+    default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false, validate: false },
   },
   run: ({ flagsReader, log }) => {
     const searchPaths = flagsReader.arrayOfStrings('searchPaths')!;
 
-    const pluginsMap = measurePerformance(log, 'Discovering playwright config files', () => {
+    let pluginsMap = measurePerformance(log, 'Discovering Playwright config files', () => {
       return getScoutPlaywrightConfigs(searchPaths, log);
     });
 
@@ -44,6 +46,21 @@ export const discoverPlaywrightConfigs: Command<void> = {
         ? 'No Playwright config files found'
         : `Found Playwright config files in '${pluginsMap.size}' plugins`;
 
+    if (!flagsReader.boolean('save')) {
+      log.info(finalMessage);
+
+      pluginsMap.forEach((data, plugin) => {
+        log.info(`${data.group} / [${plugin}] plugin:`);
+        data.configs.map((file) => {
+          log.info(`- ${file}`);
+        });
+      });
+    }
+
+    if (flagsReader.boolean('validate')) {
+      pluginsMap = validateWithScoutCiConfig(log, pluginsMap);
+    }
+
     if (flagsReader.boolean('save')) {
       const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH);
 
@@ -56,17 +73,9 @@ export const discoverPlaywrightConfigs: Command<void> = {
         JSON.stringify(Object.fromEntries(pluginsMap), null, 2)
       );
 
-      log.info(`${finalMessage}. Saved to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`);
-      return;
+      log.info(
+        `${finalMessage}.\nSaved '${pluginsMap.size}' plugins to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`
+      );
     }
-
-    log.info(finalMessage);
-
-    pluginsMap.forEach((data, plugin) => {
-      log.info(`${data.group} / [${plugin}] plugin:`);
-      data.configs.map((file) => {
-        log.info(`- ${file}`);
-      });
-    });
   },
 };

src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts

@@ -7,11 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { ToolingLog } from '@kbn/tooling-log';
 import fastGlob from 'fast-glob';
-import { getScoutPlaywrightConfigs } from './search_configs';
+import yaml from 'js-yaml';
+import { ToolingLog } from '@kbn/tooling-log';
+import { getScoutPlaywrightConfigs, validateWithScoutCiConfig } from './search_configs';
 
 jest.mock('fast-glob');
+jest.mock('js-yaml');
 
 describe('getScoutPlaywrightConfigs', () => {
   let mockLog: ToolingLog;
@@ -80,3 +82,99 @@ describe('getScoutPlaywrightConfigs', () => {
     );
   });
 });
+
+describe('validateWithScoutCiConfig', () => {
+  let mockLog: ToolingLog;
+  const mockScoutCiConfig = {
+    ui_tests: {
+      enabled: ['pluginA', 'pluginB'],
+      disabled: ['pluginC'],
+    },
+  };
+
+  beforeEach(() => {
+    mockLog = new ToolingLog({ level: 'verbose', writeTo: process.stdout });
+    jest.spyOn(mockLog, 'warning').mockImplementation(jest.fn());
+    (yaml.load as jest.Mock).mockReturnValue(mockScoutCiConfig);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should return only enabled plugins', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginA',
+        {
+          group: 'group1',
+          pluginPath: 'pluginPathA',
+          usesParallelWorkers: true,
+          configs: ['configA'],
+        },
+      ],
+      [
+        'pluginB',
+        {
+          group: 'group1',
+          pluginPath: 'pluginPathB',
+          usesParallelWorkers: false,
+          configs: ['configB1', 'configB2'],
+        },
+      ],
+      [
+        'pluginC',
+        {
+          group: 'group2',
+          pluginPath: 'pluginPathC',
+          usesParallelWorkers: true,
+          configs: ['configC'],
+        },
+      ],
+    ]);
+
+    const result = validateWithScoutCiConfig(mockLog, pluginsWithConfigs);
+    expect(result.size).toBe(2);
+    expect(result.has('pluginA')).toBe(true);
+    expect(result.has('pluginB')).toBe(true);
+    expect(result.has('pluginC')).toBe(false);
+  });
+
+  it('should throw an error if plugins are not registered in Scout CI config', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginX',
+        {
+          group: 'groupX',
+          pluginPath: 'pluginPathX',
+          usesParallelWorkers: true,
+          configs: ['configX'],
+        },
+      ],
+    ]);
+
+    expect(() => validateWithScoutCiConfig(mockLog, pluginsWithConfigs)).toThrow(
+      "The following plugins are not registered in Scout CI config '.buildkite/scout_ci_config.yml':\n- pluginX"
+    );
+  });
+
+  it('should log a warning for disabled plugins', () => {
+    const pluginsWithConfigs = new Map([
+      [
+        'pluginC',
+        {
+          group: 'group2',
+          pluginPath: 'pluginPathC',
+          usesParallelWorkers: true,
+          configs: ['configC'],
+        },
+      ],
+    ]);
+
+    validateWithScoutCiConfig(mockLog, pluginsWithConfigs);
+
+    expect(mockLog.warning).toHaveBeenCalledWith(
+      `The following plugins are disabled in '.buildkite/scout_ci_config.yml' and will be excluded from CI run\n- pluginC`
+    );
+  });
+});

src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts

@@ -10,6 +10,10 @@
 import fastGlob from 'fast-glob';
 import path from 'path';
 import { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/repo-info';
+import fs from 'fs';
+import yaml from 'js-yaml';
+import { createFailError } from '@kbn/dev-cli-errors';
 
 export const DEFAULT_TEST_PATH_PATTERNS = ['src/platform/plugins', 'x-pack/**/plugins'];
 
@@ -88,3 +92,56 @@ export const getScoutPlaywrightConfigs = (searchPaths: string[], log: ToolingLog
 
   return pluginsWithConfigs;
 };
+
+export const validateWithScoutCiConfig = (
+  log: ToolingLog,
+  pluginsWithConfigs: Map<string, PluginScoutConfig>
+) => {
+  const scoutCiConfigRelPath = path.join('.buildkite', 'scout_ci_config.yml');
+  const scoutCiConfigPath = path.resolve(REPO_ROOT, scoutCiConfigRelPath);
+  const ciConfig = yaml.load(fs.readFileSync(scoutCiConfigPath, 'utf8')) as {
+    ui_tests: {
+      enabled?: string[];
+      disabled?: string[];
+    };
+  };
+
+  const enabledPlugins = new Set(ciConfig.ui_tests.enabled || []);
+  const disabledPlugins = new Set(ciConfig.ui_tests.disabled || []);
+  const allRegisteredPlugins = new Set([...enabledPlugins, ...disabledPlugins]);
+
+  const unregisteredPlugins: string[] = [];
+  const filteredPlugins = new Map<string, PluginScoutConfig>();
+
+  for (const [pluginName, config] of pluginsWithConfigs.entries()) {
+    if (!allRegisteredPlugins.has(pluginName)) {
+      unregisteredPlugins.push(pluginName);
+    } else if (enabledPlugins.has(pluginName)) {
+      filteredPlugins.set(pluginName, config);
+    }
+  }
+
+  if (unregisteredPlugins.length > 0) {
+    throw createFailError(
+      `The following plugins are not registered in Scout CI config '${scoutCiConfigRelPath}':\n${unregisteredPlugins
+        .map((plugin) => {
+          return `- ${plugin}`;
+        })
+        .join('\n')}\nRead more: src/platform/packages/shared/kbn-scout/README.md`
+    );
+  }
+
+  if (disabledPlugins.size > 0) {
+    log.warning(
+      `The following plugins are disabled in '${scoutCiConfigRelPath}' and will be excluded from CI run\n${[
+        ...disabledPlugins,
+      ]
+        .map((plugin) => {
+          return `- ${plugin}`;
+        })
+        .join('\n')}`
+    );
+  }
+
+  return filteredPlugins;
+};