diff --git a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml index 22b9223e62373..2bda7dcbcee52 100644 --- a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml +++ b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml @@ -83,5 +83,7 @@ steps: PING_SLACK_TEAM: "@appex-qa-team" retry: automatic: + - exit_status: 10 + limit: 0 - exit_status: '*' limit: 1 diff --git a/.buildkite/pipelines/pull_request/scout_tests.yml b/.buildkite/pipelines/pull_request/scout_tests.yml index 127056611fcb1..0d52fb37c12d3 100644 --- a/.buildkite/pipelines/pull_request/scout_tests.yml +++ b/.buildkite/pipelines/pull_request/scout_tests.yml @@ -8,5 +8,8 @@ steps: SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh' retry: automatic: + # No retry when Scout CI config validation fails + - exit_status: 10 + limit: 0 - exit_status: '*' limit: 1 diff --git a/.buildkite/scout_ci_config.yml b/.buildkite/scout_ci_config.yml new file mode 100644 index 0000000000000..d11a25593419c --- /dev/null +++ b/.buildkite/scout_ci_config.yml @@ -0,0 +1,7 @@ +# Define which plugins should be run or skipped in Scout CI pipeline +ui_tests: + enabled: + - discover_enhanced + - maps + - observability_onboarding + disabled: diff --git a/.buildkite/scripts/steps/checks/check_scout_config.sh b/.buildkite/scripts/steps/checks/check_scout_config.sh new file mode 100644 index 0000000000000..763a3025469d8 --- /dev/null +++ b/.buildkite/scripts/steps/checks/check_scout_config.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +echo --- Check for unregistered Scout Playwright configs +node scripts/scout discover-playwright-configs --validate diff --git a/.buildkite/scripts/steps/checks/quick_checks.txt b/.buildkite/scripts/steps/checks/quick_checks.txt index 93c192d683dfc..1b738699d8022 100644 --- a/.buildkite/scripts/steps/checks/quick_checks.txt +++ b/.buildkite/scripts/steps/checks/quick_checks.txt @@ -18,3 +18,4 @@ .buildkite/scripts/steps/checks/renovate.sh .buildkite/scripts/steps/checks/native_modules.sh .buildkite/scripts/steps/checks/styled_components_mapping.sh +.buildkite/scripts/steps/checks/check_scout_config.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d70d619ea3fd6..72be99c8698ff 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1620,6 +1620,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql /.ci/.storybook @elastic/kibana-operations # QA - Appex QA +.buildkite/scout_ci_config.yml @elastic/appex-qa /x-pack/test/.gitignore @elastic/appex-qa /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/es/roles.yml @elastic/appex-qa /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/oblt/roles.yml @elastic/appex-qa diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 0e236e883fae4..6c6072f1b3777 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -9,6 +9,7 @@ 3. Key Components 4. How to Use 5. Contributing +6. Running tests on CI ### Overview @@ -306,3 +307,20 @@ export const scoutTestFixtures = mergeTests( - `Fixtures` can combine browser interactions with API requests, but they should be used wisely, especially with the `test` scope: a new instance of the fixture is created for **every test block**. If a fixture performs expensive operations (API setup, data ingestion), excessive usage can **slow down** the test suite runtime. Consider using `worker` scope when appropriate to reuse instances across tests within a worker. - **Add Unit Tests:** Include tests for new logic where applicable, ensuring it works as expected. - **Playwright documentation:** [Official best practices](https://playwright.dev/docs/best-practices) + + +### Running tests on CI +Scout is still in active development, which means frequent code changes may sometimes cause test failures. To maintain stability, we currently do not run Scout tests for every PR and encourage teams to limit the number of tests they add for now. + +If a test is difficult to stabilize within a reasonable timeframe, we reserve the right to disable it or even all tests for particular plugin. + +To manage Scout test execution, we use the `.buildkite/scout_ci_config.yml` file, where Kibana plugins with Scout tests are registered. If you're unsure about the stability of your tests, please add your plugin under the `disabled` section. + +You can check whether your plugin is already registered by running: +```bash +node scripts/scout discover-playwright-configs --validate +``` +On CI we run Scout tests only for `enabled` plugins: + +For PRs, Scout tests run only if there are changes to registered plugins or Scout-related packages. +On merge commits, Scout tests run in a non-blocking mode. diff --git a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts index f1c22e78db556..bf166d5c9bb2c 100644 --- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts +++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts @@ -13,6 +13,7 @@ import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info'; import path from 'path'; import { getScoutPlaywrightConfigs, DEFAULT_TEST_PATH_PATTERNS } from '../config'; import { measurePerformance } from '../common'; +import { validateWithScoutCiConfig } from '../config/discovery'; /** * Discover Playwright configuration files with Scout tests @@ -24,18 +25,19 @@ export const discoverPlaywrightConfigs: Command = { Common usage: node scripts/scout discover-playwright-configs --searchPaths - node scripts/scout discover-playwright-configs --save + node scripts/scout discover-playwright-configs --validate // validate if all plugins are registered in the Scout CI config + node scripts/scout discover-playwright-configs --save // save the discovered Playwright config files to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}' node scripts/scout discover-playwright-configs `, flags: { string: ['searchPaths'], - boolean: ['save'], - default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false }, + boolean: ['save', 'validate'], + default: { searchPaths: DEFAULT_TEST_PATH_PATTERNS, save: false, validate: false }, }, run: ({ flagsReader, log }) => { const searchPaths = flagsReader.arrayOfStrings('searchPaths')!; - const pluginsMap = measurePerformance(log, 'Discovering playwright config files', () => { + let pluginsMap = measurePerformance(log, 'Discovering Playwright config files', () => { return getScoutPlaywrightConfigs(searchPaths, log); }); @@ -44,6 +46,21 @@ export const discoverPlaywrightConfigs: Command = { ? 'No Playwright config files found' : `Found Playwright config files in '${pluginsMap.size}' plugins`; + if (!flagsReader.boolean('save')) { + log.info(finalMessage); + + pluginsMap.forEach((data, plugin) => { + log.info(`${data.group} / [${plugin}] plugin:`); + data.configs.map((file) => { + log.info(`- ${file}`); + }); + }); + } + + if (flagsReader.boolean('validate')) { + pluginsMap = validateWithScoutCiConfig(log, pluginsMap); + } + if (flagsReader.boolean('save')) { const dirPath = path.dirname(SCOUT_PLAYWRIGHT_CONFIGS_PATH); @@ -56,17 +73,9 @@ export const discoverPlaywrightConfigs: Command = { JSON.stringify(Object.fromEntries(pluginsMap), null, 2) ); - log.info(`${finalMessage}. Saved to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'`); - return; + log.info( + `${finalMessage}.\nSaved '${pluginsMap.size}' plugins to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}'` + ); } - - log.info(finalMessage); - - pluginsMap.forEach((data, plugin) => { - log.info(`${data.group} / [${plugin}] plugin:`); - data.configs.map((file) => { - log.info(`- ${file}`); - }); - }); }, }; diff --git a/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts b/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts index 3bcb0fe58f518..f1c89bf3adc35 100644 --- a/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts +++ b/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts @@ -7,11 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { ToolingLog } from '@kbn/tooling-log'; import fastGlob from 'fast-glob'; -import { getScoutPlaywrightConfigs } from './search_configs'; +import yaml from 'js-yaml'; +import { ToolingLog } from '@kbn/tooling-log'; +import { getScoutPlaywrightConfigs, validateWithScoutCiConfig } from './search_configs'; jest.mock('fast-glob'); +jest.mock('js-yaml'); describe('getScoutPlaywrightConfigs', () => { let mockLog: ToolingLog; @@ -80,3 +82,99 @@ describe('getScoutPlaywrightConfigs', () => { ); }); }); + +describe('validateWithScoutCiConfig', () => { + let mockLog: ToolingLog; + const mockScoutCiConfig = { + ui_tests: { + enabled: ['pluginA', 'pluginB'], + disabled: ['pluginC'], + }, + }; + + beforeEach(() => { + mockLog = new ToolingLog({ level: 'verbose', writeTo: process.stdout }); + jest.spyOn(mockLog, 'warning').mockImplementation(jest.fn()); + (yaml.safeLoad as jest.Mock).mockReturnValue(mockScoutCiConfig); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should return only enabled plugins', () => { + const pluginsWithConfigs = new Map([ + [ + 'pluginA', + { + group: 'group1', + pluginPath: 'pluginPathA', + usesParallelWorkers: true, + configs: ['configA'], + }, + ], + [ + 'pluginB', + { + group: 'group1', + pluginPath: 'pluginPathB', + usesParallelWorkers: false, + configs: ['configB1', 'configB2'], + }, + ], + [ + 'pluginC', + { + group: 'group2', + pluginPath: 'pluginPathC', + usesParallelWorkers: true, + configs: ['configC'], + }, + ], + ]); + + const result = validateWithScoutCiConfig(mockLog, pluginsWithConfigs); + expect(result.size).toBe(2); + expect(result.has('pluginA')).toBe(true); + expect(result.has('pluginB')).toBe(true); + expect(result.has('pluginC')).toBe(false); + }); + + it('should throw an error if plugins are not registered in Scout CI config', () => { + const pluginsWithConfigs = new Map([ + [ + 'pluginX', + { + group: 'groupX', + pluginPath: 'pluginPathX', + usesParallelWorkers: true, + configs: ['configX'], + }, + ], + ]); + + expect(() => validateWithScoutCiConfig(mockLog, pluginsWithConfigs)).toThrow( + "The following plugins are not registered in Scout CI config '.buildkite/scout_ci_config.yml':\n- pluginX" + ); + }); + + it('should log a warning for disabled plugins', () => { + const pluginsWithConfigs = new Map([ + [ + 'pluginC', + { + group: 'group2', + pluginPath: 'pluginPathC', + usesParallelWorkers: true, + configs: ['configC'], + }, + ], + ]); + + validateWithScoutCiConfig(mockLog, pluginsWithConfigs); + + expect(mockLog.warning).toHaveBeenCalledWith( + `The following plugins are disabled in '.buildkite/scout_ci_config.yml' and will be excluded from CI run\n- pluginC` + ); + }); +}); diff --git a/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts b/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts index f085998c43f47..cf5fbdfe0097f 100644 --- a/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts +++ b/src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts @@ -10,6 +10,10 @@ import fastGlob from 'fast-glob'; import path from 'path'; import { ToolingLog } from '@kbn/tooling-log'; +import { REPO_ROOT } from '@kbn/repo-info'; +import fs from 'fs'; +import yaml from 'js-yaml'; +import { createFailError } from '@kbn/dev-cli-errors'; export const DEFAULT_TEST_PATH_PATTERNS = ['src/platform/plugins', 'x-pack/**/plugins']; @@ -88,3 +92,56 @@ export const getScoutPlaywrightConfigs = (searchPaths: string[], log: ToolingLog return pluginsWithConfigs; }; + +export const validateWithScoutCiConfig = ( + log: ToolingLog, + pluginsWithConfigs: Map +) => { + const scoutCiConfigRelPath = path.join('.buildkite', 'scout_ci_config.yml'); + const scoutCiConfigPath = path.resolve(REPO_ROOT, scoutCiConfigRelPath); + const ciConfig = yaml.safeLoad(fs.readFileSync(scoutCiConfigPath, 'utf8')) as { + ui_tests: { + enabled?: string[]; + disabled?: string[]; + }; + }; + + const enabledPlugins = new Set(ciConfig.ui_tests.enabled || []); + const disabledPlugins = new Set(ciConfig.ui_tests.disabled || []); + const allRegisteredPlugins = new Set([...enabledPlugins, ...disabledPlugins]); + + const unregisteredPlugins: string[] = []; + const filteredPlugins = new Map(); + + for (const [pluginName, config] of pluginsWithConfigs.entries()) { + if (!allRegisteredPlugins.has(pluginName)) { + unregisteredPlugins.push(pluginName); + } else if (enabledPlugins.has(pluginName)) { + filteredPlugins.set(pluginName, config); + } + } + + if (unregisteredPlugins.length > 0) { + throw createFailError( + `The following plugins are not registered in Scout CI config '${scoutCiConfigRelPath}':\n${unregisteredPlugins + .map((plugin) => { + return `- ${plugin}`; + }) + .join('\n')}\nRead more: src/platform/packages/shared/kbn-scout/README.md` + ); + } + + if (disabledPlugins.size > 0) { + log.warning( + `The following plugins are disabled in '${scoutCiConfigRelPath}' and will be excluded from CI run\n${[ + ...disabledPlugins, + ] + .map((plugin) => { + return `- ${plugin}`; + }) + .join('\n')}` + ); + } + + return filteredPlugins; +};