diff --git a/.buildkite/scout_ci_config.yml b/.buildkite/scout_ci_config.yml index d47ee7afd2ab2..bca177b012064 100644 --- a/.buildkite/scout_ci_config.yml +++ b/.buildkite/scout_ci_config.yml @@ -24,3 +24,7 @@ packages: # so they don't rerun alongside plugin/package Scout tests discovered later. - kbn-scout - kbn-scout-release-testing # Release tests will run separately as part of the release process + +# Define test configs to be excluded from automatic discovery & execution in CI environment (process.env.CI=true) +excluded_configs: + - x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts diff --git a/.buildkite/scripts/steps/test/scout/test_run_builder.sh b/.buildkite/scripts/steps/test/scout/test_run_builder.sh index 3be04062647ef..c790b32599de7 100755 --- a/.buildkite/scripts/steps/test/scout/test_run_builder.sh +++ b/.buildkite/scripts/steps/test/scout/test_run_builder.sh @@ -10,8 +10,8 @@ node scripts/scout run-playwright-test-check echo '--- Update Scout Test Config Manifests' node scripts/scout.js update-test-config-manifests -echo '--- Discover Playwright Configs and upload to Buildkite artifacts (only stateful tests)' -node scripts/scout discover-playwright-configs --target ech --save +echo '--- Discover Playwright Configs and upload to Buildkite artifacts' +node scripts/scout discover-playwright-configs --target ech --include-custom-servers --save cp .scout/test_configs/scout_playwright_configs.json scout_playwright_configs.json buildkite-agent artifact upload "scout_playwright_configs.json" diff --git a/src/platform/packages/private/kbn-scout-info/src/paths.ts b/src/platform/packages/private/kbn-scout-info/src/paths.ts index c0abceff60c9a..67c29135f2148 100644 --- a/src/platform/packages/private/kbn-scout-info/src/paths.ts +++ b/src/platform/packages/private/kbn-scout-info/src/paths.ts @@ -7,8 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import path from 'node:path'; import { REPO_ROOT } from '@kbn/repo-info'; +import path from 'node:path'; export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout'); @@ -33,7 +33,7 @@ export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve( ); export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB = - '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout'; + '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout{_*,}'; export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX = new RegExp( `(?:src|x-pack)` + diff --git a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts index f6f2ac3443d0d..a86ab209a491c 100644 --- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts +++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.test.ts @@ -11,8 +11,11 @@ import type { FlagsReader } from '@kbn/dev-cli-runner'; import type { ScoutTestableModuleWithConfigs } from '@kbn/scout-reporting/src/registry'; import type { ToolingLog } from '@kbn/tooling-log'; import fs from 'fs'; -import { filterModulesByScoutCiConfig } from '../tests_discovery/search_configs'; import { getServerRunFlagsFromTags } from '../tests_discovery/tag_utils'; +import { + filterModulesByScoutCiConfig, + getScoutCiExcludedConfigs, +} from '../tests_discovery/search_configs'; import type { ModuleDiscoveryInfo } from './config_discovery'; import { runDiscoverPlaywrightConfigs } from './config_discovery'; @@ -51,6 +54,7 @@ jest.mock('@kbn/scout-info', () => ({ jest.mock('../tests_discovery/search_configs', () => ({ filterModulesByScoutCiConfig: jest.fn(), + getScoutCiExcludedConfigs: jest.fn(), })); jest.mock('@kbn/scout-reporting/src/registry', () => { @@ -132,6 +136,7 @@ describe('runDiscoverPlaywrightConfigs', () => { flagsReader.arrayOfStrings.mockReturnValue([]); (filterModulesByScoutCiConfig as jest.Mock).mockReturnValue(mockFilteredModules); + (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([]); // Default mock modules mockTestableModules.modules = [ @@ -322,6 +327,145 @@ describe('runDiscoverPlaywrightConfigs', () => { expect(foundMessage![0]).toContain('1 package(s)'); // packageA }); + it('includes custom-server configs alongside defaults when "include-custom-servers" is true', () => { + flagsReader.enum.mockReturnValue('all'); + flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers'); + + mockTestableModules.modules = [ + { + name: 'pluginCustom', + group: 'groupCustom', + type: 'plugin' as const, + visibility: 'private' as const, + root: 'x-pack/platform/plugins/private/pluginCustom', + configs: [ + { + path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts', + category: 'ui', + type: 'playwright', + manifest: { + path: 'x-pack/platform/plugins/private/pluginCustom/test/scout_custom/config.playwright.config.ts', + exists: true, + sha1: 'custom123', + tests: [ + { + id: 'customTest1', + title: 'Custom Test 1', + expectedStatus: 'passed', + location: { file: 'custom.spec.ts', line: 1, column: 1 }, + tags: ['@local-stateful-classic'], + }, + ], + }, + }, + { + path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts', + category: 'ui', + type: 'playwright', + manifest: { + path: 'x-pack/platform/plugins/private/pluginCustom/config.playwright.config.ts', + exists: true, + sha1: 'normal456', + tests: [ + { + id: 'normalTest1', + title: 'Normal Test 1', + expectedStatus: 'passed', + location: { file: 'normal.spec.ts', line: 1, column: 1 }, + tags: ['@local-stateful-classic'], + }, + ], + }, + }, + ], + }, + ]; + + runDiscoverPlaywrightConfigs(flagsReader, log); + + const infoCalls = log.info.mock.calls; + const configLogs = infoCalls.filter((call) => call[0].startsWith('- ')); + expect(configLogs.length).toBeGreaterThan(0); + expect(configLogs.some((call) => call[0].includes('/test/scout_custom/'))).toBe(true); + expect( + configLogs.some((call) => call[0].includes('/pluginCustom/config.playwright.config.ts')) + ).toBe(true); + }); + + it('excludes configs listed in scout_ci_config.yml when running in CI', () => { + const originalCi = process.env.CI; + process.env.CI = 'true'; + + flagsReader.enum.mockReturnValue('all'); + flagsReader.boolean.mockImplementation((flag) => flag === 'include-custom-servers'); + + const excludedConfigPath = + 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout_cspm_agentless/ui/parallel.playwright.config.ts'; + + (getScoutCiExcludedConfigs as jest.Mock).mockReturnValue([excludedConfigPath]); + + mockTestableModules.modules = [ + { + name: 'pluginCspm', + group: 'security', + type: 'plugin' as const, + visibility: 'private' as const, + root: 'x-pack/solutions/security/plugins/cloud_security_posture', + configs: [ + { + path: excludedConfigPath, + category: 'ui', + type: 'playwright', + manifest: { + path: excludedConfigPath, + exists: true, + sha1: 'exclude123', + tests: [ + { + id: 'excludedTest', + title: 'Excluded Test', + expectedStatus: 'passed', + location: { file: 'excluded.spec.ts', line: 1, column: 1 }, + tags: ['@local-stateful-classic'], + }, + ], + }, + }, + { + path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts', + category: 'ui', + type: 'playwright', + manifest: { + path: 'x-pack/solutions/security/plugins/cloud_security_posture/test/scout/ui/config.playwright.config.ts', + exists: true, + sha1: 'include456', + tests: [ + { + id: 'includedTest', + title: 'Included Test', + expectedStatus: 'passed', + location: { file: 'included.spec.ts', line: 1, column: 1 }, + tags: ['@local-stateful-classic'], + }, + ], + }, + }, + ], + }, + ]; + + runDiscoverPlaywrightConfigs(flagsReader, log); + + const infoCalls = log.info.mock.calls; + const configLogs = infoCalls.filter((call) => call[0].startsWith('- ')); + expect(configLogs.some((call) => call[0].includes(excludedConfigPath))).toBe(false); + expect( + configLogs.some((call) => call[0].includes('/test/scout/ui/config.playwright.config.ts')) + ).toBe(true); + + process.env.CI = originalCi; + }); + it('filters config tags to only include cross tags', () => { flagsReader.enum.mockReturnValue('ech'); // ESS_ONLY = ['@local-stateful-classic'] flagsReader.boolean.mockReturnValue(false); diff --git a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts index 37f4f52ffe1d0..54c34bc33c56f 100644 --- a/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts +++ b/src/platform/packages/shared/kbn-scout/src/cli/config_discovery.ts @@ -12,7 +12,10 @@ import { SCOUT_PLAYWRIGHT_CONFIGS_PATH } from '@kbn/scout-info'; import { testableModules } from '@kbn/scout-reporting/src/registry'; import type { ToolingLog } from '@kbn/tooling-log'; import { saveFlattenedConfigGroups, saveModuleDiscoveryInfo } from '../tests_discovery/file_utils'; -import { filterModulesByScoutCiConfig } from '../tests_discovery/search_configs'; +import { + filterModulesByScoutCiConfig, + getScoutCiExcludedConfigs, +} from '../tests_discovery/search_configs'; import { collectUniqueTags, getServerRunFlagsFromTags, @@ -82,6 +85,45 @@ const filterModulesByTargetTags = ( .filter((module) => module.configs.length > 0); }; +const CUSTOM_SERVERS_PATH_PATTERN = /\/test\/scout_[^/]+/; + +const filterModulesByCustomServerPaths = ( + modules: ModuleDiscoveryInfo[], + includeCustomServers: boolean +): ModuleDiscoveryInfo[] => { + if (includeCustomServers) { + return modules; + } + + return modules + .map((module) => ({ + ...module, + configs: module.configs.filter((config) => { + const isCustomServerConfig = CUSTOM_SERVERS_PATH_PATTERN.test(config.path); + return !isCustomServerConfig; + }), + })) + .filter((module) => module.configs.length > 0); +}; + +const filterModulesByExcludedConfigPaths = ( + modules: ModuleDiscoveryInfo[], + excludedConfigPaths: string[] +): ModuleDiscoveryInfo[] => { + if (excludedConfigPaths.length === 0) { + return modules; + } + + const excludedSet = new Set(excludedConfigPaths); + + return modules + .map((module) => ({ + ...module, + configs: module.configs.filter((config) => !excludedSet.has(config.path)), + })) + .filter((module) => module.configs.length > 0); +}; + // Logs discovered modules in non-flattened format const logDiscoveredModules = (modules: ModuleDiscoveryInfo[], log: ToolingLog): void => { const { plugins: pluginCount, packages: packageCount } = countModulesByType(modules); @@ -163,16 +205,24 @@ export const runDiscoverPlaywrightConfigs = (flagsReader: FlagsReader, log: Tool const target = (flagsReader.enum('target', TARGET_TYPES) || 'all') as TargetType; const targetTags = getTestTagsForTarget(target); const flatten = flagsReader.boolean('flatten'); + const includeCustomServers = flagsReader.boolean('include-custom-servers'); // Build initial module discovery info const modulesWithTests = buildModuleDiscoveryInfo(); // Filter modules by target tags and compute server run flags - const filteredModules = filterModulesByTargetTags(modulesWithTests, targetTags); + const filteredModulesByTags = filterModulesByTargetTags(modulesWithTests, targetTags); + const filteredModules = filterModulesByCustomServerPaths( + filteredModulesByTags, + includeCustomServers + ); + const filteredModulesWithExcludedConfigs = process.env.CI + ? filterModulesByExcludedConfigPaths(filteredModules, getScoutCiExcludedConfigs()) + : filteredModules; // Handle output based on flatten flag if (flatten) { - handleFlattenedOutput(filteredModules, flagsReader, log); + handleFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log); } else { - handleNonFlattenedOutput(filteredModules, flagsReader, log); + handleNonFlattenedOutput(filteredModulesWithExcludedConfigs, flagsReader, log); } }; @@ -202,14 +252,15 @@ export const discoverPlaywrightConfigsCmd: Command = { validate against CI configuration, or save filtered results to a file. Options: - --target Filter configs by deployment target: - - 'all': deployment-agnostic tags (default) - - 'mki': serverless-only tags - - 'ech': stateful-only tags - --validate Validate that all discovered modules are registered in Scout CI config - --save Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}' - --flatten Output configs in flattened format grouped by mode, group, and scout command - (useful for Cloud test execution) + --target Filter configs by deployment target: + - 'all': deployment-agnostic tags (default) + - 'mki': serverless-only tags + - 'ech': stateful-only tags + --include-custom-servers Include configs under 'test/scout_*' paths for custom server setups + --validate Validate that all discovered modules are registered in Scout CI config + --save Validate and save enabled modules to '${SCOUT_PLAYWRIGHT_CONFIGS_PATH}' + --flatten Output configs in flattened format grouped by mode, group, and scout command + (useful for Cloud test execution) Examples: # Discover all deployment-agnostic configs @@ -218,6 +269,9 @@ export const discoverPlaywrightConfigsCmd: Command = { # Discover serverless-only configs node scripts/scout discover-playwright-configs --target mki + # Discover local custom-server configs only + node scripts/scout discover-playwright-configs --include-custom-servers + # Validate discovered configs against CI configuration node scripts/scout discover-playwright-configs --validate @@ -229,8 +283,14 @@ export const discoverPlaywrightConfigsCmd: Command = { `, flags: { string: ['target'], - boolean: ['save', 'validate', 'flatten'], - default: { target: 'all', save: false, validate: false, flatten: false }, + boolean: ['save', 'validate', 'flatten', 'include-custom-servers'], + default: { + target: 'all', + save: false, + validate: false, + flatten: false, + 'include-custom-servers': false, + }, }, run: ({ flagsReader, log }) => { runDiscoverPlaywrightConfigs(flagsReader, log); diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/observability_complete.serverless.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/observability_complete.serverless.config.ts deleted file mode 100644 index 9fb3f2e2c128c..0000000000000 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/observability_complete.serverless.config.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { servers as defaultConfig } from '../../default/serverless/observability_complete.serverless.config'; -import type { ScoutServerConfig } from '../../../../../types'; - -/** - * Custom Scout server configuration for OAS (OpenAPI Specification) schema validation tests. - * Enables the OAS endpoint which is required for schema validation. - * - * This config is automatically used when running tests from: - * dashboard/test/scout_oas_schema/ - * - * Usage: - * node scripts/scout.js start-server --arch serverless --domain observability_complete --serverConfigSet oas_schema - */ -export const servers: ScoutServerConfig = { - ...defaultConfig, - kbnTestServer: { - ...defaultConfig.kbnTestServer, - serverArgs: [ - ...defaultConfig.kbnTestServer.serverArgs, - // Enable OpenAPI specification endpoint for schema validation tests - '--server.oas.enabled=true', - ], - }, -}; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/search.serverless.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/search.serverless.config.ts deleted file mode 100644 index 25839a30c34c1..0000000000000 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/search.serverless.config.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { servers as defaultConfig } from '../../default/serverless/search.serverless.config'; -import type { ScoutServerConfig } from '../../../../../types'; - -/** - * Custom Scout server configuration for OAS (OpenAPI Specification) schema validation tests. - * Enables the OAS endpoint which is required for schema validation. - * - * This config is automatically used when running tests from: - * dashboard/test/scout_oas_schema/ - * - * Usage: - * node scripts/scout.js start-server --arch serverless --domain search --serverConfigSet oas_schema - */ -export const servers: ScoutServerConfig = { - ...defaultConfig, - kbnTestServer: { - ...defaultConfig.kbnTestServer, - serverArgs: [ - ...defaultConfig.kbnTestServer.serverArgs, - // Enable OpenAPI specification endpoint for schema validation tests - '--server.oas.enabled=true', - ], - }, -}; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/security_complete.serverless.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/security_complete.serverless.config.ts deleted file mode 100644 index 3478dda62498e..0000000000000 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/serverless/security_complete.serverless.config.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { servers as defaultConfig } from '../../default/serverless/security_complete.serverless.config'; -import type { ScoutServerConfig } from '../../../../../types'; - -/** - * Custom Scout server configuration for OAS (OpenAPI Specification) schema validation tests. - * Enables the OAS endpoint which is required for schema validation. - * - * This config is automatically used when running tests from: - * dashboard/test/scout_oas_schema/ - * - * Usage: - * node scripts/scout.js start-server --arch serverless --domain security_complete --serverConfigSet oas_schema - */ -export const servers: ScoutServerConfig = { - ...defaultConfig, - kbnTestServer: { - ...defaultConfig.kbnTestServer, - serverArgs: [ - ...defaultConfig.kbnTestServer.serverArgs, - // Enable OpenAPI specification endpoint for schema validation tests - '--server.oas.enabled=true', - ], - }, -}; diff --git a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/stateful/classic.stateful.config.ts b/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/stateful/classic.stateful.config.ts deleted file mode 100644 index bbedefe0c50ba..0000000000000 --- a/src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/oas_schema/stateful/classic.stateful.config.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import type { ScoutServerConfig } from '../../../../../types'; -import { defaultConfig } from '../../default/stateful/base.config'; - -/** - * Custom Scout server configuration for OAS (OpenAPI Specification) schema validation tests. - * Enables the OAS endpoint which is required for schema validation. - * - * This config is automatically used when running tests from: - * dashboard/test/scout_oas_schema/ - * - * Usage: - * node scripts/scout.js start-server --arch stateful --domain classic --serverConfigSet oas_schema - */ -export const servers: ScoutServerConfig = { - ...defaultConfig, - kbnTestServer: { - ...defaultConfig.kbnTestServer, - serverArgs: [ - ...defaultConfig.kbnTestServer.serverArgs, - // Enable OpenAPI specification endpoint for schema validation tests - '--server.oas.enabled=true', - ], - }, -}; diff --git a/src/platform/packages/shared/kbn-scout/src/tests_discovery/search_configs.ts b/src/platform/packages/shared/kbn-scout/src/tests_discovery/search_configs.ts index 36b9d6c0ee5d0..b17c0320d4b83 100644 --- a/src/platform/packages/shared/kbn-scout/src/tests_discovery/search_configs.ts +++ b/src/platform/packages/shared/kbn-scout/src/tests_discovery/search_configs.ts @@ -15,6 +15,29 @@ import yaml from 'js-yaml'; import path from 'path'; import type { ModuleDiscoveryInfo } from './types'; +interface ScoutCiConfig { + plugins: { + enabled?: string[]; + disabled?: string[]; + }; + packages: { + enabled?: string[]; + disabled?: string[]; + }; + excluded_configs?: string[]; +} + +const readScoutCiConfig = (): ScoutCiConfig => { + const scoutCiConfigRelPath = path.join('.buildkite', 'scout_ci_config.yml'); + const scoutCiConfigPath = path.resolve(REPO_ROOT, scoutCiConfigRelPath); + return yaml.load(fs.readFileSync(scoutCiConfigPath, 'utf8')) as ScoutCiConfig; +}; + +export const getScoutCiExcludedConfigs = (): string[] => { + const ciConfig = readScoutCiConfig(); + return ciConfig.excluded_configs ?? []; +}; + /** * Filters modules based on Scout CI configuration. * Validates that all modules are registered in the CI config ('scout_ci_config.yml') and @@ -30,17 +53,7 @@ export const filterModulesByScoutCiConfig = ( modulesWithTests: ModuleDiscoveryInfo[] ): ModuleDiscoveryInfo[] => { const scoutCiConfigRelPath = path.join('.buildkite', 'scout_ci_config.yml'); - const scoutCiConfigPath = path.resolve(REPO_ROOT, scoutCiConfigRelPath); - const ciConfig = yaml.load(fs.readFileSync(scoutCiConfigPath, 'utf8')) as { - plugins: { - enabled?: string[]; - disabled?: string[]; - }; - packages: { - enabled?: string[]; - disabled?: string[]; - }; - }; + const ciConfig = readScoutCiConfig(); const enabledPlugins = new Set(ciConfig.plugins.enabled || []); const disabledPlugins = new Set(ciConfig.plugins.disabled || []); diff --git a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/README.md b/src/platform/plugins/shared/dashboard/test/scout_oas_schema/README.md deleted file mode 100644 index 5ae218e43063d..0000000000000 --- a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## Dashboard OAS Schema Validation Tests - -This directory contains Scout tests for validating the Dashboard REST API OpenAPI (OAS) schema. - -These tests use a **custom server configuration** that enables the OAS endpoint (`--server.oas.enabled=true`). - -### How to run tests - -First start the servers with the custom config: - -```bash -// ESS (stateful) -node scripts/scout.js start-server --arch stateful --domain classic --serverConfigSet oas_schema - -// Serverless -node scripts/scout.js start-server --arch serverless --domain search --serverConfigSet oas_schema -``` - -Then run the tests in another terminal: - -```bash -// ESS (stateful) -npx playwright test --config=src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/playwright.config.ts --grep=@local-stateful-classic --project=local - -// Serverless -npx playwright test --config=src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/playwright.config.ts --grep=@local-serverless-search --project=local -``` - -### Server Configuration - -The custom server config is located at: -`src/platform/packages/shared/kbn-scout/src/servers/configs/custom/oas_schema/` - -It extends the default config and adds: - -``` ---server.oas.enabled=true -``` diff --git a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/index.ts b/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/index.ts deleted file mode 100644 index 2744113c17a85..0000000000000 --- a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import type { ScoutTestFixtures, ScoutWorkerFixtures } from '@kbn/scout'; -import { apiTest as baseApiTest } from '@kbn/scout'; - -export const apiTest = baseApiTest.extend({}); - -/** The base API path for dashboard endpoints (no leading slash for apiClient). */ -export const DASHBOARD_API_PATH = 'api/dashboards'; diff --git a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/schema_snapshot.json b/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/schema_snapshot.json deleted file mode 100644 index 3a669f02e3f0d..0000000000000 --- a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/fixtures/schema_snapshot.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "type": "object", - "description": "Markdown embeddable schema", - "properties": { - "content": { - "type": "string" - }, - "description": { - "type": "string" - }, - "hide_title": { - "type": "boolean" - }, - "title": { - "type": "string" - } - }, - "additionalProperties": false, - "required": ["content"] - }, - { - "type": "object", - "properties": {}, - "additionalProperties": true - } -] diff --git a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/playwright.config.ts b/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/playwright.config.ts deleted file mode 100644 index 5175ba4c743f0..0000000000000 --- a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/playwright.config.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { createPlaywrightConfig } from '@kbn/scout'; - -/** - * Playwright config for OAS schema validation tests. - * - * These tests use a custom server configuration that enables the OAS endpoint. - * The custom config is located at: - * src/platform/packages/shared/kbn-scout/src/servers/configs/custom/oas_schema/ - * - * See README.md for usage instructions. - */ -export default createPlaywrightConfig({ - testDir: './tests', -}); diff --git a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/tests/schema.spec.ts b/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/tests/schema.spec.ts deleted file mode 100644 index d9ab734a4e8c3..0000000000000 --- a/src/platform/plugins/shared/dashboard/test/scout_oas_schema/api/tests/schema.spec.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import type { RoleApiCredentials } from '@kbn/scout'; -import { expect } from '@kbn/scout/api'; -import { tags } from '@kbn/scout'; -import { apiTest, DASHBOARD_API_PATH } from '../fixtures'; -import snapshot from '../fixtures/schema_snapshot.json'; - -/** - * Dashboard REST schema validation tests. - * - * These tests require the OAS (OpenAPI Specification) endpoint to be enabled, - * which is configured in the custom server config at: - * src/platform/packages/shared/kbn-scout/src/servers/configs/custom/oas_schema/ - * - * See README.md for usage instructions. - */ -apiTest.describe('dashboard REST schema', { tag: tags.deploymentAgnostic }, () => { - let viewerCredentials: RoleApiCredentials; - - apiTest.beforeAll(async ({ requestAuth }) => { - viewerCredentials = await requestAuth.getApiKey('viewer'); - }); - - /** - * Only additive changes are allowed to Dashboard REST schemas - * - * This test exists to ping #kibana-presentation of any embeddable schema changes - * since the dashboard schema includes embeddable schemas. - * - * If this test is failing, the changed embeddable schema needs to be be reviewed - * to ensure its ready for public distribution. - * - * Once an embeddable schema has been published, - * it can only be changed with additive changes. - */ - apiTest('Registered embeddable schemas have not changed', async ({ apiClient }) => { - // OAS paths are stored with leading slashes, so we need to use the full path here - const oasPath = `/${DASHBOARD_API_PATH}`; - const response = await apiClient.get( - `api/oas?pathStartsWith=${oasPath}&access=internal&version=1`, - { - headers: { - ...viewerCredentials.apiKeyHeader, - }, - responseType: 'json', - } - ); - - expect(response.statusCode).toBe(200); - expect(response.body.paths?.[oasPath]).toBeDefined(); - - const createBodySchema = - response.body.paths[oasPath].post.requestBody.content[ - 'application/json; Elastic-Api-Version=1' - ].schema; - const panelsSchema = createBodySchema.properties.data.properties.panels; - expect(panelsSchema).toBeDefined(); - expect(panelsSchema.items?.anyOf?.length).toBeGreaterThan(0); - - const panelSchema = panelsSchema.items.anyOf.find( - (schema: { properties: Record }) => 'config' in schema.properties - ); - expect(panelSchema).toBeDefined(); - - const configSchema = panelSchema.properties.config; - expect(configSchema.anyOf).toHaveLength(2); - expect(configSchema.anyOf).toStrictEqual(snapshot); - }); -});