From 4ca890dbc828e893db4486eadc2e435e72180fa0 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 26 Aug 2025 16:26:22 +0200 Subject: [PATCH 1/6] [scout] update docs with recent changes and examples --- .../packages/shared/kbn-scout/README.md | 181 +++++++++++++++--- 1 file changed, 150 insertions(+), 31 deletions(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index cf7a2d05db29c..0ada1799210f4 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -31,58 +31,73 @@ src/platform/packages/shared/kbn-scout/ │ ├── common/ │ │ ├── services/ │ │ ├── utils/ +│ │ └── constants.ts │ ├── config/ +│ │ ├── discovery/ │ │ ├── loader/ │ │ ├── schema/ -│ │ └── serverless/ -│ │ └── stateful/ -│ │ └── config.ts +│ │ ├── serverless/ +│ │ ├── stateful/ +│ │ ├── utils/ +│ │ ├── config.ts +│ │ └── constants.ts +│ ├── execution/ │ ├── playwright/ │ │ ├── config/ -│ │ └── fixtures -│ │ │ └── test/ -│ │ │ └── worker/ -│ │ └── page_objects/ -│ │ └── runner -│ │ │ └── config_validator.ts +│ │ ├── fixtures/ +│ │ │ └── scope/ +│ │ │ ├── test/ +│ │ │ └── worker/ +│ │ ├── global_hooks/ +│ │ ├── page_objects/ +│ │ ├── runner/ +│ │ │ ├── config_loader.ts +│ │ │ ├── config_validator.ts +│ │ │ ├── flags.ts │ │ │ └── run_tests.ts +│ │ ├── test/ +│ │ ├── types/ +│ │ ├── utils/ +│ │ ├── expect.ts +│ │ └── tags.ts │ ├── servers/ +│ │ ├── flags.ts │ │ ├── run_elasticsearch.ts -│ │ └── run_kibana_server.ts +│ │ ├── run_kibana_server.ts │ │ └── start_servers.ts -│ ├── types/ -│ └── index.ts -├── package.json -├── tsconfig.json +│ └── types/ +└── README.md ``` ### Key Components +The `kbn-scout` package has been updated with a new structure to better organize components by their scope and functionality. Here's an overview of the key components: + 1. **src/cli/** Contains the logic to start servers, with or without running tests. It is accessed through the `scripts/scout` script. 2. **src/common/** -`services` directory includes test helpers used across UI and API integration tests, such as Kibana and Elasticsearch `clients`, `esArchiver`, and `samlSessionManager`. These services are used to initialize instances and expose them to tests via Playwright worker fixtures. +The `services` directory includes test helpers used across UI and API integration tests, such as Kibana and Elasticsearch `clients`, `esArchiver`, and `samlSessionManager`. These services are used to initialize instances and expose them to tests via Playwright worker fixtures. The `utils` directory contains shared utilities, while `constants.ts` defines common constants used throughout the framework. 3. **src/config/** -`config` directory holds configurations for running servers locally. `serverless` and `stateful` directories contain deployment-specific configurations. Configuration attributes are defined in `schema` directory. -The `Config` class in config.ts serves as the main entry point. It is instantiated using the config loader in -the `loader` directory. This instance is compatible with the `kbn-test` input format and is passed to functions -for starting servers. +The `config` directory holds configurations for running servers locally. The `serverless` and `stateful` directories contain deployment-specific configurations. Configuration attributes are defined in the `schema` directory. The `discovery` directory contains logic for finding and validating test configurations, while `utils` provides configuration-related utilities. The `Config` class in `config.ts` serves as the main entry point. It is instantiated using the config loader in the `loader` directory. This instance is compatible with the `kbn-test` input format and is passed to functions for starting servers. + +4. **src/execution/** -4. **src/playwright/** +Contains CI execution-related logic to group tests into lanes that run efficiently within time constraints. + +5. **src/playwright/** #### Config -`playwright` directory manages the default Playwright configuration. It exports the `createPlaywrightConfig` function, which is used by Kibana plugins to define Scout playwright configurations and serves as the entry point to run tests. +The `playwright` directory manages the default Playwright configuration. It exports the `createPlaywrightConfig` function, which is used by Kibana plugins to define Scout playwright configurations and serves as the entry point to run tests. ```ts import { createPlaywrightConfig } from '@kbn/scout'; -// eslint-disable-next-line import/no-default-export export default createPlaywrightConfig({ testDir: './tests', workers: 2, @@ -91,13 +106,11 @@ export default createPlaywrightConfig({ Scout relies on configuration to determine the test files and opt-in [parallel test execution](https://playwright.dev/docs/test-parallel) against the single Elastic cluster. -The Playwright configuration should only be created this way to ensure compatibility with Scout functionality. For configuration -verification, we use a marker `VALID_CONFIG_MARKER`, and Scout will throw an error if the configuration is invalid. +The Playwright configuration should only be created this way to ensure compatibility with Scout functionality. For configuration verification, we use a marker `VALID_CONFIG_MARKER`, and Scout will throw an error if the configuration is invalid. #### Fixtures -The `fixtures` directory contains core Scout capabilities required for testing the majority of Kibana plugins. [Fixtures](https://playwright.dev/docs/test-fixtures) can be -scoped to either `test` or `worker`. Scope decides when to init a new fixture instance: once per worker or for every test function. It is important to choose the correct scope to keep test execution optimally fast: if **a new instance is not needed for every test**, the fixture should be scoped to **worker**. Otherwise, it should be scoped to **test**. +The `fixtures/scope` directory contains core Scout capabilities required for testing the majority of Kibana plugins. [Fixtures](https://playwright.dev/docs/test-fixtures) can be scoped to either `test` or `worker`. Scope decides when to init a new fixture instance: once per worker or for every test function. It is important to choose the correct scope to keep test execution optimally fast: if **a new instance is not needed for every test**, the fixture should be scoped to **worker**. Otherwise, it should be scoped to **test**. **Core `worker` scoped fixtures:** @@ -128,6 +141,10 @@ test.beforeEach(async ({ browserAuth }) => { If a new fixture depends on a fixture with a `test` scope, it must also be `test` scoped. +#### Global Hooks and Test Utilities + +The `global_hooks` directory contains setup and teardown logic that applies globally across test executions. It is a crutual feature for parallel tests, as it is required to ingest Elasticsearch data before any test runs. The `test` directory provides test-specific utilities, while `types` contains TypeScript type definitions. The `utils` directory includes various utility functions for test execution. + #### Page Objects The `page_objects` directory contains all the Page Objects that represent Platform core functionality such as Discover, Dashboard, Index Management, etc. @@ -154,10 +171,112 @@ test.beforeEach(async ({ pageObjects }) => { }); ``` -5. **src/servers/** +6. **src/servers/** + +Here we have logic to start Kibana and Elasticsearch servers using `kbn-test` functionality in Scout flavor. The instance of the `Config` class is passed to start servers for the specific deployment type. The `flags.ts` file contains server-related command-line flags and options. The `loadServersConfig` function not only returns a `kbn-test` compatible config instance, but also converts it to `ScoutServiceConfig` format and saves it on disk to `./scout/servers/local.json` in the Kibana root directory. Scout `config` fixture reads it and expose to UI tests. + +### Test Types and Directory Structure + +Scout supports two distinct types of tests: UI and API, each with their own directory structure and import patterns: + +#### Setting up Test Directory + +To get started with Scout testing for your plugin, you need to create the appropriate directory structure in your plugin's root directory: + +``` +your-plugin/ +├── test/ +│ └── scout/ +│ ├── ui/ +│ │ ├── playwright.config.ts +│ │ ├── parallel.playwright.config.ts +│ │ └── parallel_tests/ # Your UI test specs, that are run in parallel +│ │ └── tests/ # Your UI test specs, that are run sequentially +│ └── api/ +│ ├── playwright.config.ts +│ └── tests/ # Your API test specs, that are run sequentially +``` + +#### UI Tests + +UI tests are designed for browser-based integration testing and provide access to browser fixtures like `page`, `pageObjects`, and `browserAuth`. + +**Test Imports for UI Testing:** + +```ts +// For sequential UI tests +import { test, expect } from '@kbn/scout'; + +// For parallel UI tests that can be space-isolated +import { spaceTest as test, expect } from '@kbn/scout'; +``` + +**When to use each:** + +- **`spaceTest`**: Use for parallel tests that can be isolated by Kibana spaces, allowing faster execution +- **`test`**: Use for sequential tests that cannot run in parallel + +**Example UI Test:** + +```ts +import { spaceTest as test, expect } from '@kbn/scout'; + +test('should display dashboard', async ({ pageObjects, page }) => { + await pageObjects.dashboard.goto(); + await expect(page.testSubj.locator('dashboardLandingPage')).toBeVisible(); +}); +``` + +#### API Tests + +API tests are designed for server-side testing and provide fixtures focused on API interactions without browser-related fixtures. + +**Test Import for API Testing:** + +```ts +// For API integration tests (server-side only, no browser fixtures) +import { apiTest as test, expect } from '@kbn/scout'; +``` + +**Example API Test:** + +```ts +import { apiTest, expect } from '@kbn/scout'; + +apiTest('POST api/painless_lab/execute is disabled', async ({ kbnClient, log }) => { + const response = await apiClient.post('api/painless_lab/execute', { + headers: { + ...COMMON_HEADERS, + ...adminApiCredentials.apiKeyHeader, + }, + responseType: 'json', + body: TEST_INPUT.script, + }); + expect(response.statusCode).toBe(404); +}); +``` + +**Key Differences:** + +- **UI tests** include browser fixtures (`page`, `pageObjects`, `browserAuth`) for UI interactions +- **API tests** exclude browser fixtures and focus on server-side operations (`kbnClient`, `esClient`, `log`, etc.) + +#### Testing Guidelines for Plugin Development + +When writing tests for your plugin, consider the following guidelines to ensure comprehensive coverage: + +**Focus on Plugin Functionality:** + +- Tests should primarily cover the specific functionality exposed by your plugin +- Focus on plugin-specific user workflows, configurations, and integrations + +**API Testing Coverage:** -Here we have logic to start Kibana and Elasticsearch servers using `kbn-test` functionality in Scout flavor. -The instance of the `Config` class is passed to start servers for the specific deployment type. The `loadServersConfig` function not only returns a `kbn-test` compatible config instance, but also converts it to `ScoutServiceConfig` format and saves it on disk to `./scout/servers/local.json` in the Kibana root directory. Scout `config` fixture reads it and expose to UI tests. +- Test all API endpoints exposed by your plugin +- Verify endpoints work correctly in both **serverless** and **stateful** deployments +- Include tests for scenarios where functionality should be **disabled** or **restricted** +- Test different user roles and permissions for your endpoints +- Cover both success and error scenarios (validation, authentication, authorization failures) ### How to Use @@ -334,7 +453,7 @@ export function createCorePageObjects(page: ScoutPage): PageObjects { #### Adding API service -1. **Create a New API service:** Add your service to the `src/playwright/fixtures/worker/apis` directory. For instance: +1. **Create a New API service:** Add your service to the `src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/apis` directory, organized by functionality (e.g., `/fleet` or `/alerting`). For instance: ```ts export interface FleetApiService { @@ -380,7 +499,7 @@ export const apiServicesFixture = coreWorkerFixtures.extend< 1. **Determine Fixture Scope:** Decide if your fixture should apply to the `test` (per-test) or `worker` (per-worker) scope. -2. **Implement the Fixture:** Add the implementation to `src/playwright/fixtures/test` or `src/playwright/fixtures/worker`. +2. **Implement the Fixture:** Add the implementation to `src/playwright/fixtures/scope/test` or `src/playwright/fixtures/scope/worker`. ```ts export const newTestFixture = base.extend({ From 3c35b82d9bdb25281d0d4089a2add72a18a8d5a4 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 27 Aug 2025 10:00:24 +0200 Subject: [PATCH 2/6] Update src/platform/packages/shared/kbn-scout/README.md Co-authored-by: Cesare de Cal --- src/platform/packages/shared/kbn-scout/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 0ada1799210f4..d64a9b434d66c 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -143,7 +143,7 @@ If a new fixture depends on a fixture with a `test` scope, it must also be `test #### Global Hooks and Test Utilities -The `global_hooks` directory contains setup and teardown logic that applies globally across test executions. It is a crutual feature for parallel tests, as it is required to ingest Elasticsearch data before any test runs. The `test` directory provides test-specific utilities, while `types` contains TypeScript type definitions. The `utils` directory includes various utility functions for test execution. +The `global_hooks` directory contains setup and teardown logic that applies globally across test executions. It is a crucial feature for parallel tests, as it is required to ingest Elasticsearch data before any test runs. The `test` directory provides test-specific utilities, while `types` contains TypeScript type definitions. The `utils` directory includes various utility functions for test execution. #### Page Objects From bab09807af8285d7484a60bcaf3e6c39669cd969 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 27 Aug 2025 10:00:53 +0200 Subject: [PATCH 3/6] Update src/platform/packages/shared/kbn-scout/README.md Co-authored-by: Cesare de Cal --- src/platform/packages/shared/kbn-scout/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index d64a9b434d66c..75be8579a4f66 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -173,7 +173,7 @@ test.beforeEach(async ({ pageObjects }) => { 6. **src/servers/** -Here we have logic to start Kibana and Elasticsearch servers using `kbn-test` functionality in Scout flavor. The instance of the `Config` class is passed to start servers for the specific deployment type. The `flags.ts` file contains server-related command-line flags and options. The `loadServersConfig` function not only returns a `kbn-test` compatible config instance, but also converts it to `ScoutServiceConfig` format and saves it on disk to `./scout/servers/local.json` in the Kibana root directory. Scout `config` fixture reads it and expose to UI tests. +Here we have logic to start Kibana and Elasticsearch servers using `kbn-test` functionality in Scout flavor. The instance of the `Config` class is passed to start servers for the specific deployment type. The `flags.ts` file contains server-related command-line flags and options. The `loadServersConfig` function not only returns a `kbn-test` compatible config instance, but also converts it to `ScoutServiceConfig` format and saves it on disk to `./scout/servers/local.json` in the Kibana root directory. Scout `config` fixture reads it and exposes it to UI tests. ### Test Types and Directory Structure From efef5da3159a6717039ecca6a9897f543d93c183 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 27 Aug 2025 10:01:26 +0200 Subject: [PATCH 4/6] Update src/platform/packages/shared/kbn-scout/README.md Co-authored-by: Cesare de Cal --- src/platform/packages/shared/kbn-scout/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 75be8579a4f66..54b84f4db6fb6 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -243,7 +243,7 @@ import { apiTest as test, expect } from '@kbn/scout'; ```ts import { apiTest, expect } from '@kbn/scout'; -apiTest('POST api/painless_lab/execute is disabled', async ({ kbnClient, log }) => { +apiTest('POST api/painless_lab/execute is disabled', async ({ apiClient, log }) => { const response = await apiClient.post('api/painless_lab/execute', { headers: { ...COMMON_HEADERS, From 672ab6dca2dd21ef3daef2f21145387f048dd5c5 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 27 Aug 2025 10:04:58 +0200 Subject: [PATCH 5/6] add /common to test dir structure --- src/platform/packages/shared/kbn-scout/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 54b84f4db6fb6..6b521f67f9203 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -192,9 +192,12 @@ your-plugin/ │ │ ├── parallel.playwright.config.ts │ │ └── parallel_tests/ # Your UI test specs, that are run in parallel │ │ └── tests/ # Your UI test specs, that are run sequentially -│ └── api/ -│ ├── playwright.config.ts -│ └── tests/ # Your API test specs, that are run sequentially +│ ├── api/ +│ │ ├── playwright.config.ts +│ │ └── tests/ # Your API test specs, that are run sequentially +│ └── common/ # For shared code across UI and API tests +│ ├── constants.ts +│ └── fixtures/ ``` #### UI Tests From b66d3fb26b12a1866df9a787b19fdb61ab85d6a3 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 27 Aug 2025 10:44:28 +0200 Subject: [PATCH 6/6] fix typos --- src/platform/packages/shared/kbn-scout/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/packages/shared/kbn-scout/README.md b/src/platform/packages/shared/kbn-scout/README.md index 6b521f67f9203..c0719078471ca 100644 --- a/src/platform/packages/shared/kbn-scout/README.md +++ b/src/platform/packages/shared/kbn-scout/README.md @@ -430,7 +430,7 @@ Move to the `src/platform/packages/shared/kbn-scout` directory to begin developm ### Adding or Modifying Components -Contributions to sharable `Fixtures`, `API services` and `Page Objects` are highly encouraged to promote reusability, stability, and ease of adoption. Follow these steps: +Contributions to shareable `Fixtures`, `API services` and `Page Objects` are highly encouraged to promote reusability, stability, and ease of adoption. Follow these steps: #### Adding Page Objects @@ -524,7 +524,7 @@ export const scoutTestFixtures = mergeTests(coreFixtures, newTestFixture); - **Reusable Code:** When creating Page Objects, API services or Fixtures that apply to more than one plugin, ensure they are added to the `kbn-scout` package. - **Adhere to Existing Structure:** Maintain consistency with the project's architecture. -- **Keep the Scope of Components Clear** When designing test components, keep in naming conventions, scope, maintainability and performance. +- **Keep the Scope of Components Clear** When designing test components, keep in mind naming conventions, scope, maintainability and performance. - `Page Objects` should focus exclusively on UI interactions (clicking buttons, filling forms, navigating page). They should not make API calls directly. - `API Services` should handle server interactions, such as sending API requests and processing responses. - `Fixtures` can combine browser interactions with API requests, but they should be used wisely, especially with the `test` scope: a new instance of the fixture is created for **every test block**. If a fixture performs expensive operations (API setup, data ingestion), excessive usage can **slow down** the test suite runtime. Consider using `worker` scope when appropriate to reuse instances across tests within a worker.