From 4de2d557cc5602503f90a8ca6470c84cd7913e73 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 21 Oct 2025 13:32:11 +0200 Subject: [PATCH] [Security Solution] Make prebuilt rules bootstrap errors visible (#239521) **Partially addresses:** https://github.com/elastic/kibana/issues/188090 ## Summary Prebuilt Rules bootstrap API endpoint may fail on a fresh deployment in Elastic Cloud. This PR rearranges assertions to provide an ability to view the error message. (cherry picked from commit 9b8ef8299d59fd31e29c91797ad4237eeba78604) # Conflicts: # x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/install_prebuilt_rules.ts --- .../prebuilt_rules/oom_testing/README.md | 124 ++++++++++++++++++ .../configs/ess_basic_license.config.ts | 24 ++++ .../prebuilt_rules/oom_testing/index.ts | 15 +++ .../install_prebuilt_rules/index.ts | 12 ++ .../install_prebuilt_rules.ts | 99 ++++++++++++++ 5 files changed, 274 insertions(+) create mode 100644 x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/README.md create mode 100644 x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/configs/ess_basic_license.config.ts create mode 100644 x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/index.ts create mode 100644 x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/index.ts create mode 100644 x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/install_prebuilt_rules.ts diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/README.md b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/README.md new file mode 100644 index 0000000000000..8f4f2c131d72c --- /dev/null +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/README.md @@ -0,0 +1,124 @@ +## Kibana Security Solution Prebuilt Rules OOM Testing + +This folder contains **FTR tests** (Functional Test Runner) designed to expose potential Out of Memory (OOM) issues in Kibana when performing memory-intensive operations related to **Security Solution Prebuilt Rules**. + +The tests focus on identifying prebuilt rules package size maximum limit, potential memory leaks or excessive memory usage during operations such as: + +- Installing or upgrading prebuilt rule packages + +- Installing or upgrading prebuilt rules by using prebuilt rule assets in the package + +- Performing related bulk actions + +### ๐Ÿ’ก Purpose + +The goal of these tests is to proactively detect memory exhaustion scenarios by simulating real-world conditions where Kibana operates under constrained memory environments. This helps improve the resilience and stability of the Security Solution features. + +### โš™๏ธ Test Environment Setup + +To effectively reproduce OOM-related behavior, the deployment should be created in Elastic Cloud with 1GB RAM limit for the Kibana instance. Elasticsearch instance isn't so important in that testing a reasonable 1GB RAM instance provide sufficient performance for the testing. ML and Integration instances as well as cold and frozen tier Elasticsearch nodes aren't required. An example Elastic Cloud configuration applicable for internal testing framework QAF (QA Framework) looks like the following + +```yaml +--- +name: '{{ deployment_name }}' +settings: + autoscaling_enabled: '{{ autoscaling_enabled }}' +metadata: + system_owned: false +resources: + elasticsearch: + - region: '{{ region }}' + settings: + dedicated_masters_threshold: 6 + plan: + cluster_topology: + - zone_count: 1 + elasticsearch: + node_attributes: + data: hot + instance_configuration_id: gcp.es.datahot.n2.68x10x45 + node_roles: + - master + - ingest + - remote_cluster_client + - data_hot + - transform + - data_content + id: hot_content + size: + value: 1024 + resource: memory + elasticsearch: + version: '{{ stack_version }}' + deployment_template: + id: gcp-storage-optimized + ref_id: main-elasticsearch + enterprise_search: [] + kibana: + - elasticsearch_cluster_ref_id: main-elasticsearch + region: '{{ region }}' + plan: + cluster_topology: + - instance_configuration_id: gcp.kibana.n2.68x32x45 + zone_count: 1 + size: + value: 1024 + resource: memory + kibana: + version: '{{ stack_version }}' + user_settings_yaml: |- + xpack.securitySolution.prebuiltRulesPackageVersion: '' + ref_id: main-kibana +``` + +Where `` is the package to be installed, e.g. `8.17.4`. + +With QAF the config provided above should be placed in `~/.qaf/config/cloud_plans/prebuilt_rules_oom_testing.yml`. + +> **_NOTE:_** Make sure to use Cloud First Testing (CFT) regions when creating the testing deployment as these regions support Kibana configuration options like `xpack.securitySolution.prebuiltRulesPackageVersion` or `xpack.fleet.registryUrl` otherwise the choice of configuration options will be restricted to those listed on the [General settings in Kibana page](https://www.elastic.co/docs/reference/kibana/configuration-reference/general-settings) and having the "C" icon. The CFT regions are `gcp-us-west2` and `aws-eu-west-1`. + +### ๐Ÿงช Running the Tests + +#### Locally + +The easiest way to run the tests locally is using our internal testing framework QAF. Please follow the steps to prepare the environment + +- Set up QAF by following to the [instructions](https://docs.elastic.dev/appex-qa/qaf/getting-started) (internal) +- Place the plan configuration provided in [Test Environment Setup](#โš™๏ธ-test-environment-setup) in `~/.qaf/config/cloud_plans/prebuilt_rules_oom_testing.yml` +- Create an ECH deployment by running the following command + +```bash +qaf elastic-cloud deployments create --stack-version --version-validation --deployment-name --environment production --no-autoscaling --no-sso --region gcp-us-west2 --plan prebuilt_rules_oom_testing +``` + +where `` specifies which Elastic Stack version should be deployed (unreleased versions will be deployed from snapshots as a **-SNAPSHOT** version) and `` specifies a deployment alias required later on to run the tests. + +For example a command to create `9.2` Elastic Stack would be + +```bash +qaf elastic-cloud deployments create --stack-version 9.2.0 --version-validation --deployment-name prebuilt-rules-oom-test-9.2.0 --environment production --no-autoscaling --no-sso --region gcp-us-west2 --plan prebuilt_rules_oom_testing +``` + +- Run the tests by running the following command + +```bash +qaf kibana ftr run-config --ec-deployment-name --kibana-repo-root /x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/configs/ess_basic_license.config.ts +``` + +where `` is the absolute path to the Kibana's root folder and `` is the deployment name used in `qaf elastic-cloud deployments create` to create an Elastic Stack deployment. + +### ๐Ÿ“ Test Structure + +The tests in this folder include scenarios that: + +- Perform repeated or concurrent installations of prebuilt rules + +- Trigger large API requests to heavy endpoints (e.g., rule package installation endpoints) + +- Each test is expected to either succeed within the memory limit or trigger a recoverable failure. An unexpected crash or unhandled OOM error indicates a potential memory issue. + +### โœ… Expected Outcomes + +- **Pass**: Kibana completes all operations without OOM crashes. + +- **Fail**: Kibana terminates with a fatal OOM error or becomes unresponsive. diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/configs/ess_basic_license.config.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/configs/ess_basic_license.config.ts new file mode 100644 index 0000000000000..10eb1d843166b --- /dev/null +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/configs/ess_basic_license.config.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../configs/ess/rules_management.basic.config') + ); + + const testConfig = { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: 'Rules Management - Prebuilt Rules OOM Testing - ESS Basic License', + }, + }; + + return testConfig; +} diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/index.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/index.ts new file mode 100644 index 0000000000000..ea13a32364387 --- /dev/null +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrProviderContext } from '../../../../../ftr_provider_context'; + +export default ({ loadTestFile }: FtrProviderContext): void => { + describe('Rules Management - Prebuilt Rules OOM Testing - ESS Basic License', function () { + this.tags('skipFIPS'); + loadTestFile(require.resolve('./install_prebuilt_rules')); + }); +}; diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/index.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/index.ts new file mode 100644 index 0000000000000..4775485081ad0 --- /dev/null +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ loadTestFile }: FtrProviderContext): void => { + loadTestFile(require.resolve('./install_prebuilt_rules')); +}; diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/install_prebuilt_rules.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/install_prebuilt_rules.ts new file mode 100644 index 0000000000000..9cc41b93bac68 --- /dev/null +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/oom_testing/install_prebuilt_rules/install_prebuilt_rules.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; +import { + ENDPOINT_PACKAGE_NAME, + PREBUILT_RULES_PACKAGE_NAME, +} from '@kbn/security-solution-plugin/common/detection_engine/constants'; +import { + GET_PREBUILT_RULES_STATUS_URL, + PERFORM_RULE_INSTALLATION_URL, + REVIEW_RULE_INSTALLATION_URL, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; +import type { FtrProviderContext } from '../../../../../../ftr_provider_context'; +import { + deleteEndpointFleetPackage, + deletePrebuiltRulesFleetPackage, +} from '../../../../utils/rules/prebuilt_rules/delete_fleet_packages'; +import { deleteAllRules, waitFor } from '../../../../../../config/services/detections_response'; + +export default ({ getService }: FtrProviderContext): void => { + const es = getService('es'); + const supertest = getService('supertest'); + const log = getService('log'); + const retryService = getService('retry'); + const detectionsApi = getService('detectionsApi'); + + describe('@ess @serverless @skipInServerlessMKI Install from mocked prebuilt rule assets', () => { + beforeEach(async () => { + await deleteAllRules(supertest, log); + + await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService }); + await deleteEndpointFleetPackage({ supertest, es, log, retryService }); + + await waitFor( + async () => { + const { body: prebuiltRulesStatus } = await supertest + .get(GET_PREBUILT_RULES_STATUS_URL) + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .expect(200); + + return ( + prebuiltRulesStatus.stats.num_prebuilt_rules_installed === 0 && + prebuiltRulesStatus.stats.num_prebuilt_rules_to_install === 0 + ); + }, + 'waitForIndexesRefreshAfterPackagesDeletion', + log + ); + }); + + it('install prebuilt rules from a package', async () => { + const { statusCode: bootstrapPrebuiltRulesStatusCode, body: bootstrapPrebuiltRulesResponse } = + await detectionsApi.bootstrapPrebuiltRules(); + + // Assert body first to be able to see error messages in case of failure + expect(bootstrapPrebuiltRulesResponse).toMatchObject({ + packages: expect.arrayContaining([ + expect.objectContaining({ + name: PREBUILT_RULES_PACKAGE_NAME, + }), + expect.objectContaining({ + name: ENDPOINT_PACKAGE_NAME, + }), + ]), + }); + expect(bootstrapPrebuiltRulesStatusCode).toBe(200); + + const { body: reviewPrebuiltRulesForInstallationResponse } = await supertest + .post(REVIEW_RULE_INSTALLATION_URL) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send() + .expect(200); + + expect(reviewPrebuiltRulesForInstallationResponse.rules.length).toBeGreaterThan(0); + + const { body: installPrebuiltRulesResponse } = await supertest + .post(PERFORM_RULE_INSTALLATION_URL) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send({ mode: 'ALL_RULES' }) + .expect(200); + + expect(installPrebuiltRulesResponse.summary.succeeded).toBeGreaterThan(0); + }); + }); +};