From 8699b034df7800c283b99ccc8092e663399675a1 Mon Sep 17 00:00:00 2001 From: Dmitrii Shevchenko Date: Fri, 14 Mar 2025 14:39:32 +0100 Subject: [PATCH] [Security Solution] Batch prebuilt rule installation (#214441) **This is a follow-up to:https://github.com/elastic/kibana/pull/211045** ## Summary This PR removes inefficiencies in prebuilt rule installation memory consumption. ### Before In the worst-case scenario: 1. All currently installed prebuilt rules were fully loaded into memory. 2. All latest rule versions from the rule packages were fully loaded into memory. 3. All base rule versions were pulled into memory, even though they were not used. 4. The algorithm then checked which rules could be installed and installed them all at once. ### After 1. Pull only aggregated information about installed rules (only `rule_id` and `versions`). 2. Pull the same lightweight aggregated info about the latest rules in the package. 3. Using the collected `rule_id`s, fetch rule assets and install them in small batches of up to 100 rules. (cherry picked from commit 6d9fc21db92dd5e02d930971a5a1cf32a97386f0) --- .../perform_rule_installation_handler.ts | 145 ++++++++++++++++++ .../perform_rule_installation_route.ts | 117 +------------- .../model/rule_groups/get_rule_groups.ts | 60 -------- 3 files changed, 147 insertions(+), 175 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts new file mode 100644 index 0000000000000..f156e06d3e5e8 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; +import type { KibanaRequest, KibanaResponseFactory } from '@kbn/core/server'; +import { SkipRuleInstallReason } from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import type { + PerformRuleInstallationResponseBody, + SkippedRuleInstall, + PerformRuleInstallationRequestBody, +} from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import type { SecuritySolutionRequestHandlerContext } from '../../../../../types'; +import { buildSiemResponse } from '../../../routes/utils'; +import { aggregatePrebuiltRuleErrors } from '../../logic/aggregate_prebuilt_rule_errors'; +import { ensureLatestRulesPackageInstalled } from '../../logic/ensure_latest_rules_package_installed'; +import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; +import { createPrebuiltRules } from '../../logic/rule_objects/create_prebuilt_rules'; +import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; +import { performTimelinesInstallation } from '../../logic/perform_timelines_installation'; +import type { RuleSignatureId, RuleVersion } from '../../../../../../common/api/detection_engine'; + +export const performRuleInstallationHandler = async ( + context: SecuritySolutionRequestHandlerContext, + request: KibanaRequest, + response: KibanaResponseFactory +) => { + const siemResponse = buildSiemResponse(response); + + try { + const ctx = await context.resolve(['core', 'alerting', 'securitySolution']); + const config = ctx.securitySolution.getConfig(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = await ctx.alerting.getRulesClient(); + const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient(); + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + const exceptionsListClient = ctx.securitySolution.getExceptionListClient(); + + const { mode } = request.body; + + // This will create the endpoint list if it does not exist yet + await exceptionsListClient?.createEndpointList(); + + // If this API is used directly without hitting any detection engine + // pages first, the rules package might be missing. + await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution); + + const allLatestVersions = await ruleAssetsClient.fetchLatestVersions(); + const currentRuleVersions = await ruleObjectsClient.fetchInstalledRuleVersions(); + const currentRuleVersionsMap = new Map( + currentRuleVersions.map((version) => [version.rule_id, version]) + ); + + const allInstallableRules = allLatestVersions.filter((latestVersion) => { + const currentVersion = currentRuleVersionsMap.get(latestVersion.rule_id); + return !currentVersion; + }); + + const ruleInstallQueue: Array<{ + rule_id: RuleSignatureId; + version: RuleVersion; + }> = []; + const ruleErrors = []; + const installedRules = []; + const skippedRules: SkippedRuleInstall[] = []; + + // Perform all the checks we can before we start the upgrade process + if (mode === 'SPECIFIC_RULES') { + const installableRuleIds = new Set(allInstallableRules.map((rule) => rule.rule_id)); + request.body.rules.forEach((rule) => { + // Check that the requested rule is not installed yet + if (currentRuleVersionsMap.has(rule.rule_id)) { + skippedRules.push({ + rule_id: rule.rule_id, + reason: SkipRuleInstallReason.ALREADY_INSTALLED, + }); + return; + } + + // Check that the requested rule is installable + if (!installableRuleIds.has(rule.rule_id)) { + ruleErrors.push({ + error: new Error( + `Rule with ID "${rule.rule_id}" and version "${rule.version}" not found` + ), + item: rule, + }); + return; + } + + ruleInstallQueue.push(rule); + }); + } else if (mode === 'ALL_RULES') { + ruleInstallQueue.push(...allInstallableRules); + } + + const BATCH_SIZE = 100; + while (ruleInstallQueue.length > 0) { + const rulesToInstall = ruleInstallQueue.splice(0, BATCH_SIZE); + const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(rulesToInstall); + + const { results, errors } = await createPrebuiltRules(detectionRulesClient, ruleAssets); + installedRules.push(...results); + ruleErrors.push(...errors); + } + + const { error: timelineInstallationError } = await performTimelinesInstallation( + ctx.securitySolution + ); + + const allErrors = aggregatePrebuiltRuleErrors(ruleErrors); + if (timelineInstallationError) { + allErrors.push({ + message: timelineInstallationError, + rules: [], + }); + } + + const body: PerformRuleInstallationResponseBody = { + summary: { + total: installedRules.length + skippedRules.length + ruleErrors.length, + succeeded: installedRules.length, + skipped: skippedRules.length, + failed: ruleErrors.length, + }, + results: { + created: installedRules.map(({ result }) => result), + skipped: skippedRules, + }, + errors: allErrors, + }; + + return response.ok({ body }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts index 7de0bb83ed000..1e07881b45964 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts @@ -5,33 +5,18 @@ * 2.0. */ -import { transformError } from '@kbn/securitysolution-es-utils'; import { PERFORM_RULE_INSTALLATION_URL, PerformRuleInstallationRequestBody, - SkipRuleInstallReason, -} from '../../../../../../common/api/detection_engine/prebuilt_rules'; -import type { - PerformRuleInstallationResponseBody, - SkippedRuleInstall, } from '../../../../../../common/api/detection_engine/prebuilt_rules'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; -import type { PromisePoolError } from '../../../../../utils/promise_pool'; -import { buildSiemResponse } from '../../../routes/utils'; -import { aggregatePrebuiltRuleErrors } from '../../logic/aggregate_prebuilt_rule_errors'; -import { ensureLatestRulesPackageInstalled } from '../../logic/ensure_latest_rules_package_installed'; -import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; -import { createPrebuiltRules } from '../../logic/rule_objects/create_prebuilt_rules'; -import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; -import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad'; -import { performTimelinesInstallation } from '../../logic/perform_timelines_installation'; import { PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS, PREBUILT_RULES_OPERATION_CONCURRENCY, } from '../../constants'; -import { getRuleGroups } from '../../model/rule_groups/get_rule_groups'; import { routeLimitedConcurrencyTag } from '../../../../../utils/route_limited_concurrency_tag'; +import { performRuleInstallationHandler } from './perform_rule_installation_handler'; export const performRuleInstallationRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -59,104 +44,6 @@ export const performRuleInstallationRoute = (router: SecuritySolutionPluginRoute }, }, }, - async (context, request, response) => { - const siemResponse = buildSiemResponse(response); - - try { - const ctx = await context.resolve(['core', 'alerting', 'securitySolution']); - const config = ctx.securitySolution.getConfig(); - const soClient = ctx.core.savedObjects.client; - const rulesClient = await ctx.alerting.getRulesClient(); - const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient(); - const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); - const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); - const exceptionsListClient = ctx.securitySolution.getExceptionListClient(); - - const { mode } = request.body; - - // This will create the endpoint list if it does not exist yet - await exceptionsListClient?.createEndpointList(); - - // If this API is used directly without hitting any detection engine - // pages first, the rules package might be missing. - await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution); - - const fetchErrors: Array> = []; - const skippedRules: SkippedRuleInstall[] = []; - - const ruleVersionsMap = await fetchRuleVersionsTriad({ - ruleAssetsClient, - ruleObjectsClient, - versionSpecifiers: mode === 'ALL_RULES' ? undefined : request.body.rules, - }); - const { currentRules, installableRules } = getRuleGroups(ruleVersionsMap); - - // Perform all the checks we can before we start the upgrade process - if (mode === 'SPECIFIC_RULES') { - const currentRuleIds = new Set(currentRules.map((rule) => rule.rule_id)); - const installableRuleIds = new Set(installableRules.map((rule) => rule.rule_id)); - request.body.rules.forEach((rule) => { - // Check that the requested rule is not installed yet - if (currentRuleIds.has(rule.rule_id)) { - skippedRules.push({ - rule_id: rule.rule_id, - reason: SkipRuleInstallReason.ALREADY_INSTALLED, - }); - return; - } - - // Check that the requested rule is installable - if (!installableRuleIds.has(rule.rule_id)) { - fetchErrors.push({ - error: new Error( - `Rule with ID "${rule.rule_id}" and version "${rule.version}" not found` - ), - item: rule, - }); - } - }); - } - - const { results: installedRules, errors: installationErrors } = await createPrebuiltRules( - detectionRulesClient, - installableRules - ); - const ruleErrors = [...fetchErrors, ...installationErrors]; - - const { error: timelineInstallationError } = await performTimelinesInstallation( - ctx.securitySolution - ); - - const allErrors = aggregatePrebuiltRuleErrors(ruleErrors); - if (timelineInstallationError) { - allErrors.push({ - message: timelineInstallationError, - rules: [], - }); - } - - const body: PerformRuleInstallationResponseBody = { - summary: { - total: installedRules.length + skippedRules.length + ruleErrors.length, - succeeded: installedRules.length, - skipped: skippedRules.length, - failed: ruleErrors.length, - }, - results: { - created: installedRules.map(({ result }) => result), - skipped: skippedRules, - }, - errors: allErrors, - }; - - return response.ok({ body }); - } catch (err) { - const error = transformError(err); - return siemResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } + performRuleInstallationHandler ); }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_groups/get_rule_groups.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_groups/get_rule_groups.ts index c9adf6db850fb..ff92f983dcfd7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_groups/get_rule_groups.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_groups/get_rule_groups.ts @@ -6,7 +6,6 @@ */ import type { RuleResponse } from '../../../../../../common/api/detection_engine/model/rule_schema'; -import type { RuleVersions } from '../../logic/diff/calculate_rule_diff'; import type { PrebuiltRuleAsset } from '../rule_assets/prebuilt_rule_asset'; export interface RuleTriad { @@ -23,62 +22,3 @@ export interface RuleTriad { */ target: PrebuiltRuleAsset; } -export interface RuleGroups { - /** - * Rules that are currently installed in Kibana - */ - currentRules: RuleResponse[]; - /** - * Rules that are ready to be installed - */ - installableRules: PrebuiltRuleAsset[]; - /** - * Rules that are installed but outdated - */ - upgradeableRules: RuleTriad[]; - /** - * All available rules - * (installed and not installed) - */ - totalAvailableRules: PrebuiltRuleAsset[]; -} - -export const getRuleGroups = (ruleVersionsMap: Map): RuleGroups => { - const currentRules: RuleResponse[] = []; - const installableRules: PrebuiltRuleAsset[] = []; - const totalAvailableRules: PrebuiltRuleAsset[] = []; - const upgradeableRules: RuleGroups['upgradeableRules'] = []; - - ruleVersionsMap.forEach(({ base, current, target }) => { - if (target != null) { - // If this rule is available in the package - totalAvailableRules.push(target); - } - - if (current != null) { - // If this rule is installed - currentRules.push(current); - } - - if (current == null && target != null) { - // If this rule is not installed - installableRules.push(target); - } - - if (current != null && target != null && current.version < target.version) { - // If this rule is installed but outdated - upgradeableRules.push({ - base, - current, - target, - }); - } - }); - - return { - currentRules, - installableRules, - upgradeableRules, - totalAvailableRules, - }; -};