Skip to content

[8.18] [Security Solution] Batch prebuilt rule installation (#214441) #214575

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kibanamachine merged 1 commit into from
Mar 14, 2025
+147 −175
Merged

[8.18] [Security Solution] Batch prebuilt rule installation (#214441) #214575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 145 additions & 0 deletions ..._engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { transformError } from '@kbn/securitysolution-es-utils';
import type { KibanaRequest, KibanaResponseFactory } from '@kbn/core/server';
import { SkipRuleInstallReason } from '../../../../../../common/api/detection_engine/prebuilt_rules';
import type {
PerformRuleInstallationResponseBody,
SkippedRuleInstall,
PerformRuleInstallationRequestBody,
} from '../../../../../../common/api/detection_engine/prebuilt_rules';
import type { SecuritySolutionRequestHandlerContext } from '../../../../../types';
import { buildSiemResponse } from '../../../routes/utils';
import { aggregatePrebuiltRuleErrors } from '../../logic/aggregate_prebuilt_rule_errors';
import { ensureLatestRulesPackageInstalled } from '../../logic/ensure_latest_rules_package_installed';
import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client';
import { createPrebuiltRules } from '../../logic/rule_objects/create_prebuilt_rules';
import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client';
import { performTimelinesInstallation } from '../../logic/perform_timelines_installation';
import type { RuleSignatureId, RuleVersion } from '../../../../../../common/api/detection_engine';

export const performRuleInstallationHandler = async (
context: SecuritySolutionRequestHandlerContext,
request: KibanaRequest<unknown, unknown, PerformRuleInstallationRequestBody>,
response: KibanaResponseFactory
) => {
const siemResponse = buildSiemResponse(response);

try {
const ctx = await context.resolve(['core', 'alerting', 'securitySolution']);
const config = ctx.securitySolution.getConfig();
const soClient = ctx.core.savedObjects.client;
const rulesClient = await ctx.alerting.getRulesClient();
const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient();
const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient);
const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient);
const exceptionsListClient = ctx.securitySolution.getExceptionListClient();

const { mode } = request.body;

// This will create the endpoint list if it does not exist yet
await exceptionsListClient?.createEndpointList();

// If this API is used directly without hitting any detection engine
// pages first, the rules package might be missing.
await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution);

const allLatestVersions = await ruleAssetsClient.fetchLatestVersions();
const currentRuleVersions = await ruleObjectsClient.fetchInstalledRuleVersions();
const currentRuleVersionsMap = new Map(
currentRuleVersions.map((version) => [version.rule_id, version])
);

const allInstallableRules = allLatestVersions.filter((latestVersion) => {
const currentVersion = currentRuleVersionsMap.get(latestVersion.rule_id);
return !currentVersion;
});

const ruleInstallQueue: Array<{
rule_id: RuleSignatureId;
version: RuleVersion;
}> = [];
const ruleErrors = [];
const installedRules = [];
const skippedRules: SkippedRuleInstall[] = [];

// Perform all the checks we can before we start the upgrade process
if (mode === 'SPECIFIC_RULES') {
const installableRuleIds = new Set(allInstallableRules.map((rule) => rule.rule_id));
request.body.rules.forEach((rule) => {
// Check that the requested rule is not installed yet
if (currentRuleVersionsMap.has(rule.rule_id)) {
skippedRules.push({
rule_id: rule.rule_id,
reason: SkipRuleInstallReason.ALREADY_INSTALLED,
});
return;
}

// Check that the requested rule is installable
if (!installableRuleIds.has(rule.rule_id)) {
ruleErrors.push({
error: new Error(
`Rule with ID "${rule.rule_id}" and version "${rule.version}" not found`
),
item: rule,
});
return;
}

ruleInstallQueue.push(rule);
});
} else if (mode === 'ALL_RULES') {
ruleInstallQueue.push(...allInstallableRules);
}

const BATCH_SIZE = 100;
while (ruleInstallQueue.length > 0) {
const rulesToInstall = ruleInstallQueue.splice(0, BATCH_SIZE);
const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(rulesToInstall);

const { results, errors } = await createPrebuiltRules(detectionRulesClient, ruleAssets);
installedRules.push(...results);
ruleErrors.push(...errors);
}

const { error: timelineInstallationError } = await performTimelinesInstallation(
ctx.securitySolution
);

const allErrors = aggregatePrebuiltRuleErrors(ruleErrors);
if (timelineInstallationError) {
allErrors.push({
message: timelineInstallationError,
rules: [],
});
}

const body: PerformRuleInstallationResponseBody = {
summary: {
total: installedRules.length + skippedRules.length + ruleErrors.length,
succeeded: installedRules.length,
skipped: skippedRules.length,
failed: ruleErrors.length,
},
results: {
created: installedRules.map(({ result }) => result),
skipped: skippedRules,
},
errors: allErrors,
};

return response.ok({ body });
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
};
117 changes: 2 additions & 115 deletions ...on_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,18 @@
* 2.0.
*/

import { transformError } from '@kbn/securitysolution-es-utils';
import {
PERFORM_RULE_INSTALLATION_URL,
PerformRuleInstallationRequestBody,
SkipRuleInstallReason,
} from '../../../../../../common/api/detection_engine/prebuilt_rules';
import type {
PerformRuleInstallationResponseBody,
SkippedRuleInstall,
} from '../../../../../../common/api/detection_engine/prebuilt_rules';
import type { SecuritySolutionPluginRouter } from '../../../../../types';
import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation';
import type { PromisePoolError } from '../../../../../utils/promise_pool';
import { buildSiemResponse } from '../../../routes/utils';
import { aggregatePrebuiltRuleErrors } from '../../logic/aggregate_prebuilt_rule_errors';
import { ensureLatestRulesPackageInstalled } from '../../logic/ensure_latest_rules_package_installed';
import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client';
import { createPrebuiltRules } from '../../logic/rule_objects/create_prebuilt_rules';
import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client';
import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad';
import { performTimelinesInstallation } from '../../logic/perform_timelines_installation';
import {
PREBUILT_RULES_OPERATION_SOCKET_TIMEOUT_MS,
PREBUILT_RULES_OPERATION_CONCURRENCY,
} from '../../constants';
import { getRuleGroups } from '../../model/rule_groups/get_rule_groups';
import { routeLimitedConcurrencyTag } from '../../../../../utils/route_limited_concurrency_tag';
import { performRuleInstallationHandler } from './perform_rule_installation_handler';

export const performRuleInstallationRoute = (router: SecuritySolutionPluginRouter) => {
router.versioned
Expand Down Expand Up @@ -59,104 +44,6 @@ export const performRuleInstallationRoute = (router: SecuritySolutionPluginRoute
},
},
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);

try {
const ctx = await context.resolve(['core', 'alerting', 'securitySolution']);
const config = ctx.securitySolution.getConfig();
const soClient = ctx.core.savedObjects.client;
const rulesClient = await ctx.alerting.getRulesClient();
const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient();
const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient);
const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient);
const exceptionsListClient = ctx.securitySolution.getExceptionListClient();

const { mode } = request.body;

// This will create the endpoint list if it does not exist yet
await exceptionsListClient?.createEndpointList();

// If this API is used directly without hitting any detection engine
// pages first, the rules package might be missing.
await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution);

const fetchErrors: Array<PromisePoolError<{ rule_id: string }>> = [];
const skippedRules: SkippedRuleInstall[] = [];

const ruleVersionsMap = await fetchRuleVersionsTriad({
ruleAssetsClient,
ruleObjectsClient,
versionSpecifiers: mode === 'ALL_RULES' ? undefined : request.body.rules,
});
const { currentRules, installableRules } = getRuleGroups(ruleVersionsMap);

// Perform all the checks we can before we start the upgrade process
if (mode === 'SPECIFIC_RULES') {
const currentRuleIds = new Set(currentRules.map((rule) => rule.rule_id));
const installableRuleIds = new Set(installableRules.map((rule) => rule.rule_id));
request.body.rules.forEach((rule) => {
// Check that the requested rule is not installed yet
if (currentRuleIds.has(rule.rule_id)) {
skippedRules.push({
rule_id: rule.rule_id,
reason: SkipRuleInstallReason.ALREADY_INSTALLED,
});
return;
}

// Check that the requested rule is installable
if (!installableRuleIds.has(rule.rule_id)) {
fetchErrors.push({
error: new Error(
`Rule with ID "${rule.rule_id}" and version "${rule.version}" not found`
),
item: rule,
});
}
});
}

const { results: installedRules, errors: installationErrors } = await createPrebuiltRules(
detectionRulesClient,
installableRules
);
const ruleErrors = [...fetchErrors, ...installationErrors];

const { error: timelineInstallationError } = await performTimelinesInstallation(
ctx.securitySolution
);

const allErrors = aggregatePrebuiltRuleErrors(ruleErrors);
if (timelineInstallationError) {
allErrors.push({
message: timelineInstallationError,
rules: [],
});
}

const body: PerformRuleInstallationResponseBody = {
summary: {
total: installedRules.length + skippedRules.length + ruleErrors.length,
succeeded: installedRules.length,
skipped: skippedRules.length,
failed: ruleErrors.length,
},
results: {
created: installedRules.map(({ result }) => result),
skipped: skippedRules,
},
errors: allErrors,
};

return response.ok({ body });
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
performRuleInstallationHandler
);
};
60 changes: 0 additions & 60 deletions ..._solution/server/lib/detection_engine/prebuilt_rules/model/rule_groups/get_rule_groups.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

import type { RuleResponse } from '../../../../../../common/api/detection_engine/model/rule_schema';
import type { RuleVersions } from '../../logic/diff/calculate_rule_diff';
import type { PrebuiltRuleAsset } from '../rule_assets/prebuilt_rule_asset';

export interface RuleTriad {
Expand All @@ -23,62 +22,3 @@ export interface RuleTriad {
*/
target: PrebuiltRuleAsset;
}
export interface RuleGroups {
/**
* Rules that are currently installed in Kibana
*/
currentRules: RuleResponse[];
/**
* Rules that are ready to be installed
*/
installableRules: PrebuiltRuleAsset[];
/**
* Rules that are installed but outdated
*/
upgradeableRules: RuleTriad[];
/**
* All available rules
* (installed and not installed)
*/
totalAvailableRules: PrebuiltRuleAsset[];
}

export const getRuleGroups = (ruleVersionsMap: Map<string, RuleVersions>): RuleGroups => {
const currentRules: RuleResponse[] = [];
const installableRules: PrebuiltRuleAsset[] = [];
const totalAvailableRules: PrebuiltRuleAsset[] = [];
const upgradeableRules: RuleGroups['upgradeableRules'] = [];

ruleVersionsMap.forEach(({ base, current, target }) => {
if (target != null) {
// If this rule is available in the package
totalAvailableRules.push(target);
}

if (current != null) {
// If this rule is installed
currentRules.push(current);
}

if (current == null && target != null) {
// If this rule is not installed
installableRules.push(target);
}

if (current != null && target != null && current.version < target.version) {
// If this rule is installed but outdated
upgradeableRules.push({
base,
current,
target,
});
}
});

return {
currentRules,
installableRules,
upgradeableRules,
totalAvailableRules,
};
};