From 850856dbeed09ab6ade47bdd0cd4a1b8cc723ce6 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Thu, 8 Apr 2021 21:37:47 -0600 Subject: [PATCH 1/9] Make the prepackaged rules functions async --- .../routes/rules/add_prepackaged_rules_route.test.ts | 2 +- .../routes/rules/add_prepackaged_rules_route.ts | 2 +- .../server/lib/detection_engine/rules/constants.ts | 8 ++++++++ .../rules/get_prepackaged_rules.test.ts | 12 +++++++----- .../detection_engine/rules/get_prepackaged_rules.ts | 6 ++++-- 5 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index 1195f9e5e1e96..f8a05bdc7379f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -25,7 +25,7 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: (): AddPrepackagedRulesSchemaDecoded[] => { + getPrepackagedRules: async (): AddPrepackagedRulesSchemaDecoded[] => { return [ { author: ['Elastic'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts index 8a8d6925b0e80..98f26cc7e8649 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts @@ -120,7 +120,7 @@ export const createPrepackagedRules = async ( await exceptionsListClient.createEndpointList(); } - const rulesFromFileSystem = getPrepackagedRules(); + const rulesFromFileSystem = await getPrepackagedRules(); const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts new file mode 100644 index 0000000000000..016985cce0b46 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EPR_PACKAGE = 'security_detection_engine'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts index 039bc8c1e2e49..79d7d7e700999 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts @@ -14,8 +14,8 @@ describe('get_existing_prepackaged_rules', () => { expect(() => getPrepackagedRules()).not.toThrow(); }); - test('no rule should have the same rule_id as another rule_id', () => { - const prePackagedRules = getPrepackagedRules(); + test('no rule should have the same rule_id as another rule_id', async () => { + const prePackagedRules = await getPrepackagedRules(); let existingRuleIds: AddPrepackagedRulesSchemaDecoded[] = []; prePackagedRules.forEach((rule) => { const foundDuplicate = existingRuleIds.reduce((accum, existingRule) => { @@ -35,14 +35,16 @@ describe('get_existing_prepackaged_rules', () => { test('should throw an exception if a pre-packaged rule is not valid', () => { // @ts-expect-error intentionally invalid argument - expect(() => getPrepackagedRules([{ not_valid_made_up_key: true }])).toThrow( + expect(async () => getPrepackagedRules([{ not_valid_made_up_key: true }])).rejects.toThrow( 'name: "(rule name unknown)", rule_id: "(rule rule_id unknown)" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "name",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "rule_id",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n "not_valid_made_up_key": true\n}' ); }); test('should throw an exception with a message having rule_id and name in it', () => { - // @ts-expect-error intentionally invalid argument - expect(() => getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }])).toThrow( + expect(async () => + // @ts-expect-error intentionally invalid argument + getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }]) + ).rejects.toThrow( 'name: "rule name", rule_id: "id-123" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n "name": "rule name",\n "rule_id": "id-123"\n}' ); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index 508238afcb6df..e5d9a4e80c7dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -52,7 +52,9 @@ export const validateAllPrepackagedRules = ( }); }; -export const getPrepackagedRules = ( +export const getPrepackagedRules = async ( // @ts-expect-error mock data is too loosely typed rules: AddPrepackagedRulesSchema[] = rawRules -): AddPrepackagedRulesSchemaDecoded[] => validateAllPrepackagedRules(rules); +): Promise => { + return validateAllPrepackagedRules(rules); +}; From 38ba79778ae3e1ec11de64d8121c9f5ad030c052 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Fri, 9 Apr 2021 08:42:46 -0600 Subject: [PATCH 2/9] Fix type for getPrepackagedRules mock --- .../routes/rules/add_prepackaged_rules_route.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index f8a05bdc7379f..aaeeecde3e624 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -25,7 +25,7 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: async (): AddPrepackagedRulesSchemaDecoded[] => { + getPrepackagedRules: async (): Promise => { return [ { author: ['Elastic'], From 93e315956094c8c4964d77294aa57930d884d7d9 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Fri, 9 Apr 2021 14:16:19 -0600 Subject: [PATCH 3/9] Install updates from saved objects & FS --- .../rules/add_prepackaged_rules_route.ts | 11 ++-- .../get_prepackaged_rules_status_route.ts | 11 ++-- .../rules/get_prepackaged_rules.test.ts | 10 ++-- .../rules/get_prepackaged_rules.ts | 39 ++++++++++++++- .../rules/rule_asset_saved_objects_client.ts | 50 +++++++++++++++++++ .../lib/detection_engine/rules/types.ts | 13 +++++ 6 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts index 98f26cc7e8649..4f9bd7d0cfd6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts @@ -25,12 +25,13 @@ import { SetupPlugins } from '../../../../plugin'; import { buildFrameworkRequest } from '../../../timeline/utils/common'; import { getIndexExists } from '../../index/get_index_exists'; -import { getPrepackagedRules } from '../../rules/get_prepackaged_rules'; +import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { installPrepackagedRules } from '../../rules/install_prepacked_rules'; import { updatePrepackagedRules } from '../../rules/update_prepacked_rules'; import { getRulesToInstall } from '../../rules/get_rules_to_install'; import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; import { transformError, buildSiemResponse } from '../utils'; import { AlertsClient } from '../../../../../../alerting/server'; @@ -110,7 +111,7 @@ export const createPrepackagedRules = async ( const savedObjectsClient = context.core.savedObjects.client; const exceptionsListClient = context.lists != null ? context.lists.getExceptionListClient() : exceptionsClient; - + const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient); if (!siemClient || !alertsClient) { throw new PrepackagedRulesError('', 404); } @@ -120,10 +121,10 @@ export const createPrepackagedRules = async ( await exceptionsListClient.createEndpointList(); } - const rulesFromFileSystem = await getPrepackagedRules(); + const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient); const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); - const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); - const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); + const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules); + const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules); const signalsIndex = siemClient.getSignalsIndex(); if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) { const signalsIndexExists = await getIndexExists(esClient.asCurrentUser, signalsIndex); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts index c67f2cb6e9545..33f9746fe9245 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts @@ -13,11 +13,12 @@ import { import type { SecuritySolutionPluginRouter } from '../../../../types'; import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../../common/constants'; import { transformError, buildSiemResponse } from '../utils'; -import { getPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { getRulesToInstall } from '../../rules/get_rules_to_install'; import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { findRules } from '../../rules/find_rules'; +import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; import { buildFrameworkRequest } from '../../../timeline/utils/common'; import { ConfigType } from '../../../../config'; import { SetupPlugins } from '../../../../plugin'; @@ -40,15 +41,17 @@ export const getPrepackagedRulesStatusRoute = ( }, }, async (context, request, response) => { + const savedObjectsClient = context.core.savedObjects.client; const siemResponse = buildSiemResponse(response); const alertsClient = context.alerting?.getAlertsClient(); + const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient); if (!alertsClient) { return siemResponse.error({ statusCode: 404 }); } try { - const rulesFromFileSystem = getPrepackagedRules(); + const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient); const customRules = await findRules({ alertsClient, perPage: 1, @@ -61,8 +64,8 @@ export const getPrepackagedRulesStatusRoute = ( const frameworkRequest = await buildFrameworkRequest(context, security, request); const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); - const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); - const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); + const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules); + const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules); const prepackagedTimelineStatus = await checkTimelinesStatus(frameworkRequest); const [validatedprepackagedTimelineStatus] = validate( prepackagedTimelineStatus, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts index 79d7d7e700999..2d92731dbbdfd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts @@ -14,8 +14,8 @@ describe('get_existing_prepackaged_rules', () => { expect(() => getPrepackagedRules()).not.toThrow(); }); - test('no rule should have the same rule_id as another rule_id', async () => { - const prePackagedRules = await getPrepackagedRules(); + test('no rule should have the same rule_id as another rule_id', () => { + const prePackagedRules = getPrepackagedRules(); let existingRuleIds: AddPrepackagedRulesSchemaDecoded[] = []; prePackagedRules.forEach((rule) => { const foundDuplicate = existingRuleIds.reduce((accum, existingRule) => { @@ -35,16 +35,16 @@ describe('get_existing_prepackaged_rules', () => { test('should throw an exception if a pre-packaged rule is not valid', () => { // @ts-expect-error intentionally invalid argument - expect(async () => getPrepackagedRules([{ not_valid_made_up_key: true }])).rejects.toThrow( + expect(() => getPrepackagedRules([{ not_valid_made_up_key: true }])).toThrow( 'name: "(rule name unknown)", rule_id: "(rule rule_id unknown)" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "name",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "rule_id",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n "not_valid_made_up_key": true\n}' ); }); test('should throw an exception with a message having rule_id and name in it', () => { - expect(async () => + expect(() => // @ts-expect-error intentionally invalid argument getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }]) - ).rejects.toThrow( + ).toThrow( 'name: "rule name", rule_id: "id-123" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n "name": "rule name",\n "rule_id": "id-123"\n}' ); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index e5d9a4e80c7dc..65c9f0b4704cd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -19,6 +19,7 @@ import { BadRequestError } from '../errors/bad_request_error'; // TODO: convert rules files to TS and add explicit type definitions import { rawRules } from './prepackaged_rules'; +import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client'; /** * Validate the rules from the file system and throw any errors indicating to the developer @@ -52,9 +53,43 @@ export const validateAllPrepackagedRules = ( }); }; -export const getPrepackagedRules = async ( +/** + * Retrieve and validate rules that were installed from Fleet as saved objects. + */ +export const getFleetInstalledRules = async ( + client: RuleAssetSavedObjectsClient +): Promise => { + const fleetResponse = await client.all(); + const fleetRules = fleetResponse.map( + // @ts-expect-error data is too loosely typed + (so) => so.attributes as AddPrepackagedRulesSchema + ); + return validateAllPrepackagedRules(fleetRules); +}; + +export const getPrepackagedRules = ( // @ts-expect-error mock data is too loosely typed rules: AddPrepackagedRulesSchema[] = rawRules -): Promise => { +): AddPrepackagedRulesSchemaDecoded[] => { return validateAllPrepackagedRules(rules); }; + +export const getLatestPrepackagedRules = async ( + client: RuleAssetSavedObjectsClient +): Promise => { + // build a map of the most version of each rule + const prepackaged = getPrepackagedRules(); + const ruleMap = new Map(prepackaged.map((r) => [r.rule_id, r])); + + // check the rules installed via fleet and create/update if the version is newer + const fleetRules = await getFleetInstalledRules(client); + const fleetUpdates = fleetRules.filter( + // @ts-expect-error data ruleMap.get() is safe because of ruleMap.has() check + (r) => !ruleMap.has(r.rule_id) || ruleMap.get(r.rule_id).version < r.version + ); + + // add the new or updated rules to the map + fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r)); + + return Array.from(ruleMap.values()); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts new file mode 100644 index 0000000000000..42ee074413a82 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SavedObjectsClientContract, + SavedObjectsFindOptions, + SavedObjectsFindResponse, +} from '../../../../../../../src/core/server'; +import { ruleAssetSavedObjectType } from '../rules/saved_object_mappings'; +import { IRuleAssetSavedObject } from '../rules/types'; + +const DEFAULT_PAGE_SIZE = 100; + +export interface RuleAssetSavedObjectsClient { + find: ( + options?: Omit + ) => Promise>; + all: () => Promise; +} + +export const ruleAssetSavedObjectsClientFactory = ( + savedObjectsClient: SavedObjectsClientContract +): RuleAssetSavedObjectsClient => { + return { + find: (options) => + savedObjectsClient.find({ + ...options, + type: ruleAssetSavedObjectType, + }), + all: async () => { + const finder = savedObjectsClient.createPointInTimeFinder({ + perPage: DEFAULT_PAGE_SIZE, + type: ruleAssetSavedObjectType, + }); + const responses: IRuleAssetSavedObject[] = []; + for await (const response of finder.find()) { + response.saved_objects.forEach((so) => responses.push(so as IRuleAssetSavedObject)); + if (response.total === 0) { + break; + } + } + await finder.close(); + return responses; + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 13a255d1b56d4..38e93b2d7812c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -163,6 +163,19 @@ export interface IRuleStatusFindType { saved_objects: IRuleStatusSavedObject[]; } +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface IRuleAssetSOAttributes extends Record { + rule_id: string | null | undefined; + version: string | null | undefined; + name: string | null | undefined; +} + +export interface IRuleAssetSavedObject { + type: string; + id: string; + attributes: Array>; +} + export interface HapiReadableStream extends Readable { hapi: { filename: string; From a1febb9895320a9b4481525030784bd6a9c22639 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Sat, 10 Apr 2021 15:18:02 -0600 Subject: [PATCH 4/9] Mock getLatestPrepackagedRules instead of getPrepackagedRules --- .../routes/rules/add_prepackaged_rules_route.test.ts | 2 +- .../routes/rules/get_prepackaged_rules_status_route.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index aaeeecde3e624..026820a8f2ff7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -25,7 +25,7 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: async (): Promise => { + getLatestPrepackagedRules: async (): Promise => { return [ { author: ['Elastic'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts index 9e843d463ab3e..3c8321ee8eb9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts @@ -23,7 +23,7 @@ import { jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: () => { + getLatestPrepackagedRules: async () => { return [ { rule_id: 'rule-1', From dc48e1a35a78c428c8b2cc88efc1f9e1e7d4d6e7 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 12 Apr 2021 11:46:52 -0600 Subject: [PATCH 5/9] Cleanup ruleAssetSavedObjectsClientFactory.all --- .../rules/rule_asset_saved_objects_client.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts index 42ee074413a82..ac0969dfc975d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts @@ -38,10 +38,7 @@ export const ruleAssetSavedObjectsClientFactory = ( }); const responses: IRuleAssetSavedObject[] = []; for await (const response of finder.find()) { - response.saved_objects.forEach((so) => responses.push(so as IRuleAssetSavedObject)); - if (response.total === 0) { - break; - } + responses.push(...response.saved_objects.map((so) => so as IRuleAssetSavedObject)); } await finder.close(); return responses; From 1e561149661622e0cfd8f9817ad85a573b454ed7 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 12 Apr 2021 12:05:01 -0600 Subject: [PATCH 6/9] Fix comment for "most recent version" --- .../server/lib/detection_engine/rules/get_prepackaged_rules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index 65c9f0b4704cd..ad6e23c4ebfb7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -77,7 +77,7 @@ export const getPrepackagedRules = ( export const getLatestPrepackagedRules = async ( client: RuleAssetSavedObjectsClient ): Promise => { - // build a map of the most version of each rule + // build a map of the most recent version of each rule const prepackaged = getPrepackagedRules(); const ruleMap = new Map(prepackaged.map((r) => [r.rule_id, r])); From 3df98addc33f7f6b38e2aa6132426980584a4fb5 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 12 Apr 2021 12:32:02 -0600 Subject: [PATCH 7/9] Switch to ruleMap.get() for less typescript errors --- .../lib/detection_engine/rules/get_prepackaged_rules.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index ad6e23c4ebfb7..5134f4f5623b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -83,10 +83,10 @@ export const getLatestPrepackagedRules = async ( // check the rules installed via fleet and create/update if the version is newer const fleetRules = await getFleetInstalledRules(client); - const fleetUpdates = fleetRules.filter( - // @ts-expect-error data ruleMap.get() is safe because of ruleMap.has() check - (r) => !ruleMap.has(r.rule_id) || ruleMap.get(r.rule_id).version < r.version - ); + const fleetUpdates = fleetRules.filter((r) => { + const rule = ruleMap.get(r.rule_id); + return rule == null || rule.version < r.version; + }); // add the new or updated rules to the map fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r)); From 84248b3b98b6bb80d623152bf1c5b9eee6f22c5e Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 12 Apr 2021 12:36:04 -0600 Subject: [PATCH 8/9] Remove unneeded constants --- .../server/lib/detection_engine/rules/constants.ts | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts deleted file mode 100644 index 016985cce0b46..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const EPR_PACKAGE = 'security_detection_engine'; From 41536f0e8a79029463d662cfde4fdc5c23528f93 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Mon, 12 Apr 2021 14:30:47 -0600 Subject: [PATCH 9/9] Fix SO.attributes sig and use custom validation --- .../rules/get_prepackaged_rules.ts | 39 ++++++++++++++++--- .../lib/detection_engine/rules/types.ts | 2 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index 5134f4f5623b5..b91557c6d7b1b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -20,6 +20,8 @@ import { BadRequestError } from '../errors/bad_request_error'; // TODO: convert rules files to TS and add explicit type definitions import { rawRules } from './prepackaged_rules'; import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client'; +import { IRuleAssetSOAttributes } from './types'; +import { SavedObjectAttributes } from '../../../../../../../src/core/types'; /** * Validate the rules from the file system and throw any errors indicating to the developer @@ -53,6 +55,36 @@ export const validateAllPrepackagedRules = ( }); }; +/** + * Validate the rules from Saved Objects created by Fleet. + */ +export const validateAllRuleSavedObjects = ( + rules: Array +): AddPrepackagedRulesSchemaDecoded[] => { + return rules.map((rule) => { + const decoded = addPrepackagedRulesSchema.decode(rule); + const checked = exactCheck(rule, decoded); + + const onLeft = (errors: t.Errors): AddPrepackagedRulesSchemaDecoded => { + const ruleName = rule.name ? rule.name : '(rule name unknown)'; + const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)'; + throw new BadRequestError( + `name: "${ruleName}", rule_id: "${ruleId}" within the security-rule saved object ` + + `is not a valid detection engine rule. Expect the system ` + + `to not work with pre-packaged rules until this rule is fixed ` + + `or the file is removed. Error is: ${formatErrors( + errors + ).join()}, Full rule contents are:\n${JSON.stringify(rule, null, 2)}` + ); + }; + + const onRight = (schema: AddPrepackagedRulesSchema): AddPrepackagedRulesSchemaDecoded => { + return schema as AddPrepackagedRulesSchemaDecoded; + }; + return pipe(checked, fold(onLeft, onRight)); + }); +}; + /** * Retrieve and validate rules that were installed from Fleet as saved objects. */ @@ -60,11 +92,8 @@ export const getFleetInstalledRules = async ( client: RuleAssetSavedObjectsClient ): Promise => { const fleetResponse = await client.all(); - const fleetRules = fleetResponse.map( - // @ts-expect-error data is too loosely typed - (so) => so.attributes as AddPrepackagedRulesSchema - ); - return validateAllPrepackagedRules(fleetRules); + const fleetRules = fleetResponse.map((so) => so.attributes); + return validateAllRuleSavedObjects(fleetRules); }; export const getPrepackagedRules = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 38e93b2d7812c..a26a8d4c9cb9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -173,7 +173,7 @@ export interface IRuleAssetSOAttributes extends Record { export interface IRuleAssetSavedObject { type: string; id: string; - attributes: Array>; + attributes: IRuleAssetSOAttributes & SavedObjectAttributes; } export interface HapiReadableStream extends Readable {