From 1ab12e24c78d26552dc529795f18b6af56fdaf7c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 16 Dec 2021 13:31:45 -0500 Subject: [PATCH 1/6] Disable legacy rule and notify user to upgrade --- .../signals/signal_rule_alert_type.ts | 623 +++--------------- 1 file changed, 77 insertions(+), 546 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 4594ce212e0a9..89c415e131755 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -4,73 +4,25 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -/* eslint-disable complexity */ import { Logger } from 'src/core/server'; -import isEmpty from 'lodash/isEmpty'; -import * as t from 'io-ts'; -import { validateNonExact, parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; + +import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { SIGNALS_ID } from '@kbn/securitysolution-rules'; -import { DEFAULT_SEARCH_AFTER_PAGE_SIZE, SERVER_APP_ID } from '../../../../common/constants'; -import { isMlRule } from '../../../../common/machine_learning/helpers'; -import { - isThresholdRule, - isEqlRule, - isThreatMatchRule, - isQueryRule, -} from '../../../../common/detection_engine/utils'; +import { SERVER_APP_ID } from '../../../../common/constants'; import { SetupPlugins } from '../../../plugin'; -import { getInputIndex } from './get_input_output_index'; -import { SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types'; -import { - getListsClient, - getExceptions, - createSearchAfterReturnType, - checkPrivileges, - hasTimestampFields, - hasReadIndexPrivileges, - getRuleRangeTuples, - isMachineLearningParams, -} from './utils'; -import { siemRuleActionGroups } from './siem_rule_action_groups'; -import { - scheduleNotificationActions, - NotificationRuleTypeParams, -} from '../notifications/schedule_notification_actions'; -import { buildRuleMessageFactory } from './rule_messages'; -import { getNotificationResultsLink } from '../notifications/utils'; +import { SignalRuleAlertTypeDefinition } from './types'; import { TelemetryEventsSender } from '../../telemetry/sender'; -import { eqlExecutor } from './executors/eql'; -import { queryExecutor } from './executors/query'; -import { threatMatchExecutor } from './executors/threat_match'; -import { thresholdExecutor } from './executors/threshold'; -import { mlExecutor } from './executors/ml'; -import { - eqlRuleParams, - machineLearningRuleParams, - queryRuleParams, - threatRuleParams, - thresholdRuleParams, - ruleParams, - RuleParams, - savedQueryRuleParams, - CompleteRule, -} from '../schemas/rule_schemas'; -import { bulkCreateFactory } from './bulk_create_factory'; -import { wrapHitsFactory } from './wrap_hits_factory'; -import { wrapSequencesFactory } from './wrap_sequences_factory'; +import { ruleParams, RuleParams } from '../schemas/rule_schemas'; import { ConfigType } from '../../../config'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; import { injectReferences, extractReferences } from './saved_object_references'; -import { - IRuleExecutionLogClient, - RuleExecutionLogClient, - truncateMessageList, -} from '../rule_execution_log'; +import { IRuleExecutionLogClient, RuleExecutionLogClient } from '../rule_execution_log'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; import { IEventLogService } from '../../../../../event_log/server'; +import { siemRuleActionGroups } from './siem_rule_action_groups'; export const signalRulesAlertType = ({ logger, @@ -96,501 +48,80 @@ export const signalRulesAlertType = ({ indexNameOverride?: string; refreshOverride?: string; ruleExecutionLogClientOverride?: IRuleExecutionLogClient; -}): SignalRuleAlertTypeDefinition => { - const { alertMergeStrategy: mergeStrategy, alertIgnoreFields: ignoreFields } = config; - return { - id: SIGNALS_ID, - name: 'SIEM signal', - actionGroups: siemRuleActionGroups, - defaultActionGroupId: 'default', - useSavedObjectReferences: { - extractReferences: (params) => extractReferences({ logger, params }), - injectReferences: (params, savedObjectReferences) => - injectReferences({ logger, params, savedObjectReferences }), - }, - validate: { - params: { - validate: (object: unknown): RuleParams => { - const [validated, errors] = validateNonExact(object, ruleParams); - if (errors != null) { - throw new Error(errors); - } - if (validated == null) { - throw new Error('Validation of rule params failed'); - } - return validated; - }, +}): SignalRuleAlertTypeDefinition => ({ + id: SIGNALS_ID, + name: 'SIEM signal', + actionGroups: siemRuleActionGroups, + defaultActionGroupId: 'default', + useSavedObjectReferences: { + extractReferences: (params) => extractReferences({ logger, params }), + injectReferences: (params, savedObjectReferences) => + injectReferences({ logger, params, savedObjectReferences }), + }, + validate: { + params: { + validate: (object: unknown): RuleParams => { + const [validated, errors] = validateNonExact(object, ruleParams); + if (errors != null) { + throw new Error(errors); + } + if (validated == null) { + throw new Error('Validation of rule params failed'); + } + return validated; }, }, - producer: SERVER_APP_ID, - minimumLicenseRequired: 'basic', - isExportable: false, - async executor({ - previousStartedAt, - startedAt, - state, - alertId, - services, - params, - spaceId, - updatedBy: updatedByUser, - rule, - }) { - const { ruleId, maxSignals, meta, outputIndex, timestampOverride, type } = params; - - const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); - let hasError: boolean = false; - let result = createSearchAfterReturnType(); - - const ruleStatusClient = ruleExecutionLogClientOverride - ? ruleExecutionLogClientOverride - : new RuleExecutionLogClient({ - underlyingClient: config.ruleExecutionLog.underlyingClient, - savedObjectsClient: services.savedObjectsClient, - eventLogService, - logger, - }); - - const completeRule: CompleteRule = { - alertId, - ruleConfig: rule, - ruleParams: params, - }; + }, + producer: SERVER_APP_ID, + minimumLicenseRequired: 'basic', + isExportable: false, + async executor({ alertId, params, rule, services, spaceId, startedAt }) { + const { meta, outputIndex, ruleId } = params; + const { name: ruleName, ruleTypeId: ruleType, throttle } = rule; + + const ruleStatusClient = ruleExecutionLogClientOverride + ? ruleExecutionLogClientOverride + : new RuleExecutionLogClient({ + underlyingClient: config.ruleExecutionLog.underlyingClient, + savedObjectsClient: services.savedObjectsClient, + eventLogService, + logger, + }); - const { - actions, - name, - schedule: { interval }, - ruleTypeId, - } = completeRule.ruleConfig; + const message = + 'It looks like you forgot to disable the rule before upgrading. ' + + 'Please disable and reenable the rule to avoid further disruption of service.'; + logger.warn(message); + + await ruleStatusClient.logStatusChange({ + message, + newStatus: RuleExecutionStatus.failed, + ruleId: alertId, + ruleType, + ruleName, + spaceId, + }); - const refresh = refreshOverride ?? actions.length ? 'wait_for' : false; - const buildRuleMessage = buildRuleMessageFactory({ - id: alertId, + // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early + if (throttle != null) { + await scheduleThrottledNotificationActions({ + alertInstance: services.alertInstanceFactory(ruleId), + throttle: throttle ?? '', + startedAt, + id: ruleId, + kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) + ?.kibana_siem_app_url, + outputIndex, ruleId, - name, - index: indexNameOverride ?? outputIndex, - }); - - logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); - logger.debug(buildRuleMessage(`interval: ${interval}`)); - let wroteWarningStatus = false; - const basicLogArguments = { - spaceId, - ruleId: alertId, - ruleName: name, - ruleType: ruleTypeId, - }; - - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus['going to run'], - }); - - const notificationRuleParams: NotificationRuleTypeParams = { - ...params, - name, - id: alertId, - }; - - // check if rule has permissions to access given index pattern - // move this collection of lines into a function in utils - // so that we can use it in create rules route, bulk, etc. - try { - if (!isMachineLearningParams(params)) { - const index = params.index; - const hasTimestampOverride = timestampOverride != null && !isEmpty(timestampOverride); - const inputIndices = await getInputIndex({ - services, - version, - index, - experimentalFeatures, - }); - const privileges = await checkPrivileges(services, inputIndices); - - wroteWarningStatus = await hasReadIndexPrivileges({ - ...basicLogArguments, - privileges, - logger, - buildRuleMessage, - ruleStatusClient, - }); - - if (!wroteWarningStatus) { - const timestampFieldCaps = await services.scopedClusterClient.asCurrentUser.fieldCaps({ - index, - fields: hasTimestampOverride - ? ['@timestamp', timestampOverride as string] - : ['@timestamp'], - include_unmapped: true, - }); - wroteWarningStatus = await hasTimestampFields({ - ...basicLogArguments, - timestampField: hasTimestampOverride ? (timestampOverride as string) : '@timestamp', - timestampFieldCapsResponse: timestampFieldCaps, - inputIndices, - ruleStatusClient, - logger, - buildRuleMessage, - }); - } - } - } catch (exc) { - const errorMessage = buildRuleMessage(`Check privileges failed to execute ${exc}`); - logger.error(errorMessage); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - message: errorMessage, - newStatus: RuleExecutionStatus['partial failure'], - }); - wroteWarningStatus = true; - } - const { tuples, remainingGap } = getRuleRangeTuples({ + signals: [], + esClient: services.scopedClusterClient.asCurrentUser, + notificationRuleParams: { + ...params, + id: alertId, + name: ruleName, + }, logger, - previousStartedAt, - from: params.from, - to: params.to, - interval, - maxSignals, - buildRuleMessage, - startedAt, }); - - if (remainingGap.asMilliseconds() > 0) { - const gapString = remainingGap.humanize(); - const gapMessage = buildRuleMessage( - `${gapString} (${remainingGap.asMilliseconds()}ms) were not queried between this rule execution and the last execution, so signals may have been missed.`, - 'Consider increasing your look behind time or adding more Kibana instances.' - ); - logger.warn(gapMessage); - hasError = true; - - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.failed, - message: gapMessage, - metrics: { executionGap: remainingGap }, - }); - } - try { - const { listClient, exceptionsClient } = getListsClient({ - services, - updatedByUser, - spaceId, - lists, - savedObjectClient: services.savedObjectsClient, - }); - - const exceptionItems = await getExceptions({ - client: exceptionsClient, - lists: params.exceptionsList ?? [], - }); - - const bulkCreate = bulkCreateFactory( - logger, - services.scopedClusterClient.asCurrentUser, - buildRuleMessage, - refresh, - indexNameOverride - ); - - const wrapHits = wrapHitsFactory({ - completeRule, - signalsIndex: indexNameOverride ?? params.outputIndex, - mergeStrategy, - ignoreFields, - }); - - const wrapSequences = wrapSequencesFactory({ - completeRule, - signalsIndex: params.outputIndex, - mergeStrategy, - ignoreFields, - }); - - if (isMlRule(type)) { - const mlRuleCompleteRule = asTypeSpecificCompleteRule( - completeRule, - machineLearningRuleParams - ); - for (const tuple of tuples) { - result = await mlExecutor({ - completeRule: mlRuleCompleteRule, - tuple, - ml, - listClient, - exceptionItems, - services, - logger, - buildRuleMessage, - bulkCreate, - wrapHits, - }); - } - } else if (isThresholdRule(type)) { - const thresholdCompleteRule = asTypeSpecificCompleteRule( - completeRule, - thresholdRuleParams - ); - for (const tuple of tuples) { - result = await thresholdExecutor({ - completeRule: thresholdCompleteRule, - tuple, - exceptionItems, - experimentalFeatures, - services, - version, - logger, - buildRuleMessage, - startedAt, - state: state as ThresholdAlertState, - bulkCreate, - wrapHits, - }); - } - } else if (isThreatMatchRule(type)) { - const threatCompleteRule = asTypeSpecificCompleteRule(completeRule, threatRuleParams); - for (const tuple of tuples) { - result = await threatMatchExecutor({ - completeRule: threatCompleteRule, - tuple, - listClient, - exceptionItems, - experimentalFeatures, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); - } - } else if (isQueryRule(type)) { - const queryCompleteRule = validateQueryRuleTypes(completeRule); - for (const tuple of tuples) { - result = await queryExecutor({ - completeRule: queryCompleteRule, - tuple, - listClient, - exceptionItems, - experimentalFeatures, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); - } - } else if (isEqlRule(type)) { - const eqlCompleteRule = asTypeSpecificCompleteRule(completeRule, eqlRuleParams); - for (const tuple of tuples) { - result = await eqlExecutor({ - completeRule: eqlCompleteRule, - tuple, - exceptionItems, - experimentalFeatures, - services, - version, - searchAfterSize, - bulkCreate, - logger, - wrapHits, - wrapSequences, - }); - } - } - - if (result.warningMessages.length) { - const warningMessage = buildRuleMessage( - truncateMessageList(result.warningMessages).join() - ); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus['partial failure'], - message: warningMessage, - }); - } - - if (result.success) { - if (actions.length) { - const fromInMs = parseScheduleDates(`now-${interval}`)?.format('x'); - const toInMs = parseScheduleDates('now')?.format('x'); - const resultsLink = getNotificationResultsLink({ - from: fromInMs, - to: toInMs, - id: alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - }); - - logger.debug( - buildRuleMessage(`Found ${result.createdSignalsCount} signals for notification.`) - ); - - if (completeRule.ruleConfig.throttle != null) { - await scheduleThrottledNotificationActions({ - alertInstance: services.alertInstanceFactory(alertId), - throttle: completeRule.ruleConfig.throttle, - startedAt, - id: alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - outputIndex: indexNameOverride ?? outputIndex, - ruleId, - signals: result.createdSignals, - esClient: services.scopedClusterClient.asCurrentUser, - notificationRuleParams, - logger, - }); - } else if (result.createdSignalsCount) { - const alertInstance = services.alertInstanceFactory(alertId); - scheduleNotificationActions({ - alertInstance, - signalsCount: result.createdSignalsCount, - signals: result.createdSignals, - resultsLink, - ruleParams: notificationRuleParams, - }); - } - } - - logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); - logger.debug( - buildRuleMessage( - `[+] Finished indexing ${result.createdSignalsCount} signals into ${ - indexNameOverride ?? outputIndex - }` - ) - ); - if (!hasError && !wroteWarningStatus && !result.warning) { - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.succeeded, - message: 'succeeded', - metrics: { - indexingDurations: result.bulkCreateTimes, - searchDurations: result.searchAfterTimes, - lastLookBackDate: result.lastLookBackDate?.toISOString(), - }, - }); - } - - logger.debug( - buildRuleMessage( - `[+] Finished indexing ${result.createdSignalsCount} ${ - !isEmpty(tuples) - ? `signals searched between date ranges ${JSON.stringify(tuples, null, 2)}` - : '' - }` - ) - ); - } else { - // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early - if (completeRule.ruleConfig.throttle != null) { - await scheduleThrottledNotificationActions({ - alertInstance: services.alertInstanceFactory(alertId), - throttle: completeRule.ruleConfig.throttle ?? '', - startedAt, - id: completeRule.alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - outputIndex, - ruleId, - signals: result.createdSignals, - esClient: services.scopedClusterClient.asCurrentUser, - notificationRuleParams, - logger, - }); - } - const errorMessage = buildRuleMessage( - 'Bulk Indexing of signals failed:', - truncateMessageList(result.errors).join() - ); - logger.error(errorMessage); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.failed, - message: errorMessage, - metrics: { - indexingDurations: result.bulkCreateTimes, - searchDurations: result.searchAfterTimes, - lastLookBackDate: result.lastLookBackDate?.toISOString(), - }, - }); - } - } catch (error) { - // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early - if (completeRule.ruleConfig.throttle != null) { - await scheduleThrottledNotificationActions({ - alertInstance: services.alertInstanceFactory(alertId), - throttle: completeRule.ruleConfig.throttle ?? '', - startedAt, - id: completeRule.alertId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - outputIndex, - ruleId, - signals: result.createdSignals, - esClient: services.scopedClusterClient.asCurrentUser, - notificationRuleParams, - logger, - }); - } - const errorMessage = error.message ?? '(no error message given)'; - const message = buildRuleMessage( - 'An error occurred during rule execution:', - `message: "${errorMessage}"` - ); - - logger.error(message); - await ruleStatusClient.logStatusChange({ - ...basicLogArguments, - newStatus: RuleExecutionStatus.failed, - message, - metrics: { - indexingDurations: result.bulkCreateTimes, - searchDurations: result.searchAfterTimes, - lastLookBackDate: result.lastLookBackDate?.toISOString(), - }, - }); - } - }, - }; -}; - -const validateQueryRuleTypes = (completeRule: CompleteRule) => { - if (completeRule.ruleParams.type === 'query') { - return asTypeSpecificCompleteRule(completeRule, queryRuleParams); - } else { - return asTypeSpecificCompleteRule(completeRule, savedQueryRuleParams); - } -}; - -/** - * This function takes a generic rule SavedObject and a type-specific schema for the rule params - * and validates the SavedObject params against the schema. If they validate, it returns a SavedObject - * where the params have been replaced with the validated params. This eliminates the need for logic that - * checks if the required type specific fields actually exist on the SO and prevents rule executors from - * accessing fields that only exist on other rule types. - * - * @param completeRule rule typed as an object with all fields from all different rule types - * @param schema io-ts schema for the specific rule type the SavedObject claims to be - */ -export const asTypeSpecificCompleteRule = ( - completeRule: CompleteRule, - schema: T -) => { - const [validated, errors] = validateNonExact(completeRule.ruleParams, schema); - if (validated == null || errors != null) { - throw new Error(`Rule attempted to execute with invalid params: ${errors}`); - } - return { - ...completeRule, - ruleParams: validated, - }; -}; + } + }, +}); From fb3655a4ae6e181c9697d5b0c9a09b5fb62d6e0c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 20 Dec 2021 11:48:38 -0500 Subject: [PATCH 2/6] Ensure rules are disabled on upgrade --- .../server/saved_objects/migrations.ts | 1 + .../signals/signal_rule_alert_type.test.ts | 599 ------------------ .../signals/signal_rule_alert_type.ts | 127 ---- .../security_solution/server/plugin.ts | 29 - 4 files changed, 1 insertion(+), 755 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index 6736fd3573adb..ac89f219e0272 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -662,6 +662,7 @@ function addRACRuleTypes( attributes: { ...doc.attributes, alertTypeId: ruleTypeMappings[ruleType], + enabled: false, params: { ...doc.attributes.params, outputIndex: '', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts deleted file mode 100644 index 10a7f38fbf389..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import moment from 'moment'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { loggingSystemMock } from 'src/core/server/mocks'; -import { getAlertMock } from '../routes/__mocks__/request_responses'; -import { signalRulesAlertType } from './signal_rule_alert_type'; -import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; -import { - getListsClient, - getExceptions, - checkPrivileges, - createSearchAfterReturnType, -} from './utils'; -import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; -import { RuleExecutorOptions, SearchAfterAndBulkCreateReturnType } from './types'; -import { RuleAlertType } from '../rules/types'; -import { listMock } from '../../../../../lists/server/mocks'; -import { getListClientMock } from '../../../../../lists/server/services/lists/list_client.mock'; -import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock'; -import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import type { TransportResult } from '@elastic/elasticsearch'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; -import { queryExecutor } from './executors/query'; -import { mlExecutor } from './executors/ml'; -import { getMlRuleParams, getQueryRuleParams } from '../schemas/rule_schemas.mock'; -import { errors } from '@elastic/elasticsearch'; -import { allowedExperimentalValues } from '../../../../common/experimental_features'; -import { scheduleNotificationActions } from '../notifications/schedule_notification_actions'; -import { ruleExecutionLogClientMock } from '../rule_execution_log/__mocks__/rule_execution_log_client'; -import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; -import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; -import { eventLogServiceMock } from '../../../../../event_log/server/mocks'; -import { createMockConfig } from '../routes/__mocks__'; - -jest.mock('./utils', () => { - const original = jest.requireActual('./utils'); - return { - ...original, - getListsClient: jest.fn(), - getExceptions: jest.fn(), - sortExceptionItems: jest.fn(), - checkPrivileges: jest.fn(), - }; -}); -jest.mock('../notifications/schedule_notification_actions'); -jest.mock('./executors/query'); -jest.mock('./executors/ml'); -jest.mock('@kbn/securitysolution-io-ts-utils', () => { - const original = jest.requireActual('@kbn/securitysolution-io-ts-utils'); - return { - ...original, - parseScheduleDates: jest.fn(), - }; -}); -jest.mock('../notifications/schedule_throttle_notification_actions'); -const mockRuleExecutionLogClient = ruleExecutionLogClientMock.create(); - -jest.mock('../rule_execution_log/rule_execution_log_client', () => ({ - RuleExecutionLogClient: jest.fn().mockImplementation(() => mockRuleExecutionLogClient), -})); - -const getPayload = ( - ruleAlert: RuleAlertType, - services: AlertServicesMock -): RuleExecutorOptions => ({ - alertId: ruleAlert.id, - services, - name: ruleAlert.name, - tags: ruleAlert.tags, - params: { - ...ruleAlert.params, - }, - state: {}, - spaceId: '', - startedAt: new Date('2019-12-13T16:50:33.400Z'), - previousStartedAt: new Date('2019-12-13T16:40:33.400Z'), - createdBy: 'elastic', - updatedBy: 'elastic', - rule: { - name: ruleAlert.name, - tags: ruleAlert.tags, - consumer: 'foo', - producer: 'foo', - ruleTypeId: 'ruleType', - ruleTypeName: 'Name of rule', - enabled: true, - schedule: { - interval: '5m', - }, - actions: ruleAlert.actions, - createdBy: 'elastic', - updatedBy: 'elastic', - createdAt: new Date('2019-12-13T16:50:33.400Z'), - updatedAt: new Date('2019-12-13T16:50:33.400Z'), - throttle: null, - notifyWhen: null, - }, -}); - -// Deprecated -describe.skip('signal_rule_alert_type', () => { - const version = '8.0.0'; - const jobsSummaryMock = jest.fn(); - const mlMock = { - mlClient: { - callAsInternalUser: jest.fn(), - close: jest.fn(), - asScoped: jest.fn(), - }, - jobServiceProvider: jest.fn().mockReturnValue({ - jobsSummary: jobsSummaryMock, - }), - anomalyDetectorsProvider: jest.fn(), - mlSystemProvider: jest.fn(), - modulesProvider: jest.fn(), - resultsServiceProvider: jest.fn(), - alertingServiceProvider: jest.fn(), - }; - let payload: jest.Mocked; - let alert: ReturnType; - let logger: ReturnType; - let alertServices: AlertServicesMock; - let eventLogService: ReturnType; - - beforeEach(() => { - alertServices = alertsMock.createAlertServices(); - logger = loggingSystemMock.createLogger(); - eventLogService = eventLogServiceMock.create(); - (getListsClient as jest.Mock).mockReturnValue({ - listClient: getListClientMock(), - exceptionsClient: getExceptionListClientMock(), - }); - (getExceptions as jest.Mock).mockReturnValue([getExceptionListItemSchemaMock()]); - (checkPrivileges as jest.Mock).mockImplementation(async (_, indices) => { - return { - index: indices.reduce( - (acc: { index: { [x: string]: { read: boolean } } }, index: string) => { - return { - [index]: { - read: true, - }, - ...acc, - }; - }, - {} - ), - }; - }); - const executorReturnValue = createSearchAfterReturnType({ - createdSignalsCount: 10, - }); - (queryExecutor as jest.Mock).mockClear(); - (queryExecutor as jest.Mock).mockResolvedValue(executorReturnValue); - (mlExecutor as jest.Mock).mockClear(); - (mlExecutor as jest.Mock).mockResolvedValue(executorReturnValue); - (parseScheduleDates as jest.Mock).mockReturnValue(moment(100)); - const value: Partial> = { - statusCode: 200, - body: { - indices: ['index1', 'index2', 'index3', 'index4'], - fields: { - '@timestamp': { - // @ts-expect-error not full interface - date: { - indices: ['index1', 'index2', 'index3', 'index4'], - searchable: true, - aggregatable: false, - }, - }, - }, - }, - }; - alertServices.scopedClusterClient.asCurrentUser.fieldCaps.mockResolvedValue( - value as TransportResult - ); - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - - alert = signalRulesAlertType({ - experimentalFeatures: allowedExperimentalValues, - logger, - eventsTelemetry: undefined, - version, - ml: mlMock, - lists: listMock.createSetup(), - config: createMockConfig(), - eventLogService, - }); - - mockRuleExecutionLogClient.logStatusChange.mockClear(); - (scheduleThrottledNotificationActions as jest.Mock).mockClear(); - }); - - describe('executor', () => { - it('should log success status if signals were created', async () => { - payload.previousStartedAt = null; - await alert.executor(payload); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( - expect.objectContaining({ - newStatus: RuleExecutionStatus.succeeded, - }) - ); - }); - - it('should warn about the gap between runs if gap is very large', async () => { - payload.previousStartedAt = moment(payload.startedAt).subtract(100, 'm').toDate(); - await alert.executor(payload); - expect(logger.warn).toHaveBeenCalled(); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - newStatus: RuleExecutionStatus['going to run'], - }) - ); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - newStatus: RuleExecutionStatus.failed, - metrics: { - executionGap: expect.any(Object), - }, - }) - ); - }); - - it('should set a warning for when rules cannot read ALL provided indices', async () => { - (checkPrivileges as jest.Mock).mockResolvedValueOnce({ - username: 'elastic', - has_all_requested: false, - cluster: {}, - index: { - 'myfa*': { - read: true, - }, - 'anotherindex*': { - read: true, - }, - 'some*': { - read: false, - }, - }, - application: {}, - }); - const newRuleAlert = getAlertMock(false, getQueryRuleParams()); - newRuleAlert.params.index = ['some*', 'myfa*', 'anotherindex*']; - payload = getPayload(newRuleAlert, alertServices) as jest.Mocked; - - await alert.executor(payload); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - newStatus: RuleExecutionStatus['partial failure'], - message: - 'This rule may not have the required read privileges to the following indices/index patterns: ["some*"]', - }) - ); - }); - - it('should set a failure status for when rules cannot read ANY provided indices', async () => { - (checkPrivileges as jest.Mock).mockResolvedValueOnce({ - username: 'elastic', - has_all_requested: false, - cluster: {}, - index: { - 'myfa*': { - read: false, - }, - 'some*': { - read: false, - }, - }, - application: {}, - }); - const newRuleAlert = getAlertMock(false, getQueryRuleParams()); - newRuleAlert.params.index = ['some*', 'myfa*', 'anotherindex*']; - payload = getPayload(newRuleAlert, alertServices) as jest.Mocked; - - await alert.executor(payload); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - newStatus: RuleExecutionStatus['partial failure'], - message: - 'This rule may not have the required read privileges to the following indices/index patterns: ["myfa*","some*"]', - }) - ); - }); - - it('should NOT warn about the gap between runs if gap small', async () => { - payload.previousStartedAt = moment().subtract(10, 'm').toDate(); - await alert.executor(payload); - expect(logger.warn).toHaveBeenCalledTimes(0); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenCalledTimes(2); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - newStatus: RuleExecutionStatus['going to run'], - }) - ); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - newStatus: RuleExecutionStatus.succeeded, - }) - ); - }); - - it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - - await alert.executor(payload); - }); - - it('should resolve results_link when meta is an empty object to use "/app/security"', async () => { - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - ruleAlert.params.meta = {}; - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - const modifiedPayload = getPayload( - ruleAlert, - alertServices - ) as jest.Mocked; - - await alert.executor(modifiedPayload); - - expect(scheduleNotificationActions).toHaveBeenCalledWith( - expect.objectContaining({ - resultsLink: `/app/security/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`, - }) - ); - }); - - it('should resolve results_link when meta is undefined use "/app/security"', async () => { - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - delete ruleAlert.params.meta; - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - const modifiedPayload = getPayload( - ruleAlert, - alertServices - ) as jest.Mocked; - - await alert.executor(modifiedPayload); - - expect(scheduleNotificationActions).toHaveBeenCalledWith( - expect.objectContaining({ - resultsLink: `/app/security/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`, - }) - ); - }); - - it('should resolve results_link with a custom link', async () => { - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - ruleAlert.params.meta = { kibana_siem_app_url: 'http://localhost' }; - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - const modifiedPayload = getPayload( - ruleAlert, - alertServices - ) as jest.Mocked; - - await alert.executor(modifiedPayload); - - expect(scheduleNotificationActions).toHaveBeenCalledWith( - expect.objectContaining({ - resultsLink: `http://localhost/detections/rules/id/${ruleAlert.id}?timerange=(global:(linkTo:!(timeline),timerange:(from:100,kind:absolute,to:100)),timeline:(linkTo:!(global),timerange:(from:100,kind:absolute,to:100)))`, - }) - ); - }); - - describe('ML rule', () => { - it('should not call checkPrivileges if ML rule', async () => { - const ruleAlert = getAlertMock(false, getMlRuleParams()); - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - payload = getPayload(ruleAlert, alertServices) as jest.Mocked; - payload.previousStartedAt = null; - (checkPrivileges as jest.Mock).mockClear(); - - await alert.executor(payload); - expect(checkPrivileges).toHaveBeenCalledTimes(0); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( - expect.objectContaining({ - newStatus: RuleExecutionStatus.succeeded, - }) - ); - }); - }); - }); - - describe('should catch error', () => { - it('when bulk indexing failed', async () => { - const result: SearchAfterAndBulkCreateReturnType = { - success: false, - warning: false, - searchAfterTimes: [], - bulkCreateTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - createdSignals: [], - warningMessages: [], - errors: ['Error that bubbled up.'], - }; - (queryExecutor as jest.Mock).mockResolvedValue(result); - await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain( - 'Bulk Indexing of signals failed: Error that bubbled up. name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' - ); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( - expect.objectContaining({ - newStatus: RuleExecutionStatus.failed, - }) - ); - }); - - it('when error was thrown', async () => { - (queryExecutor as jest.Mock).mockRejectedValue({}); - await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain('An error occurred during rule execution'); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( - expect.objectContaining({ - newStatus: RuleExecutionStatus.failed, - }) - ); - }); - - it('and log failure with the default message', async () => { - (queryExecutor as jest.Mock).mockReturnValue( - elasticsearchClientMock.createErrorTransportRequestPromise( - new errors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: 400, - body: { error: { type: 'some_error_type' } }, - }) - ) - ) - ); - await alert.executor(payload); - expect(logger.error).toHaveBeenCalled(); - expect(logger.error.mock.calls[0][0]).toContain('An error occurred during rule execution'); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( - expect.objectContaining({ - newStatus: RuleExecutionStatus.failed, - }) - ); - }); - - it('should call scheduleThrottledNotificationActions if result is false to prevent the throttle from being reset', async () => { - const result: SearchAfterAndBulkCreateReturnType = { - success: false, - warning: false, - searchAfterTimes: [], - bulkCreateTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - createdSignals: [], - warningMessages: [], - errors: ['Error that bubbled up.'], - }; - (queryExecutor as jest.Mock).mockResolvedValue(result); - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - ruleAlert.throttle = '1h'; - const payLoadWithThrottle = getPayload( - ruleAlert, - alertServices - ) as jest.Mocked; - payLoadWithThrottle.rule.throttle = '1h'; - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - await alert.executor(payLoadWithThrottle); - expect(scheduleThrottledNotificationActions).toHaveBeenCalledTimes(1); - }); - - it('should NOT call scheduleThrottledNotificationActions if result is false and the throttle is not set', async () => { - const result: SearchAfterAndBulkCreateReturnType = { - success: false, - warning: false, - searchAfterTimes: [], - bulkCreateTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - createdSignals: [], - warningMessages: [], - errors: ['Error that bubbled up.'], - }; - (queryExecutor as jest.Mock).mockResolvedValue(result); - await alert.executor(payload); - expect(scheduleThrottledNotificationActions).toHaveBeenCalledTimes(0); - }); - - it('should call scheduleThrottledNotificationActions if an error was thrown to prevent the throttle from being reset', async () => { - (queryExecutor as jest.Mock).mockRejectedValue({}); - const ruleAlert = getAlertMock(false, getQueryRuleParams()); - ruleAlert.throttle = '1h'; - const payLoadWithThrottle = getPayload( - ruleAlert, - alertServices - ) as jest.Mocked; - payLoadWithThrottle.rule.throttle = '1h'; - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - await alert.executor(payLoadWithThrottle); - expect(scheduleThrottledNotificationActions).toHaveBeenCalledTimes(1); - }); - - it('should NOT call scheduleThrottledNotificationActions if an error was thrown to prevent the throttle from being reset if throttle is not defined', async () => { - const result: SearchAfterAndBulkCreateReturnType = { - success: false, - warning: false, - searchAfterTimes: [], - bulkCreateTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - createdSignals: [], - warningMessages: [], - errors: ['Error that bubbled up.'], - }; - (queryExecutor as jest.Mock).mockRejectedValue(result); - await alert.executor(payload); - expect(scheduleThrottledNotificationActions).toHaveBeenCalledTimes(0); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts deleted file mode 100644 index 89c415e131755..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from 'src/core/server'; - -import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; -import { SIGNALS_ID } from '@kbn/securitysolution-rules'; - -import { SERVER_APP_ID } from '../../../../common/constants'; -import { SetupPlugins } from '../../../plugin'; -import { SignalRuleAlertTypeDefinition } from './types'; -import { TelemetryEventsSender } from '../../telemetry/sender'; -import { ruleParams, RuleParams } from '../schemas/rule_schemas'; -import { ConfigType } from '../../../config'; -import { ExperimentalFeatures } from '../../../../common/experimental_features'; -import { injectReferences, extractReferences } from './saved_object_references'; -import { IRuleExecutionLogClient, RuleExecutionLogClient } from '../rule_execution_log'; -import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; -import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; -import { IEventLogService } from '../../../../../event_log/server'; -import { siemRuleActionGroups } from './siem_rule_action_groups'; - -export const signalRulesAlertType = ({ - logger, - eventsTelemetry, - experimentalFeatures, - version, - ml, - lists, - config, - eventLogService, - indexNameOverride, - ruleExecutionLogClientOverride, - refreshOverride, -}: { - logger: Logger; - eventsTelemetry: TelemetryEventsSender | undefined; - experimentalFeatures: ExperimentalFeatures; - version: string; - ml: SetupPlugins['ml'] | undefined; - lists: SetupPlugins['lists'] | undefined; - config: ConfigType; - eventLogService: IEventLogService; - indexNameOverride?: string; - refreshOverride?: string; - ruleExecutionLogClientOverride?: IRuleExecutionLogClient; -}): SignalRuleAlertTypeDefinition => ({ - id: SIGNALS_ID, - name: 'SIEM signal', - actionGroups: siemRuleActionGroups, - defaultActionGroupId: 'default', - useSavedObjectReferences: { - extractReferences: (params) => extractReferences({ logger, params }), - injectReferences: (params, savedObjectReferences) => - injectReferences({ logger, params, savedObjectReferences }), - }, - validate: { - params: { - validate: (object: unknown): RuleParams => { - const [validated, errors] = validateNonExact(object, ruleParams); - if (errors != null) { - throw new Error(errors); - } - if (validated == null) { - throw new Error('Validation of rule params failed'); - } - return validated; - }, - }, - }, - producer: SERVER_APP_ID, - minimumLicenseRequired: 'basic', - isExportable: false, - async executor({ alertId, params, rule, services, spaceId, startedAt }) { - const { meta, outputIndex, ruleId } = params; - const { name: ruleName, ruleTypeId: ruleType, throttle } = rule; - - const ruleStatusClient = ruleExecutionLogClientOverride - ? ruleExecutionLogClientOverride - : new RuleExecutionLogClient({ - underlyingClient: config.ruleExecutionLog.underlyingClient, - savedObjectsClient: services.savedObjectsClient, - eventLogService, - logger, - }); - - const message = - 'It looks like you forgot to disable the rule before upgrading. ' + - 'Please disable and reenable the rule to avoid further disruption of service.'; - logger.warn(message); - - await ruleStatusClient.logStatusChange({ - message, - newStatus: RuleExecutionStatus.failed, - ruleId: alertId, - ruleType, - ruleName, - spaceId, - }); - - // NOTE: Since this is throttled we have to call it even on an error condition, otherwise it will "reset" the throttle and fire early - if (throttle != null) { - await scheduleThrottledNotificationActions({ - alertInstance: services.alertInstanceFactory(ruleId), - throttle: throttle ?? '', - startedAt, - id: ruleId, - kibanaSiemAppUrl: (meta as { kibana_siem_app_url?: string } | undefined) - ?.kibana_siem_app_url, - outputIndex, - ruleId, - signals: [], - esClient: services.scopedClusterClient.asCurrentUser, - notificationRuleParams: { - ...params, - id: alertId, - name: ruleName, - }, - logger, - }); - } - }, -}); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 98d63e1917f73..89748ddfb6440 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -37,8 +37,6 @@ import { createThresholdAlertType, } from './lib/detection_engine/rule_types'; import { initRoutes } from './routes'; -import { isAlertExecutor } from './lib/detection_engine/signals/types'; -import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule_alert_type'; import { ManifestTask } from './endpoint/lib/artifacts'; import { CheckMetadataTransformsTask } from './endpoint/lib/metadata'; import { initSavedObjects } from './saved_objects'; @@ -73,10 +71,6 @@ import { registerEventLogProvider } from './lib/detection_engine/rule_execution_ import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features'; import { EndpointMetadataService } from './endpoint/services/metadata'; import { CreateRuleOptions } from './lib/detection_engine/rule_types/types'; -// eslint-disable-next-line no-restricted-imports -import { legacyRulesNotificationAlertType } from './lib/detection_engine/notifications/legacy_rules_notification_alert_type'; -// eslint-disable-next-line no-restricted-imports -import { legacyIsNotificationAlertExecutor } from './lib/detection_engine/notifications/legacy_types'; import { createSecurityRuleTypeWrapper } from './lib/detection_engine/rule_types/create_security_rule_type_wrapper'; import { RequestContextFactory } from './request_context_factory'; @@ -280,29 +274,6 @@ export class Plugin implements ISecuritySolutionPlugin { plugins.features.registerKibanaFeature(getKibanaPrivilegesFeaturePrivileges(ruleTypes)); plugins.features.registerKibanaFeature(getCasesKibanaFeature()); - // Continue to register legacy rules against alerting client exposed through rule-registry - if (plugins.alerting != null) { - const signalRuleType = signalRulesAlertType({ - logger, - eventsTelemetry: this.telemetryEventsSender, - version: pluginContext.env.packageInfo.version, - ml: plugins.ml, - lists: plugins.lists, - config, - experimentalFeatures, - eventLogService, - }); - const ruleNotificationType = legacyRulesNotificationAlertType({ logger }); - - if (isAlertExecutor(signalRuleType)) { - plugins.alerting.registerType(signalRuleType); - } - - if (legacyIsNotificationAlertExecutor(ruleNotificationType)) { - plugins.alerting.registerType(ruleNotificationType); - } - } - const exceptionListsSetupEnabled = () => { return plugins.taskManager && plugins.lists; }; From 021ec0fac4accc4dda7a742b3ea8cfdc3e193eb3 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 20 Dec 2021 18:16:30 -0500 Subject: [PATCH 3/6] Fix dupe detection on upgrade --- .../server/utils/create_persistence_rule_type_wrapper.ts | 7 +++++-- .../rule_registry/server/utils/persistence_types.ts | 3 +++ .../rule_types/factories/bulk_create_factory.ts | 3 ++- .../rule_types/factories/wrap_hits_factory.ts | 8 ++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 2d914e5e0945e..172a8663ced3c 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -46,7 +46,9 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper body: { query: { ids: { - values: alertChunk.map((alert) => alert._id), + values: alertChunk + .flatMap((alert) => [alert._id, alert._meta?.legacyId]) + .filter((item) => item != null) as string[], }, }, aggs: { @@ -81,8 +83,9 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper } const augmentedAlerts = filteredAlerts.map((alert) => { + const { _meta, ...rest } = alert; return { - ...alert, + ...rest, _source: { [VERSION]: ruleDataClient.kibanaVersion, ...commonRuleFields, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 61136b432a552..5819d09f323f2 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -20,6 +20,9 @@ import { IRuleDataClient } from '../rule_data_client'; export type PersistenceAlertService = ( alerts: Array<{ _id: string; + _meta?: { + legacyId: string; + }; _source: T; }>, refresh: boolean | 'wait_for' diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index 07b93f04e965f..c8578499db82f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -30,7 +30,7 @@ export const bulkCreateFactory = refreshForBulkCreate: RefreshTypes ) => async >( - wrappedDocs: Array> + wrappedDocs: Array & { _meta: { legacyId: string } }> ): Promise> => { if (wrappedDocs.length === 0) { return { @@ -47,6 +47,7 @@ export const bulkCreateFactory = const { createdAlerts } = await alertWithPersistence( wrappedDocs.map((doc) => ({ _id: doc._id, + _meta: doc._meta, // `fields` should have already been merged into `doc._source` _source: doc._source, })), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 81a4af31881fb..7ce39320af2e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -37,6 +37,14 @@ export const wrapHitsFactory = return { _id: id, _index: '', + _meta: { + legacyId: generateId( + event._index, + event._id, + String(event._version), + completeRule.alertId + ), + }, _source: { ...buildBulkBody( spaceId, From c5853fcb16cb2115e6a188082351fd01708d7978 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 20 Dec 2021 18:21:10 -0500 Subject: [PATCH 4/6] Revert "Fix dupe detection on upgrade" This reverts commit 021ec0fac4accc4dda7a742b3ea8cfdc3e193eb3. --- .../server/utils/create_persistence_rule_type_wrapper.ts | 7 ++----- .../rule_registry/server/utils/persistence_types.ts | 3 --- .../rule_types/factories/bulk_create_factory.ts | 3 +-- .../rule_types/factories/wrap_hits_factory.ts | 8 -------- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 172a8663ced3c..2d914e5e0945e 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -46,9 +46,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper body: { query: { ids: { - values: alertChunk - .flatMap((alert) => [alert._id, alert._meta?.legacyId]) - .filter((item) => item != null) as string[], + values: alertChunk.map((alert) => alert._id), }, }, aggs: { @@ -83,9 +81,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper } const augmentedAlerts = filteredAlerts.map((alert) => { - const { _meta, ...rest } = alert; return { - ...rest, + ...alert, _source: { [VERSION]: ruleDataClient.kibanaVersion, ...commonRuleFields, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 5819d09f323f2..61136b432a552 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -20,9 +20,6 @@ import { IRuleDataClient } from '../rule_data_client'; export type PersistenceAlertService = ( alerts: Array<{ _id: string; - _meta?: { - legacyId: string; - }; _source: T; }>, refresh: boolean | 'wait_for' diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index c8578499db82f..07b93f04e965f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -30,7 +30,7 @@ export const bulkCreateFactory = refreshForBulkCreate: RefreshTypes ) => async >( - wrappedDocs: Array & { _meta: { legacyId: string } }> + wrappedDocs: Array> ): Promise> => { if (wrappedDocs.length === 0) { return { @@ -47,7 +47,6 @@ export const bulkCreateFactory = const { createdAlerts } = await alertWithPersistence( wrappedDocs.map((doc) => ({ _id: doc._id, - _meta: doc._meta, // `fields` should have already been merged into `doc._source` _source: doc._source, })), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 7ce39320af2e9..81a4af31881fb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -37,14 +37,6 @@ export const wrapHitsFactory = return { _id: id, _index: '', - _meta: { - legacyId: generateId( - event._index, - event._id, - String(event._version), - completeRule.alertId - ), - }, _source: { ...buildBulkBody( spaceId, From feaa9235e9b4c6b0433fc6c3327047a9454a2e2f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 21 Dec 2021 14:09:31 -0500 Subject: [PATCH 5/6] Add legacy notification --- x-pack/plugins/security_solution/server/plugin.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index c6bc7920e9a00..a0d2bacba254a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -71,6 +71,10 @@ import { registerEventLogProvider } from './lib/detection_engine/rule_execution_ import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features'; import { EndpointMetadataService } from './endpoint/services/metadata'; import { CreateRuleOptions } from './lib/detection_engine/rule_types/types'; +// eslint-disable-next-line no-restricted-imports +import { legacyRulesNotificationAlertType } from './lib/detection_engine/notifications/legacy_rules_notification_alert_type'; +// eslint-disable-next-line no-restricted-imports +import { legacyIsNotificationAlertExecutor } from './lib/detection_engine/notifications/legacy_types'; import { createSecurityRuleTypeWrapper } from './lib/detection_engine/rule_types/create_security_rule_type_wrapper'; import { RequestContextFactory } from './request_context_factory'; @@ -275,6 +279,14 @@ export class Plugin implements ISecuritySolutionPlugin { plugins.features.registerKibanaFeature(getKibanaPrivilegesFeaturePrivileges(ruleTypes)); plugins.features.registerKibanaFeature(getCasesKibanaFeature()); + if (plugins.alerting != null) { + const ruleNotificationType = legacyRulesNotificationAlertType({ logger }); + + if (legacyIsNotificationAlertExecutor(ruleNotificationType)) { + plugins.alerting.registerType(ruleNotificationType); + } + } + const exceptionListsSetupEnabled = () => { return plugins.taskManager && plugins.lists; }; From 7d326d69cbce959213e94cbbe25ea4375e62d2c3 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 10 Jan 2022 10:28:50 -0500 Subject: [PATCH 6/6] Add tests for 8.0 security_solution rule migration --- .../server/saved_objects/migrations.test.ts | 32 +++++++++++++++++++ .../server/saved_objects/migrations.ts | 4 +-- .../spaces_only/tests/alerting/migrations.ts | 13 ++++++++ .../functional/es_archives/alerts/data.json | 1 + 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 08312d0be0419..5e2d8efedbcb3 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -11,6 +11,7 @@ import { RawRule } from '../types'; import { SavedObjectUnsanitizedDoc } from 'kibana/server'; import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; import { migrationMocks } from 'src/core/server/mocks'; +import { RuleType, ruleTypeMappings } from '@kbn/securitysolution-rules'; const migrationContext = migrationMocks.createContext(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); @@ -2056,6 +2057,37 @@ describe('successful migrations', () => { ); }); + test('doesnt change AAD rule params if not a siem.signals rule', () => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, isPreconfigured)['8.0.0']; + const alert = getMockData( + { params: { outputIndex: 'output-index', type: 'query' }, alertTypeId: 'not.siem.signals' }, + true + ); + expect(migration800(alert, migrationContext).attributes.alertTypeId).toEqual( + 'not.siem.signals' + ); + expect(migration800(alert, migrationContext).attributes.enabled).toEqual(true); + expect(migration800(alert, migrationContext).attributes.params.outputIndex).toEqual( + 'output-index' + ); + }); + + test.each(Object.keys(ruleTypeMappings) as RuleType[])( + 'Changes AAD rule params accordingly if rule is a siem.signals %p rule', + (ruleType) => { + const migration800 = getMigrations(encryptedSavedObjectsSetup, isPreconfigured)['8.0.0']; + const alert = getMockData( + { params: { outputIndex: 'output-index', type: ruleType }, alertTypeId: 'siem.signals' }, + true + ); + expect(migration800(alert, migrationContext).attributes.alertTypeId).toEqual( + ruleTypeMappings[ruleType] + ); + expect(migration800(alert, migrationContext).attributes.enabled).toEqual(false); + expect(migration800(alert, migrationContext).attributes.params.outputIndex).toEqual(''); + } + ); + describe('Metrics Inventory Threshold rule', () => { test('Migrates incorrect action group spelling', () => { const migration800 = getMigrations(encryptedSavedObjectsSetup, isPreconfigured)['8.0.0']; diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index ac89f219e0272..e664095e8c846 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -131,7 +131,7 @@ export function getMigrations( (doc: SavedObjectUnsanitizedDoc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations( addThreatIndicatorPathToThreatMatchRules, - addRACRuleTypes, + addSecuritySolutionAADRuleTypes, fixInventoryThresholdGroupId ) ); @@ -652,7 +652,7 @@ function setLegacyId(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsani }; } -function addRACRuleTypes( +function addSecuritySolutionAADRuleTypes( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc { const ruleType = doc.attributes.params.type; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts index 14c1679d3a1b2..baa9eeb3ce036 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts @@ -360,5 +360,18 @@ export default function createGetTests({ getService }: FtrProviderContext) { 'metrics.inventory_threshold.fired' ); }); + + it('8.0 migrates and disables pre-existing rules', async () => { + const response = await es.get<{ alert: RawRule }>( + { + index: '.kibana', + id: 'alert:38482620-ef1b-11eb-ad71-7de7959be71c', + }, + { meta: true } + ); + expect(response.statusCode).to.eql(200); + expect(response.body._source?.alert?.alertTypeId).to.be('siem.queryRule'); + expect(response.body._source?.alert?.enabled).to.be(false); + }); }); } diff --git a/x-pack/test/functional/es_archives/alerts/data.json b/x-pack/test/functional/es_archives/alerts/data.json index 249f14caf28f7..afa54208512f4 100644 --- a/x-pack/test/functional/es_archives/alerts/data.json +++ b/x-pack/test/functional/es_archives/alerts/data.json @@ -346,6 +346,7 @@ "consumer" : "alertsFixture", "params" : { "ruleId" : "4ec223b9-77fa-4895-8539-6b3e586a2858", + "type": "query", "exceptionsList" : [ { "id" : "endpoint_list",