From d86a9a297c061d76f58c08353531169cbf88cd9a Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Fri, 3 Jul 2020 09:57:44 +0200 Subject: [PATCH 01/30] WIP --- .../detection_engine/schemas/common/schemas.ts | 7 ++++++- .../common/detection_engine/types.ts | 1 + .../rules/select_rule_type/index.tsx | 18 +++++++++++++++++- .../rules/select_rule_type/translations.ts | 14 ++++++++++++++ .../detection_engine/rules/create/index.tsx | 2 ++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index 6e43bd645fd7b..b676b7be5a958 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -275,7 +275,12 @@ export type To = t.TypeOf; export const toOrUndefined = t.union([to, t.undefined]); export type ToOrUndefined = t.TypeOf; -export const type = t.keyof({ machine_learning: null, query: null, saved_query: null }); +export const type = t.keyof({ + machine_learning: null, + query: null, + saved_query: null, + treshold: null, +}); export type Type = t.TypeOf; export const typeOrUndefined = t.union([type, t.undefined]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/types.ts b/x-pack/plugins/security_solution/common/detection_engine/types.ts index 431d716a9f205..c28cac8a8d6fe 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/types.ts @@ -15,5 +15,6 @@ export const RuleTypeSchema = t.keyof({ query: null, saved_query: null, machine_learning: null, + treshold: null, }); export type RuleType = t.TypeOf; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx index 3dad53f532a5b..dd8d65aba43d1 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx @@ -22,6 +22,8 @@ import { FieldHook } from '../../../../shared_imports'; import { useKibana } from '../../../../common/lib/kibana'; import * as i18n from './translations'; +const isTresholdRule = (ruleType: RuleType) => ruleType === 'treshold'; + const MlCardDescription = ({ subscriptionUrl, hasValidLicense = false, @@ -75,6 +77,7 @@ export const SelectRuleType: React.FC = ({ ); const setMl = useCallback(() => setType('machine_learning'), [setType]); const setQuery = useCallback(() => setType('query'), [setType]); + const setTreshold = useCallback(() => setType('treshold'), [setType]); const mlCardDisabled = isReadOnly || !hasValidLicense || !isMlAdmin; const licensingUrl = useKibana().services.application.getUrlForApp('kibana', { path: '#/management/stack/license_management', @@ -97,7 +100,7 @@ export const SelectRuleType: React.FC = ({ selectable={{ isDisabled: isReadOnly, onClick: setQuery, - isSelected: !isMlRule(ruleType), + isSelected: !isMlRule(ruleType) && !isTresholdRule(ruleType), }} /> @@ -117,6 +120,19 @@ export const SelectRuleType: React.FC = ({ }} /> + + } + selectable={{ + isDisabled: isReadOnly, + onClick: setTreshold, + isSelected: isTresholdRule(ruleType), + }} + /> + ); diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts index 8b92d20616f7c..748d104fd4a8c 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts @@ -33,3 +33,17 @@ export const ML_TYPE_DESCRIPTION = i18n.translate( defaultMessage: 'Select ML job to detect anomalous activity.', } ); + +export const TRESHOLD_TYPE_TITLE = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.tresholdTypeTitle', + { + defaultMessage: 'Threshold', + } +); + +export const TRESHOLD_TYPE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.tresholdTypeDescription', + { + defaultMessage: 'Aggregate query results to detect when number of matches exceeds threshold.', + } +); diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx index 4be408039d6f6..0ae9f29fef13f 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx @@ -286,6 +286,8 @@ const CreateRulePageComponent: React.FC = () => { return null; } + console.error('aaa', stepsData); + return ( <> From ad2d17c762f810f9860f8b58e5561e7e18f7c247 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 6 Jul 2020 08:06:55 +0200 Subject: [PATCH 02/30] WIP --- .../schemas/common/schemas.ts | 13 ++- .../request/add_prepackaged_rules_schema.ts | 2 + .../schemas/request/create_rules_schema.ts | 3 + .../request/create_rules_type_dependents.ts | 17 ++++ .../schemas/request/import_rules_schema.ts | 2 + .../schemas/request/patch_rules_schema.ts | 2 + .../schemas/request/update_rules_schema.ts | 2 + .../schemas/response/rules_schema.ts | 18 ++++ .../common/detection_engine/types.ts | 2 +- .../rules/description_step/helpers.tsx | 8 ++ .../rules/description_step/translations.tsx | 7 ++ .../rules/select_rule_type/index.tsx | 14 +-- .../rules/select_rule_type/translations.ts | 8 +- .../rules/step_define_rule/index.tsx | 16 ++++ .../rules/threshold_input/index.tsx | 72 +++++++++++++++ .../detection_engine/rules/types.ts | 3 + .../detection_engine/rules/create/helpers.ts | 1 + .../pages/detection_engine/rules/helpers.tsx | 22 ++++- .../pages/detection_engine/rules/types.ts | 3 + .../routes/rules/create_rules_bulk_route.ts | 2 + .../routes/rules/create_rules_route.ts | 12 +++ .../routes/rules/find_rules_route.ts | 3 + .../routes/rules/import_rules_route.ts | 3 + .../routes/rules/patch_rules_bulk_route.ts | 2 + .../routes/rules/patch_rules_route.ts | 2 + .../routes/rules/update_rules_bulk_route.ts | 2 + .../routes/rules/update_rules_route.ts | 2 + .../detection_engine/routes/rules/utils.ts | 1 + .../detection_engine/rules/create_rules.ts | 2 + .../rules/install_prepacked_rules.ts | 2 + .../lib/detection_engine/rules/patch_rules.ts | 3 + .../lib/detection_engine/rules/types.ts | 4 + .../rules/update_prepacked_rules.ts | 2 + .../detection_engine/rules/update_rules.ts | 3 + .../lib/detection_engine/rules/utils.ts | 2 + .../signals/build_events_query.ts | 17 ++++ .../detection_engine/signals/build_rule.ts | 1 + .../signals/bulk_create_threshold_signals.ts | 90 +++++++++++++++++++ .../signals/find_threshold_signals.ts | 34 +++++++ .../detection_engine/signals/get_filter.ts | 1 + .../signals/signal_params_schema.ts | 1 + .../signals/signal_rule_alert_type.ts | 66 ++++++++++++++ .../signals/single_search_after.ts | 2 + .../server/lib/detection_engine/types.ts | 2 + 44 files changed, 460 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/alerts/components/rules/threshold_input/index.tsx create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index b676b7be5a958..77f9cb03c7db7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -279,7 +279,7 @@ export const type = t.keyof({ machine_learning: null, query: null, saved_query: null, - treshold: null, + threshold: null, }); export type Type = t.TypeOf; @@ -374,6 +374,17 @@ export type Threat = t.TypeOf; export const threatOrUndefined = t.union([threat, t.undefined]); export type ThreatOrUndefined = t.TypeOf; +export const threshold = t.exact( + t.type({ + field: t.string, + value: PositiveIntegerGreaterThanZero, + }) +); +export type Threshold = t.TypeOf; + +export const thresholdOrUndefined = t.union([threshold, t.undefined]); +export type thresholdOrUndefined = t.TypeOf; + export const created_at = IsoDateString; export const updated_at = IsoDateString; export const updated_by = t.string; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index bf96be5e688fa..aebc3361f6e49 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -25,6 +25,7 @@ import { To, type, Threat, + threshold, ThrottleOrNull, note, References, @@ -111,6 +112,7 @@ export const addPrepackagedRulesSchema = t.intersection([ tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode + threshold, // defaults to "undefined" if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index 0debe01e5a4d7..fc07ab1d912c4 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -28,6 +28,8 @@ import { To, type, Threat, + threshold, + Threshold, ThrottleOrNull, note, Version, @@ -106,6 +108,7 @@ export const createRulesSchema = t.intersection([ tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode + threshold, // defaults to "undefined" if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index aad2a2c4a9206..89d843a7ef7d1 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty } from 'lodash/fp'; import { CreateRulesSchema } from './create_rules_schema'; export const validateAnomalyThreshold = (rule: CreateRulesSchema): string[] => { @@ -92,6 +93,21 @@ export const validateTimelineTitle = (rule: CreateRulesSchema): string[] => { return []; }; +export const validateThreshold = (rule: CreateRulesSchema): string[] => { + if (rule.type === 'threshold') { + if (!rule.threshold) { + return ['when "type" is "threshold", "threshold" ois required']; + } else if (isEmpty(rule.threshold.field)) { + return ['"threshold.field" cannot be an empty string']; + } else if (rule.threshold.value <= 0) { + return ['"threshold.value" has to be bigger than 0']; + } else { + return []; + } + } + return []; +}; + export const createRuleValidateTypeDependents = (schema: CreateRulesSchema): string[] => { return [ ...validateAnomalyThreshold(schema), @@ -101,5 +117,6 @@ export const createRuleValidateTypeDependents = (schema: CreateRulesSchema): str ...validateMachineLearningJobId(schema), ...validateTimelineId(schema), ...validateTimelineTitle(schema), + ...validateThreshold(schema), ]; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index f61a1546e3e8a..d141ca56828b6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -27,6 +27,7 @@ import { To, type, Threat, + threshold, ThrottleOrNull, note, Version, @@ -125,6 +126,7 @@ export const importRulesSchema = t.intersection([ tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode + threshold, // defaults to "undefined" if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index 070f3ccfd03b0..dd325c1a5034f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -33,6 +33,7 @@ import { enabled, tags, threat, + threshold, throttle, references, to, @@ -89,6 +90,7 @@ export const patchRulesSchema = t.exact( tags, to, threat, + threshold, throttle, timestamp_override, references, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index 98082c2de838a..4f284eedef3fd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -28,6 +28,7 @@ import { To, type, Threat, + threshold, ThrottleOrNull, note, version, @@ -114,6 +115,7 @@ export const updateRulesSchema = t.intersection([ tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode + threshold, // defaults to "undefined" if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index c0fec2b2eefc2..942abb370e949 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -44,6 +44,8 @@ import { timeline_title, type, threat, + threshold, + thresholdOrUndefined, throttle, job_status, status_date, @@ -123,6 +125,9 @@ export const dependentRulesSchema = t.partial({ // ML fields anomaly_threshold, machine_learning_job_id, + + // Threshold fields + threshold: thresholdOrUndefined, }); /** @@ -225,6 +230,18 @@ export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] } }; +export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'threshold') { + return [ + t.exact(t.type({ query: dependentRulesSchema.props.query })), + t.exact(t.type({ language: dependentRulesSchema.props.language })), + t.exact(t.type({ threshold: dependentRulesSchema.props.threshold })), + ]; + } else { + return []; + } +}; + export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed => { const dependents: t.Mixed[] = [ t.exact(requiredRulesSchema), @@ -233,6 +250,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed ...addTimelineTitle(typeAndTimelineOnly), ...addQueryFields(typeAndTimelineOnly), ...addMlFields(typeAndTimelineOnly), + ...addThresholdFields(typeAndTimelineOnly), ]; if (dependents.length > 1) { diff --git a/x-pack/plugins/security_solution/common/detection_engine/types.ts b/x-pack/plugins/security_solution/common/detection_engine/types.ts index c28cac8a8d6fe..7c752bca49dbd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/types.ts @@ -15,6 +15,6 @@ export const RuleTypeSchema = t.keyof({ query: null, saved_query: null, machine_learning: null, - treshold: null, + threshold: null, }); export type RuleType = t.TypeOf; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/helpers.tsx index a0d43c3abf5c1..28117d7ba976e 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/helpers.tsx @@ -292,6 +292,14 @@ export const buildRuleTypeDescription = (label: string, ruleType: RuleType): Lis }, ]; } + case 'threshold': { + return [ + { + title: label, + description: i18n.THRESHOLD_TYPE_DESCRIPTION, + }, + ]; + } default: return assertUnreachable(ruleType); } diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/translations.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/translations.tsx index 3e639ede7a18b..4611d7bb8e319 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/translations.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/translations.tsx @@ -41,6 +41,13 @@ export const QUERY_TYPE_DESCRIPTION = i18n.translate( } ); +export const THRESHOLD_TYPE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.thresholdRuleTypeDescription', + { + defaultMessage: 'Threshold', + } +); + export const ML_JOB_STARTED = i18n.translate( 'xpack.securitySolution.detectionEngine.ruleDescription.mlJobStartedDescription', { diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx index dd8d65aba43d1..11d06a5451f65 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/index.tsx @@ -22,7 +22,7 @@ import { FieldHook } from '../../../../shared_imports'; import { useKibana } from '../../../../common/lib/kibana'; import * as i18n from './translations'; -const isTresholdRule = (ruleType: RuleType) => ruleType === 'treshold'; +const isThresholdRule = (ruleType: RuleType) => ruleType === 'threshold'; const MlCardDescription = ({ subscriptionUrl, @@ -77,7 +77,7 @@ export const SelectRuleType: React.FC = ({ ); const setMl = useCallback(() => setType('machine_learning'), [setType]); const setQuery = useCallback(() => setType('query'), [setType]); - const setTreshold = useCallback(() => setType('treshold'), [setType]); + const setThreshold = useCallback(() => setType('threshold'), [setType]); const mlCardDisabled = isReadOnly || !hasValidLicense || !isMlAdmin; const licensingUrl = useKibana().services.application.getUrlForApp('kibana', { path: '#/management/stack/license_management', @@ -100,7 +100,7 @@ export const SelectRuleType: React.FC = ({ selectable={{ isDisabled: isReadOnly, onClick: setQuery, - isSelected: !isMlRule(ruleType) && !isTresholdRule(ruleType), + isSelected: !isMlRule(ruleType) && !isThresholdRule(ruleType), }} /> @@ -123,13 +123,13 @@ export const SelectRuleType: React.FC = ({ } selectable={{ isDisabled: isReadOnly, - onClick: setTreshold, - isSelected: isTresholdRule(ruleType), + onClick: setThreshold, + isSelected: isThresholdRule(ruleType), }} /> diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts index 748d104fd4a8c..3b85a7dfc765c 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/select_rule_type/translations.ts @@ -34,15 +34,15 @@ export const ML_TYPE_DESCRIPTION = i18n.translate( } ); -export const TRESHOLD_TYPE_TITLE = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.tresholdTypeTitle', +export const THRESHOLD_TYPE_TITLE = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.thresholdTypeTitle', { defaultMessage: 'Threshold', } ); -export const TRESHOLD_TYPE_DESCRIPTION = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.tresholdTypeDescription', +export const THRESHOLD_TYPE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.ruleTypeField.thresholdTypeDescription', { defaultMessage: 'Aggregate query results to detect when number of matches exceeds threshold.', } diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx index b56e1794eef63..da058a9b1208d 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx @@ -35,6 +35,7 @@ import { MlJobSelect } from '../ml_job_select'; import { PickTimeline } from '../pick_timeline'; import { StepContentWrapper } from '../step_content_wrapper'; import { NextStep } from '../next_step'; +import { ThresholdInput } from '../threshold_input'; import { Field, Form, @@ -64,6 +65,10 @@ const stepDefineDefaultValue: DefineStepRule = { filters: [], saved_id: undefined, }, + threshold: { + field: 'host.name', + value: 1000, + }, timeline: { id: null, title: DEFAULT_TIMELINE_TITLE, @@ -246,6 +251,17 @@ const StepDefineRuleComponent: FC = ({ /> + {/* {myStepData.ruleType === 'threshold' && ( */} + + {/* )} */} ; +// type EventArg = Event | React.MouseEvent; + +const CommonUseField = getUseField({ component: Field }); + +export const ThresholdInput = ({ field }: ThresholdInputProps) => { + // const threshold = field.value as number; + // const onThresholdChange = useCallback( + // (event: EventArg) => { + // const thresholdValue = Number((event as Event).target.value); + // field.setValue(thresholdValue); + // }, + // [field] + // ); + + return ( + + + + + {/* */} + + {'>='} + + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts index d991cc35b8dfe..41951cd3c1a60 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts @@ -16,6 +16,7 @@ import { rule_name_override, severity_mapping, timestamp_override, + threshold, } from '../../../../../common/detection_engine/schemas/common/schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -60,6 +61,7 @@ export const NewRuleSchema = t.intersection([ saved_id: t.string, tags: t.array(t.string), threat: t.array(t.unknown), + threshold, throttle: t.union([t.string, t.null]), to: t.string, updated_by: t.string, @@ -131,6 +133,7 @@ export const RuleSchema = t.intersection([ saved_id: t.string, status: t.string, status_date: t.string, + threshold, timeline_id: t.string, timeline_title: t.string, timestamp_override, diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts index b7cf94bb4f319..e48a1792658a8 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts @@ -93,6 +93,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep saved_id: ruleFields.queryBar?.saved_id, ...(ruleType === 'query' && ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), + ...(ruleType === 'threshold' && { threshold: ruleFields.threshold }), }; return { diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx index 2a792f7d35eaa..94f8211babd69 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx @@ -84,6 +84,12 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ id: rule.timeline_id ?? null, title: rule.timeline_title ?? null, }, + threshold: rule.threshold + ? { + field: rule.threshold.field, + value: parseInt(rule.threshold.value, 10), + } + : undefined, }); export const getScheduleStepsData = (rule: Rule): ScheduleStepRule => { @@ -243,6 +249,18 @@ export const redirectToDetections = ( hasEncryptionKey != null && (!isSignalIndexExists || !isAuthenticated || !hasEncryptionKey); +const getRuleSpecificRuleParamKeys = (ruleType: RuleType) => { + if (isMlRule(ruleType)) { + return ['anomaly_threshold', 'machine_learning_job_id']; + } + + if (ruleType === 'threshold') { + return ['index', 'filters', 'language', 'query', 'saved_id', 'threshold']; + } + + return ['index', 'filters', 'language', 'query', 'saved_id']; +}; + export const getActionMessageRuleParams = (ruleType: RuleType): string[] => { const commonRuleParamsKeys = [ 'id', @@ -265,9 +283,7 @@ export const getActionMessageRuleParams = (ruleType: RuleType): string[] => { const ruleParamsKeys = [ ...commonRuleParamsKeys, - ...(isMlRule(ruleType) - ? ['anomaly_threshold', 'machine_learning_job_id'] - : ['index', 'filters', 'language', 'query', 'saved_id']), + ...getRuleSpecificRuleParamKeys(ruleType), ].sort(); return ruleParamsKeys; diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts index f453b5a95994d..d18508b527fda 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts @@ -17,6 +17,7 @@ import { RiskScoreMapping, RuleNameOverride, SeverityMapping, + Threshold, TimestampOverride, } from '../../../../../common/detection_engine/schemas/common/schemas'; @@ -99,6 +100,7 @@ export interface DefineStepRule extends StepRuleData { queryBar: FieldValueQueryBar; ruleType: RuleType; timeline: FieldValueTimeline; + threshold: Threshold; } export interface ScheduleStepRule extends StepRuleData { @@ -122,6 +124,7 @@ export interface DefineStepRuleJson { saved_id?: string; query?: string; language?: string; + threshold?: Threshold; timeline_id?: string; timeline_title?: string; type: RuleType; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 2942413057e37..acd800e54040c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -90,6 +90,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => severity_mapping: severityMapping, tags, threat, + threshold, throttle, timestamp_override: timestampOverride, to, @@ -177,6 +178,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 310a9da56282d..89d6a1c62fa13 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -44,6 +44,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void if (validationErrors.length) { return siemResponse.error({ statusCode: 400, body: validationErrors }); } + + console.log(JSON.stringify(request.body, null, 2)); + const { actions: actionsRest, anomaly_threshold: anomalyThreshold, @@ -75,6 +78,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void severity_mapping: severityMapping, tags, threat, + threshold, throttle, timestamp_override: timestampOverride, to, @@ -125,6 +129,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void }); } } + + console.log('aaa'); + const createdRule = await createRules({ alertsClient, anomalyThreshold, @@ -159,6 +166,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void to, type, threat, + threshold, timestampOverride, references, note, @@ -167,6 +175,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void actions: throttle === 'rule' ? actions : [], // Only enable actions if throttle is rule, otherwise we are a notification and should not enable it, }); + console.log('bbb'); const ruleActions = await updateRulesNotifications({ ruleAlertId: createdRule.id, alertsClient, @@ -177,6 +186,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void name, }); + console.log('ccc'); const ruleStatuses = await ruleStatusSavedObjectsClientFactory(savedObjectsClient).find({ perPage: 1, sortField: 'statusDate', @@ -184,11 +194,13 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void search: `${createdRule.id}`, searchFields: ['alertId'], }); + console.log('ddd'); const [validated, errors] = transformValidate( createdRule, ruleActions, ruleStatuses.saved_objects[0] ); + console.log('eee', errors); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index eceb953762090..fb7d2596344a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -80,7 +80,10 @@ export const findRulesRoute = (router: IRouter) => { }) ); + console.log('aaa'); + const [validated, errors] = transformValidateFindAlerts(rules, ruleActions, ruleStatuses); + console.log('bbb', errors); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 43aa1ecd31922..18eea7c45585f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -161,6 +161,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP severity_mapping: severityMapping, tags, threat, + threshold, timestamp_override: timestampOverride, to, type, @@ -222,6 +223,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP to, type, threat, + threshold, timestampOverride, references, note, @@ -264,6 +266,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP to, type, threat, + threshold, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index c3d6f920e47a9..5099cf5de958f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -85,6 +85,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threshold, timestamp_override: timestampOverride, throttle, references, @@ -143,6 +144,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index eb9624e6412e9..3b3efd2ed166d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -76,6 +76,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { to, type, threat, + threshold, timestamp_override: timestampOverride, throttle, references, @@ -142,6 +143,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index c1ab1be2dbd0a..518024387fed3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -88,6 +88,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threshold, throttle, timestamp_override: timestampOverride, references, @@ -156,6 +157,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 717f388cfc1e9..299b99c4d37b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -78,6 +78,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { to, type, threat, + threshold, throttle, timestamp_override: timestampOverride, references, @@ -146,6 +147,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 9e93dc051a041..900a8ccfb4c23 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -144,6 +144,7 @@ export const transformAlertToRule = ( to: alert.params.to, type: alert.params.type, threat: alert.params.threat ?? [], + threshold: alert.params.threshold ?? undefined, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index b4e246718efd7..e6f9e6b2a0bb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -43,6 +43,7 @@ export const createRules = async ({ severityMapping, tags, threat, + threshold, timestampOverride, to, type, @@ -87,6 +88,7 @@ export const createRules = async ({ severity, severityMapping, threat, + threshold, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 8a86a0f103371..3af0c3f55b485 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -47,6 +47,7 @@ export const installPrepackagedRules = ( to, type, threat, + threshold, timestamp_override: timestampOverride, references, note, @@ -92,6 +93,7 @@ export const installPrepackagedRules = ( to, type, threat, + threshold, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index 577d8d426b63d..e0814647b4c39 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -42,6 +42,7 @@ export const patchRules = async ({ severityMapping, tags, threat, + threshold, timestampOverride, to, type, @@ -83,6 +84,7 @@ export const patchRules = async ({ severityMapping, tags, threat, + threshold, timestampOverride, to, type, @@ -121,6 +123,7 @@ export const patchRules = async ({ severity, severityMapping, threat, + threshold, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 7b793ffbdb362..b845990fd94ef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -59,6 +59,7 @@ import { TagsOrUndefined, ToOrUndefined, ThreatOrUndefined, + ThresholdOrUndefined, TypeOrUndefined, ReferencesOrUndefined, PerPageOrUndefined, @@ -204,6 +205,7 @@ export interface CreateRulesOptions { severityMapping: SeverityMapping; tags: Tags; threat: Threat; + threshold: ThresholdOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; @@ -247,6 +249,7 @@ export interface UpdateRulesOptions { severityMapping: SeverityMapping; tags: Tags; threat: Threat; + threshold: ThresholdOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; @@ -288,6 +291,7 @@ export interface PatchRulesOptions { severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + threshold: ThresholdOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index 6466cc596d891..bf97784e8d917 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -45,6 +45,7 @@ export const updatePrepackagedRules = async ( to, type, threat, + threshold, timestamp_override: timestampOverride, references, version, @@ -93,6 +94,7 @@ export const updatePrepackagedRules = async ( to, type, threat, + threshold, references, version, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 5cc68db25afc8..ae1ed6e119daf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -44,6 +44,7 @@ export const updateRules = async ({ severityMapping, tags, threat, + threshold, timestampOverride, to, type, @@ -86,6 +87,7 @@ export const updateRules = async ({ severityMapping, tags, threat, + threshold, timestampOverride, to, type, @@ -133,6 +135,7 @@ export const updateRules = async ({ severity, severityMapping, threat, + threshold, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index 861d02a8203e6..49c02f92ff336 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -29,6 +29,7 @@ import { TagsOrUndefined, ToOrUndefined, ThreatOrUndefined, + ThresholdOrUndefined, TypeOrUndefined, ReferencesOrUndefined, AuthorOrUndefined, @@ -82,6 +83,7 @@ export interface UpdateProperties { severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + threshold: ThresholdOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index c75dddf896fd1..4736b6135f486 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -20,6 +20,7 @@ export const buildEventsSearchQuery = ({ filter, size, searchAfterSortId, + threshold, }: BuildEventsSearchQuery) => { const filterWithTime = [ filter, @@ -58,6 +59,18 @@ export const buildEventsSearchQuery = ({ }, }, ]; + const aggregations = threshold + ? { + aggs: { + threshold: { + terms: { + field: threshold.field, + min_doc_count: threshold.value, + }, + }, + }, + } + : {}; const searchQuery = { allowNoIndices: true, index, @@ -74,6 +87,7 @@ export const buildEventsSearchQuery = ({ ], }, }, + ...aggregations, sort: [ { '@timestamp': { @@ -83,6 +97,9 @@ export const buildEventsSearchQuery = ({ ], }, }; + + console.log('searchQuery', JSON.stringify(searchQuery, null, 2)); + if (searchAfterSortId) { return { ...searchQuery, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index fc8b26450c852..9e118f77a73e7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -83,5 +83,6 @@ export const buildRule = ({ exceptions_list: ruleParams.exceptionsList ?? [], machine_learning_job_id: ruleParams.machineLearningJobId, anomaly_threshold: ruleParams.anomalyThreshold, + threshold: ruleParams.threshold, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts new file mode 100644 index 0000000000000..80839545951d5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { flow, omit } from 'lodash/fp'; +import set from 'set-value'; +import { SearchResponse } from 'elasticsearch'; + +import { Logger } from '../../../../../../../src/core/server'; +import { AlertServices } from '../../../../../alerts/server'; +import { RuleAlertAction } from '../../../../common/detection_engine/types'; +import { RuleTypeParams, RefreshTypes } from '../types'; +import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; +import { AnomalyResults, Anomaly } from '../../machine_learning'; + +interface BulkCreateMlSignalsParams { + actions: RuleAlertAction[]; + someResult: AnomalyResults; + ruleParams: RuleTypeParams; + services: AlertServices; + logger: Logger; + id: string; + signalsIndex: string; + name: string; + createdAt: string; + createdBy: string; + updatedAt: string; + updatedBy: string; + interval: string; + enabled: boolean; + refresh: RefreshTypes; + tags: string[]; + throttle: string; +} + +interface EcsAnomaly extends Anomaly { + '@timestamp': string; +} + +export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => { + const { + by_field_name: entityName, + by_field_value: entityValue, + influencers, + timestamp, + } = anomaly; + let errantFields = (influencers ?? []).map((influencer) => ({ + name: influencer.influencer_field_name, + value: influencer.influencer_field_values, + })); + + if (entityName && entityValue) { + errantFields = [...errantFields, { name: entityName, value: [entityValue] }]; + } + + const omitDottedFields = omit(errantFields.map((field) => field.name)); + const setNestedFields = errantFields.map((field) => (_anomaly: Anomaly) => + set(_anomaly, field.name, field.value) + ); + const setTimestamp = (_anomaly: Anomaly) => + set(_anomaly, '@timestamp', new Date(timestamp).toISOString()); + + return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly); +}; + +const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse => { + const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({ + ...rest, + _source: transformAnomalyFieldsToEcs(_source), + })); + + return { + ...results, + hits: { + ...results.hits, + hits: transformedHits, + }, + }; +}; + +export const bulkCreateMlSignals = async ( + params: BulkCreateMlSignalsParams +): Promise => { + const anomalyResults = params.someResult; + const ecsResults = transformAnomalyResultsToEcs(anomalyResults); + + return singleBulkCreate({ ...params, filteredEvents: ecsResults }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts new file mode 100644 index 0000000000000..e4590e25eb714 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import dateMath from '@elastic/datemath'; + +import { singleSearchAfter } from './single_search_after'; + +export const findThresholdSignals = async ({ + from, + to, + inputIndexPattern, + services, + logger, + filter, + threshold, +}: {}) => { + const { + searchResult, + searchDuration, + }: { searchResult: SignalSearchResponse; searchDuration: string } = await singleSearchAfter({ + index: inputIndexPattern, + from, + to, + services, + logger, + filter, + pageSize: 0, + threshold, + }); + return searchResult; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 4bd9de734f448..c5fd42e12a075 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -50,6 +50,7 @@ export const getFilter = async ({ lists, }: GetFilterArgs): Promise => { switch (type) { + case 'threshold': case 'query': { if (query != null && language != null && index != null) { return getQueryFilter(query, language, filters || [], index, lists); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 2583cf2c8da91..cd4805747ccc9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -37,6 +37,7 @@ const signalSchema = schema.object({ severity: schema.string(), severityMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + threshold: schema.nullable(schema.object({ field: schema.string(), value: schema.number() })), timestampOverride: schema.nullable(schema.string()), to: schema.string(), type: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a33af1e48585f..35d2a8368e06b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -7,6 +7,7 @@ /* eslint-disable complexity */ import { Logger, KibanaRequest } from 'src/core/server'; +import deepMerge from 'deepmerge'; import { SIGNALS_ID, @@ -26,6 +27,7 @@ import { getGapBetweenRuns, parseScheduleDates, getListsClient, getExceptions } import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; +import { findThresholdSignals } from './find_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { scheduleNotificationActions, @@ -78,6 +80,7 @@ export const signalRulesAlertType = ({ savedId, query, to, + threshold, type, exceptionsList, } = params; @@ -224,6 +227,69 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } + } else if (type === 'threshold') { + console.log('ruleTytpe', 'threshold'); + const inputIndex = await getInputIndex(services, version, index); + const esFilter = await getFilter({ + type, + filters, + language, + query, + savedId, + services, + index: inputIndex, + lists: exceptionItems ?? [], + }); + + const thresholdResults = await findThresholdSignals({ + inputIndexPattern: inputIndex, + from, + to, + services, + logger, + filter: esFilter, + threshold, + }); + + console.log( + 'thresholdResults', + thresholdResults, + JSON.stringify(thresholdResults.aggregations.threshold.buckets, null, 2) + ); + + const thresholdCount = thresholdResults.aggregations.threshold.buckets.length; + if (thresholdCount) { + logger.info(buildRuleMessage(`Found ${thresholdCount} signals from Threshold aggs.`)); + } + + // const { + // success, + // bulkCreateDuration, + // createdItemsCount, + // } = await bulkCreateThresholdSignals({ + // actions, + // throttle, + // someResult: thresholdResults, + // ruleParams: params, + // services, + // logger, + // id: alertId, + // signalsIndex: outputIndex, + // name, + // createdBy, + // createdAt, + // updatedBy, + // updatedAt, + // interval, + // enabled, + // refresh, + // tags, + // }); + // result.success = success; + // result.createdSignalsCount = createdItemsCount; + // if (bulkCreateDuration) { + // result.bulkCreateTimes.push(bulkCreateDuration); + // } } else { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 409f374d7df1e..91f5765cade21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -32,6 +32,7 @@ export const singleSearchAfter = async ({ filter, logger, pageSize, + threshold, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; @@ -44,6 +45,7 @@ export const singleSearchAfter = async ({ filter, size: pageSize, searchAfterSortId, + threshold, }); const start = performance.now(); const nextSearchAfterResult: SignalSearchResponse = await services.callCluster( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index 365222d62d322..4b4f5147c9a42 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -9,6 +9,7 @@ import { Description, NoteOrUndefined, ThreatOrUndefined, + ThresholdOrUndefined, FalsePositives, From, Immutable, @@ -71,6 +72,7 @@ export interface RuleTypeParams { severity: Severity; severityMapping: SeverityMappingOrUndefined; threat: ThreatOrUndefined; + threshold: ThresholdOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: RuleType; From b2bfd9c775ae6aa14204aaf56f5b73d059797133 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 6 Jul 2020 12:20:40 +0200 Subject: [PATCH 03/30] WIP --- .../components/alerts_table/actions.tsx | 77 ++++++++++++++++++- .../public/graphql/introspection.json | 35 +++++++++ .../security_solution/public/graphql/types.ts | 12 +++ .../timelines/containers/index.gql_query.ts | 2 + .../server/graphql/ecs/schema.gql.ts | 1 + .../security_solution/server/graphql/types.ts | 35 +++++++++ .../signals/build_bulk_body.ts | 2 + .../signals/bulk_create_threshold_signals.ts | 40 ++++++++-- .../signals/signal_rule_alert_type.ts | 60 ++++++++------- .../signals/single_bulk_create.ts | 5 ++ .../server/lib/ecs_fields/index.ts | 1 + 11 files changed, 233 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/actions.tsx index ba392e9904cc4..944c32e4fd0b7 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/alerts_table/actions.tsx @@ -4,8 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ +/* eslint-disable complexity */ + import dateMath from '@elastic/datemath'; -import { getOr, isEmpty } from 'lodash/fp'; +import { get, getOr, isEmpty } from 'lodash/fp'; import moment from 'moment'; import { updateAlertStatus } from '../../containers/detection_engine/alerts/api'; @@ -98,6 +100,8 @@ export const sendAlertToTimelineAction = async ({ ecsData, updateTimelineIsLoading, }: SendAlertToTimelineActionProps) => { + console.error('sendAlertToTimelineAction', ecsData); + let openAlertInBasicTimeline = true; const noteContent = ecsData.signal?.rule?.note != null ? ecsData.signal?.rule?.note[0] : ''; const timelineId = @@ -168,7 +172,76 @@ export const sendAlertToTimelineAction = async ({ } } - if (openAlertInBasicTimeline) { + if ( + ecsData.signal?.rule?.type?.length && + ecsData.signal?.rule?.type[0] === 'threshold' && + openAlertInBasicTimeline + ) { + createTimeline({ + from, + timeline: { + ...timelineDefaults, + dataProviders: [ + { + and: [], + id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-timeline-1-alert-id-${ecsData._id}`, + name: ecsData._id, + enabled: true, + excluded: false, + kqlQuery: '', + queryMatch: { + field: '_id', + value: ecsData._id, + operator: ':', + }, + }, + { + and: [], + id: `event-details-value-default-draggable-plain-column-renderer-formatted-field-value-timeline-1-cc2e514d80f3d7abad5b167f6a9efb2761efea2bae4a54834ef807d69daefefd-host_name-${get( + ecsData.signal?.rule?.threshold.field, + ecsData + )}`, + name: ecsData.signal?.rule?.threshold.field, + enabled: true, + excluded: false, + kqlQuery: '', + queryMatch: { + field: ecsData.signal?.rule?.threshold.field, + value: get(ecsData.signal?.rule?.threshold.field, ecsData), + operator: ':', + }, + }, + ], + id: 'timeline-1', + dateRange: { + start: from, + end: to, + }, + eventType: 'all', + kqlQuery: { + filterQuery: { + kuery: { + kind: ecsData.signal?.rule?.language?.length + ? ecsData.signal?.rule?.language[0] + : 'kuery', + expression: ecsData.signal?.rule?.query?.length ? ecsData.signal?.rule?.query[0] : '', + }, + serializedQuery: ecsData.signal?.rule?.query?.length + ? ecsData.signal?.rule?.query[0] + : '', + }, + filterQueryDraft: { + kind: ecsData.signal?.rule?.language?.length + ? ecsData.signal?.rule?.language[0] + : 'kuery', + expression: ecsData.signal?.rule?.query?.length ? ecsData.signal?.rule?.query[0] : '', + }, + }, + }, + to, + ruleNote: noteContent, + }); + } else { createTimeline({ from, timeline: { diff --git a/x-pack/plugins/security_solution/public/graphql/introspection.json b/x-pack/plugins/security_solution/public/graphql/introspection.json index 69356f8fc8aa7..c4955d273b442 100644 --- a/x-pack/plugins/security_solution/public/graphql/introspection.json +++ b/x-pack/plugins/security_solution/public/graphql/introspection.json @@ -4749,6 +4749,14 @@ "type": { "kind": "SCALAR", "name": "ToStringArray", "ofType": null }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "threshold", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "ToAny", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -12327,6 +12335,33 @@ ], "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "Threshold", + "description": "", + "fields": [ + { + "name": "field", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "String", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "value", + "description": "", + "args": [], + "type": { "kind": "SCALAR", "name": "Int", "ofType": null }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "EcsEdges", diff --git a/x-pack/plugins/security_solution/public/graphql/types.ts b/x-pack/plugins/security_solution/public/graphql/types.ts index 1171e93793536..26cd3ec775f5b 100644 --- a/x-pack/plugins/security_solution/public/graphql/types.ts +++ b/x-pack/plugins/security_solution/public/graphql/types.ts @@ -1044,6 +1044,8 @@ export interface RuleField { version?: Maybe; note?: Maybe; + + threshold?: Maybe; } export interface SuricataEcsFields { @@ -2184,6 +2186,12 @@ export interface ResponseFavoriteTimeline { favorite?: Maybe; } +export interface Threshold { + field?: Maybe; + + value?: Maybe; +} + export interface EcsEdges { node: Ecs; @@ -5032,6 +5040,10 @@ export namespace GetTimelineQuery { filters: Maybe; note: Maybe; + + type: Maybe; + + threshold: Maybe; }; export type Suricata = { diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.gql_query.ts b/x-pack/plugins/security_solution/public/timelines/containers/index.gql_query.ts index e2a268e750b4a..6d70d2fa1cf54 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.gql_query.ts +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.gql_query.ts @@ -210,6 +210,8 @@ export const timelineQuery = gql` to filters note + type + threshold } } suricata { diff --git a/x-pack/plugins/security_solution/server/graphql/ecs/schema.gql.ts b/x-pack/plugins/security_solution/server/graphql/ecs/schema.gql.ts index 52011e1416717..38bb66e2d91ec 100644 --- a/x-pack/plugins/security_solution/server/graphql/ecs/schema.gql.ts +++ b/x-pack/plugins/security_solution/server/graphql/ecs/schema.gql.ts @@ -416,6 +416,7 @@ export const ecsSchema = gql` updated_by: ToStringArray version: ToStringArray note: ToStringArray + threshold: ToAny } type SignalField { diff --git a/x-pack/plugins/security_solution/server/graphql/types.ts b/x-pack/plugins/security_solution/server/graphql/types.ts index 2db3052bae66f..7139d5d7b8656 100644 --- a/x-pack/plugins/security_solution/server/graphql/types.ts +++ b/x-pack/plugins/security_solution/server/graphql/types.ts @@ -1046,6 +1046,8 @@ export interface RuleField { version?: Maybe; note?: Maybe; + + threshold?: Maybe; } export interface SuricataEcsFields { @@ -2186,6 +2188,12 @@ export interface ResponseFavoriteTimeline { favorite?: Maybe; } +export interface Threshold { + field?: Maybe; + + value?: Maybe; +} + export interface EcsEdges { node: Ecs; @@ -4907,6 +4915,8 @@ export namespace RuleFieldResolvers { version?: VersionResolver, TypeParent, TContext>; note?: NoteResolver, TypeParent, TContext>; + + threshold?: ThresholdResolver, TypeParent, TContext>; } export type IdResolver< @@ -5064,6 +5074,11 @@ export namespace RuleFieldResolvers { Parent = RuleField, TContext = SiemContext > = Resolver; + export type ThresholdResolver< + R = Maybe, + Parent = RuleField, + TContext = SiemContext + > = Resolver; } export namespace SuricataEcsFieldsResolvers { @@ -8975,6 +8990,25 @@ export namespace ResponseFavoriteTimelineResolvers { > = Resolver; } +export namespace ThresholdResolvers { + export interface Resolvers { + field?: FieldResolver, TypeParent, TContext>; + + value?: ValueResolver, TypeParent, TContext>; + } + + export type FieldResolver< + R = Maybe, + Parent = Threshold, + TContext = SiemContext + > = Resolver; + export type ValueResolver< + R = Maybe, + Parent = Threshold, + TContext = SiemContext + > = Resolver; +} + export namespace EcsEdgesResolvers { export interface Resolvers { node?: NodeResolver; @@ -9329,6 +9363,7 @@ export type IResolvers = { ResponseNote?: ResponseNoteResolvers.Resolvers; ResponseTimeline?: ResponseTimelineResolvers.Resolvers; ResponseFavoriteTimeline?: ResponseFavoriteTimelineResolvers.Resolvers; + Threshold?: ThresholdResolvers.Resolvers; EcsEdges?: EcsEdgesResolvers.Resolvers; EventsTimelineData?: EventsTimelineDataResolvers.Resolvers; OsFields?: OsFieldsResolvers.Resolvers; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 75c4d75cedf1d..b7e4aa446205d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -57,7 +57,9 @@ export const buildBulkBody = ({ tags, throttle, }); + // console.log('buildRule', JSON.stringify(rule, null, 2)); const signal = buildSignal(doc, rule); + // console.log('buildSignal', JSON.stringify(signal, null, 2)); const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 80839545951d5..97a680199bbf5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import uuid from 'uuid'; import { flow, omit } from 'lodash/fp'; import set from 'set-value'; import { SearchResponse } from 'elasticsearch'; @@ -65,10 +66,29 @@ export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => { return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly); }; -const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse => { - const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({ - ...rest, - _source: transformAnomalyFieldsToEcs(_source), +const transformThresholdResultsToEcs = ( + results: ThresholdResults, + filter, + threshold +): SearchResponse => { + console.log( + 'transformThresholdResultsToEcs', + JSON.stringify(results), + JSON.stringify(filter), + JSON.stringify(threshold) + ); + + const transformedHits = results.aggregations.threshold.buckets.map(({ key, doc_count }) => ({ + // ...rest, + _index: '', + _id: uuid.v4(), + _source: { + '@timestamp': new Date().toISOString(), + [threshold.field.split('.')[0]]: { + [threshold.field.split('.')[1]]: key, + }, + threshold_count: doc_count, + }, })); return { @@ -80,11 +100,17 @@ const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse => { - const anomalyResults = params.someResult; - const ecsResults = transformAnomalyResultsToEcs(anomalyResults); + const thresholdResults = params.someResult; + const ecsResults = transformThresholdResultsToEcs( + thresholdResults, + params.filter, + params.threshold + ); + + console.log('ecsResults', JSON.stringify(ecsResults, null, 2)); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 35d2a8368e06b..5adbe4eca1346 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,6 +29,7 @@ import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; +import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -262,34 +263,37 @@ export const signalRulesAlertType = ({ logger.info(buildRuleMessage(`Found ${thresholdCount} signals from Threshold aggs.`)); } - // const { - // success, - // bulkCreateDuration, - // createdItemsCount, - // } = await bulkCreateThresholdSignals({ - // actions, - // throttle, - // someResult: thresholdResults, - // ruleParams: params, - // services, - // logger, - // id: alertId, - // signalsIndex: outputIndex, - // name, - // createdBy, - // createdAt, - // updatedBy, - // updatedAt, - // interval, - // enabled, - // refresh, - // tags, - // }); - // result.success = success; - // result.createdSignalsCount = createdItemsCount; - // if (bulkCreateDuration) { - // result.bulkCreateTimes.push(bulkCreateDuration); - // } + const { + success, + bulkCreateDuration, + createdItemsCount, + } = await bulkCreateThresholdSignals({ + actions, + throttle, + threshold, + someResult: thresholdResults, + ruleParams: params, + filter: esFilter, + services, + logger, + id: alertId, + signalsIndex: outputIndex, + name, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + refresh, + tags, + }); + result.success = success; + result.createdSignalsCount = createdItemsCount; + if (bulkCreateDuration) { + result.bulkCreateTimes.push(bulkCreateDuration); + } + console.log('resulttttt', result); } else { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 3d4e7384714eb..b01d1a6380639 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -82,7 +82,9 @@ export const singleBulkCreate = async ({ tags, throttle, }: SingleBulkCreateParams): Promise => { + // console.log('singleBulkCreate', ruleParams.threshold); filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); + // console.log('filterDuplicateRules', filteredEvents.hits.hits.length); if (filteredEvents.hits.hits.length === 0) { logger.debug(`all events were duplicates`); return { success: true, createdItemsCount: 0 }; @@ -96,6 +98,7 @@ export const singleBulkCreate = async ({ // while preventing duplicates from being added to the // signals index if rules are re-run over the same time // span. Also allow for versioning. + // console.log('singleBulkCreate2', JSON.stringify(ruleParams, null, 2)); const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ { create: { @@ -124,12 +127,14 @@ export const singleBulkCreate = async ({ throttle, }), ]); + // console.log('singleBulkCreate3'); const start = performance.now(); const response: BulkResponse = await services.callCluster('bulk', { index: signalsIndex, refresh, body: bulkBody, }); + // console.log('singleBulkCreate4'); const end = performance.now(); logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); logger.debug(`took property says bulk took: ${response.took} milliseconds`); diff --git a/x-pack/plugins/security_solution/server/lib/ecs_fields/index.ts b/x-pack/plugins/security_solution/server/lib/ecs_fields/index.ts index ff474c4a841f6..f8ec910adc47c 100644 --- a/x-pack/plugins/security_solution/server/lib/ecs_fields/index.ts +++ b/x-pack/plugins/security_solution/server/lib/ecs_fields/index.ts @@ -322,6 +322,7 @@ export const signalFieldsMap: Readonly> = { 'signal.rule.updated_by': 'signal.rule.updated_by', 'signal.rule.version': 'signal.rule.version', 'signal.rule.note': 'signal.rule.note', + 'signal.rule.threshold': 'signal.rule.threshold', }; export const ruleFieldsMap: Readonly> = { From 50b0f2b7ab9649a32c50e62818e088eb304b719c Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 6 Jul 2020 14:01:02 +0200 Subject: [PATCH 04/30] WIP --- .../rules/step_define_rule/index.tsx | 49 ++++++++++++++----- .../detection_engine/rules/create/helpers.ts | 7 ++- .../signals/bulk_create_threshold_signals.ts | 14 +++--- .../signals/signal_rule_alert_type.ts | 14 +++--- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx index da058a9b1208d..e632756ff9d65 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiButtonEmpty, EuiFormRow } from '@elastic/eui'; +import { EuiButtonEmpty, EuiFormRow, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React, { FC, memo, useCallback, useState, useEffect } from 'react'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; @@ -99,6 +99,7 @@ const StepDefineRuleComponent: FC = ({ setForm, setStepData, }) => { + // console.error('aaaa', defaultValues); const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); @@ -161,6 +162,8 @@ const StepDefineRuleComponent: FC = ({ setOpenTimelineSearch(false); }, []); + // console.error('form', myStepData, form); + return isReadOnlyView ? ( = ({ /> - {/* {myStepData.ruleType === 'threshold' && ( */} - - {/* )} */} + + + + + + {'>='} + + + + + => { - console.log( - 'transformThresholdResultsToEcs', - JSON.stringify(results), - JSON.stringify(filter), - JSON.stringify(threshold) - ); + // console.log( + // 'transformThresholdResultsToEcs', + // JSON.stringify(results), + // JSON.stringify(filter), + // JSON.stringify(threshold) + // ); const transformedHits = results.aggregations.threshold.buckets.map(({ key, doc_count }) => ({ // ...rest, @@ -110,7 +110,7 @@ export const bulkCreateThresholdSignals = async ( params.threshold ); - console.log('ecsResults', JSON.stringify(ecsResults, null, 2)); + // console.log('ecsResults', JSON.stringify(ecsResults, null, 2)); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 5adbe4eca1346..e2ce8d5138fd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -229,7 +229,7 @@ export const signalRulesAlertType = ({ result.bulkCreateTimes.push(bulkCreateDuration); } } else if (type === 'threshold') { - console.log('ruleTytpe', 'threshold'); + // console.log('ruleTytpe', 'threshold'); const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -252,11 +252,11 @@ export const signalRulesAlertType = ({ threshold, }); - console.log( - 'thresholdResults', - thresholdResults, - JSON.stringify(thresholdResults.aggregations.threshold.buckets, null, 2) - ); + // console.log( + // 'thresholdResults', + // thresholdResults, + // JSON.stringify(thresholdResults.aggregations.threshold.buckets, null, 2) + // ); const thresholdCount = thresholdResults.aggregations.threshold.buckets.length; if (thresholdCount) { @@ -293,7 +293,7 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } - console.log('resulttttt', result); + // console.log('resulttttt', result); } else { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ From d21cd06c42dfb86fe318866d7191b607cd3adf6c Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 7 Jul 2020 18:26:57 +0200 Subject: [PATCH 05/30] type --- .../common/detection_engine/schemas/common/schemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index 77f9cb03c7db7..b5e906bea286a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -383,7 +383,7 @@ export const threshold = t.exact( export type Threshold = t.TypeOf; export const thresholdOrUndefined = t.union([threshold, t.undefined]); -export type thresholdOrUndefined = t.TypeOf; +export type ThresholdOrUndefined = t.TypeOf; export const created_at = IsoDateString; export const updated_at = IsoDateString; From f6ffe56d89a03b46d63394826a8e92445c77eff4 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Wed, 8 Jul 2020 19:07:14 +0200 Subject: [PATCH 06/30] WIP --- .../request/create_rules_type_dependents.ts | 2 - .../rules/description_step/index.tsx | 4 ++ .../rules/step_define_rule/index.tsx | 40 +++++++++++-------- .../rules/step_define_rule/schema.tsx | 25 ++++++++++++ .../detection_engine/rules/create/helpers.ts | 2 +- .../pages/detection_engine/rules/helpers.tsx | 2 +- .../signals/build_events_query.ts | 23 ++++++----- .../signals/signal_params_schema.ts | 4 +- .../signals/signal_rule_alert_type.ts | 13 +++--- 9 files changed, 77 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index 89d843a7ef7d1..12ebca3fddbba 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -97,8 +97,6 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => { if (rule.type === 'threshold') { if (!rule.threshold) { return ['when "type" is "threshold", "threshold" ois required']; - } else if (isEmpty(rule.threshold.field)) { - return ['"threshold.field" cannot be an empty string']; } else if (rule.threshold.value <= 0) { return ['"threshold.value" has to be bigger than 0']; } else { diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx index 8f3a76c6aea57..8d6fb1e3bc7b1 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx @@ -179,6 +179,10 @@ export const getDescriptionItem = ( (singleThreat: IMitreEnterpriseAttack) => singleThreat.tactic.name !== 'none' ); return buildThreatDescription({ label, threat }); + } else if (field === 'threshold') { + console.error('thresholffi', field); + + return []; } else if (field === 'references') { const urls: string[] = get(field, data); return buildUrlsDescription(label, urls); diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx index e632756ff9d65..9a275f75d6c70 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/index.tsx @@ -35,7 +35,7 @@ import { MlJobSelect } from '../ml_job_select'; import { PickTimeline } from '../pick_timeline'; import { StepContentWrapper } from '../step_content_wrapper'; import { NextStep } from '../next_step'; -import { ThresholdInput } from '../threshold_input'; +import { getCategorizedFieldNames } from '../../../../timelines/components/edit_data_provider/helpers'; import { Field, Form, @@ -53,6 +53,7 @@ const CommonUseField = getUseField({ component: Field }); interface StepDefineRuleProps extends RuleStepProps { defaultValues?: DefineStepRule | null; } +const FIELD_COMBO_BOX_WIDTH = 195; const stepDefineDefaultValue: DefineStepRule = { anomalyThreshold: 50, @@ -66,7 +67,7 @@ const stepDefineDefaultValue: DefineStepRule = { saved_id: undefined, }, threshold: { - field: 'host.name', + field: ['host.name'], value: 1000, }, timeline: { @@ -99,11 +100,10 @@ const StepDefineRuleComponent: FC = ({ setForm, setStepData, }) => { - // console.error('aaaa', defaultValues); const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); - const [localIsMlRule, setIsMlRule] = useState(false); + const [localRuleType, setLocalRuleType] = useState(defaultValues.ruleType); const [indicesConfig] = useUiSetting$(DEFAULT_INDEX_KEY); const [myStepData, setMyStepData] = useState({ ...stepDefineDefaultValue, @@ -162,8 +162,6 @@ const StepDefineRuleComponent: FC = ({ setOpenTimelineSearch(false); }, []); - // console.error('form', myStepData, form); - return isReadOnlyView ? ( = ({ isMlAdmin: hasMlAdminPermissions(mlCapabilities), }} /> - + <> = ({ /> - + <> = ({ /> - - + + = ({ 'data-test-subj': 'detectionEngineStepAboutRuleName', euiFieldProps: { fullWidth: true, - // disabled: isLoading, + disabled: isLoading, + singleSelection: { asPlainText: true }, + noSuggestions: false, + options: getCategorizedFieldNames(browserFields), + placeholder: 'All results', + onCreateOption: undefined, + style: { width: `${FIELD_COMBO_BOX_WIDTH}px` }, }, }} /> @@ -280,7 +287,7 @@ const StepDefineRuleComponent: FC = ({ euiFieldProps: { fullWidth: true, type: 'number', - // disabled: isLoading, + disabled: isLoading, }, }} /> @@ -302,15 +309,14 @@ const StepDefineRuleComponent: FC = ({ if (deepEqual(index, indicesConfig) && indexModified) { setIndexModified(false); } else if (!deepEqual(index, indicesConfig) && !indexModified) { + // TODO: refactor to form.subscribe() + setMyStepData((currentValue) => ({ ...currentValue, index })); setIndexModified(true); } } - if (isMlRule(ruleType) && !localIsMlRule) { - setIsMlRule(true); - clearErrors(); - } else if (!isMlRule(ruleType) && localIsMlRule) { - setIsMlRule(false); + if (ruleType !== localRuleType) { + setLocalRuleType(ruleType); clearErrors(); } diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/schema.tsx index 190d4484b156b..0899784830ec0 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_define_rule/schema.tsx @@ -172,4 +172,29 @@ export const schema: FormSchema = { } ), }, + threshold: { + field: { + type: FIELD_TYPES.COMBO_BOX, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldLabel', + { + defaultMessage: 'Field', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldHelpText', + { + defaultMessage: 'Select a field to group results by', + } + ), + }, + value: { + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdValueLabel', + { + defaultMessage: 'Threshold', + } + ), + }, + }, }; diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts index f9313f643de6a..7f19a93cc325d 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts @@ -95,7 +95,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold.field, + field: ruleFields.threshold.field[0] ?? '', value: parseInt(ruleFields.threshold.value, 10), }, }), diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx index 94f8211babd69..e11492bb8e786 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx @@ -86,7 +86,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ }, threshold: rule.threshold ? { - field: rule.threshold.field, + field: [rule.threshold.field], value: parseInt(rule.threshold.value, 10), } : undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 4736b6135f486..41e7909cb04e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty } from 'lodash/fp'; + interface BuildEventsSearchQuery { index: string[]; from: string; @@ -59,18 +61,19 @@ export const buildEventsSearchQuery = ({ }, }, ]; - const aggregations = threshold - ? { - aggs: { - threshold: { - terms: { - field: threshold.field, - min_doc_count: threshold.value, + const aggregations = + threshold && !isEmpty(threshold.field) + ? { + aggs: { + threshold: { + terms: { + field: threshold.field, + min_doc_count: threshold.value, + }, }, }, - }, - } - : {}; + } + : {}; const searchQuery = { allowNoIndices: true, index, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index cd4805747ccc9..574bc2dfec7f6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -37,7 +37,9 @@ const signalSchema = schema.object({ severity: schema.string(), severityMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), - threshold: schema.nullable(schema.object({ field: schema.string(), value: schema.number() })), + threshold: schema.nullable( + schema.object({ field: schema.nullable(schema.string()), value: schema.number() }) + ), timestampOverride: schema.nullable(schema.string()), to: schema.string(), type: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index e2ce8d5138fd5..75a3af692f236 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -252,13 +252,14 @@ export const signalRulesAlertType = ({ threshold, }); - // console.log( - // 'thresholdResults', - // thresholdResults, - // JSON.stringify(thresholdResults.aggregations.threshold.buckets, null, 2) - // ); + console.log( + 'thresholdResults', + thresholdResults + // JSON.stringify(thresholdResults.aggregations?.threshold.buckets, null, 2) + ); - const thresholdCount = thresholdResults.aggregations.threshold.buckets.length; + const thresholdCount = 0; + // const thresholdCount = thresholdResults.aggregations.threshold.buckets.length; if (thresholdCount) { logger.info(buildRuleMessage(`Found ${thresholdCount} signals from Threshold aggs.`)); } From 1dbf8636adcc7508af6867539f9396f534010d88 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 10:02:15 +0200 Subject: [PATCH 07/30] WIP --- .../components/alerts_table/actions.tsx | 8 +- .../rules/step_define_rule/index.tsx | 4 +- .../pages/detection_engine/rules/types.ts | 5 +- .../signals/build_events_query.ts | 20 +-- .../detection_engine/signals/build_signal.ts | 3 + .../signals/bulk_create_threshold_signals.ts | 150 ++++++++++++------ .../signals/find_threshold_signals.ts | 39 ++++- .../signals/signal_rule_alert_type.ts | 4 +- .../signals/single_search_after.ts | 5 +- 9 files changed, 158 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 99239dc764a2f..875b978101bf3 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -107,8 +107,6 @@ export const sendAlertToTimelineAction = async ({ ecsData, updateTimelineIsLoading, }: SendAlertToTimelineActionProps) => { - console.error('sendAlertToTimelineAction', ecsData); - let openAlertInBasicTimeline = true; const noteContent = ecsData.signal?.rule?.note != null ? ecsData.signal?.rule?.note[0] : ''; const timelineId = @@ -150,7 +148,7 @@ export const sendAlertToTimelineAction = async ({ timeline.timelineType ); - createTimeline({ + return createTimeline({ from, timeline: { ...timeline, @@ -195,7 +193,7 @@ export const sendAlertToTimelineAction = async ({ ecsData.signal?.rule?.type[0] === 'threshold' && openAlertInBasicTimeline ) { - createTimeline({ + return createTimeline({ from, timeline: { ...timelineDefaults, @@ -260,7 +258,7 @@ export const sendAlertToTimelineAction = async ({ ruleNote: noteContent, }); } else { - createTimeline({ + return createTimeline({ from, timeline: { ...timelineDefaults, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 08ea4b2809780..5a0f0a96bbbdc 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -103,7 +103,9 @@ const StepDefineRuleComponent: FC = ({ const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); - const [localRuleType, setLocalRuleType] = useState(defaultValues.ruleType); + const [localRuleType, setLocalRuleType] = useState( + defaultValues?.ruleType || stepDefineDefaultValue.ruleType + ); const [indicesConfig] = useUiSetting$(DEFAULT_INDEX_KEY); const [myStepData, setMyStepData] = useState({ ...stepDefineDefaultValue, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index d18508b527fda..c6eab9c6f5c05 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -100,7 +100,10 @@ export interface DefineStepRule extends StepRuleData { queryBar: FieldValueQueryBar; ruleType: RuleType; timeline: FieldValueTimeline; - threshold: Threshold; + threshold?: { + field: string[]; + value: number; + }; } export interface ScheduleStepRule extends StepRuleData { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 41e7909cb04e6..3db012bced953 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -4,9 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty } from 'lodash/fp'; - interface BuildEventsSearchQuery { + aggregations: unknown; index: string[]; from: string; to: string; @@ -16,13 +15,13 @@ interface BuildEventsSearchQuery { } export const buildEventsSearchQuery = ({ + aggregations, index, from, to, filter, size, searchAfterSortId, - threshold, }: BuildEventsSearchQuery) => { const filterWithTime = [ filter, @@ -61,19 +60,6 @@ export const buildEventsSearchQuery = ({ }, }, ]; - const aggregations = - threshold && !isEmpty(threshold.field) - ? { - aggs: { - threshold: { - terms: { - field: threshold.field, - min_doc_count: threshold.value, - }, - }, - }, - } - : {}; const searchQuery = { allowNoIndices: true, index, @@ -90,7 +76,7 @@ export const buildEventsSearchQuery = ({ ], }, }, - ...aggregations, + ...(aggregations ? { aggregations } : {}), sort: [ { '@timestamp': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 77a63c63ff97a..b97b86495f188 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -56,6 +56,9 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): S if (doc._source.event != null) { return { ...signal, original_event: doc._source.event }; } + if (doc.threshold_count != null) { + return { ...signal, threshold_count: doc.threshold_count }; + } return signal; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 2d057d67ee741..2dac1ecac9a54 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -5,7 +5,7 @@ */ import uuid from 'uuid'; -import { flow, omit } from 'lodash/fp'; +import { flow, omit, reduce, get, isEmpty } from 'lodash/fp'; import set from 'set-value'; import { SearchResponse } from 'elasticsearch'; @@ -16,7 +16,7 @@ import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { AnomalyResults, Anomaly } from '../../machine_learning'; -interface BulkCreateMlSignalsParams { +interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; someResult: AnomalyResults; ruleParams: RuleTypeParams; @@ -36,34 +36,93 @@ interface BulkCreateMlSignalsParams { throttle: string; } -interface EcsAnomaly extends Anomaly { - '@timestamp': string; -} +interface ThresholdResults {} + +const getNestedQueryFilters = (filtersObj) => { + if (Array.isArray(filtersObj.bool?.filter)) { + return reduce( + (acc, filterItem) => { + const nestedFilter = getNestedQueryFilters(filterItem); + + if (nestedFilter) { + return { ...acc, ...nestedFilter }; + } -export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => { - const { - by_field_name: entityName, - by_field_value: entityValue, - influencers, - timestamp, - } = anomaly; - let errantFields = (influencers ?? []).map((influencer) => ({ - name: influencer.influencer_field_name, - value: influencer.influencer_field_values, - })); - - if (entityName && entityValue) { - errantFields = [...errantFields, { name: entityName, value: [entityValue] }]; + return acc; + }, + {}, + filtersObj.bool.filter + ); + } else { + return filtersObj.bool.should && filtersObj.bool.should[0].match; } +}; + +const getThresholdSignalQueryFields = (filter) => { + const filters = get('bool.filter', filter); + + return reduce( + (acc, item) => { + if (item.match_phrase) { + return { ...acc, ...item.match_phrase }; + } + + if (item.bool.should && item.bool.should[0].match) { + return { ...acc, ...item.bool.should[0].match }; + } + + if (item.bool?.filter) { + return { ...acc, ...getNestedQueryFilters(item) }; + } - const omitDottedFields = omit(errantFields.map((field) => field.name)); - const setNestedFields = errantFields.map((field) => (_anomaly: Anomaly) => - set(_anomaly, field.name, field.value) + return acc; + }, + {}, + filters ); - const setTimestamp = (_anomaly: Anomaly) => - set(_anomaly, '@timestamp', new Date(timestamp).toISOString()); +}; + +const getTransformedHits = (results, threshold, signalQueryFields) => { + if (isEmpty(threshold.field)) { + if (results.hits.total.value < threshold.value) { + return []; + } + + const source = { + '@timestamp': new Date().toISOString(), + ...signalQueryFields, + }; + + return [ + { + _index: '', + _id: uuid.v4(), + _source: source, + threshold_count: results.hits.total.value, + }, + ]; + } - return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly); + if (!results.aggregations?.threshold) { + return []; + } + + return results.aggregations.threshold.buckets.map(({ key, doc_count }) => { + const source = { + '@timestamp': new Date().toISOString(), + ...signalQueryFields, + }; + + set(source, threshold.field, key); + + return { + // ...rest, + _index: '', + _id: uuid.v4(), + _source: source, + threshold_count: doc_count, + }; + }); }; const transformThresholdResultsToEcs = ( @@ -71,37 +130,34 @@ const transformThresholdResultsToEcs = ( filter, threshold ): SearchResponse => { - // console.log( - // 'transformThresholdResultsToEcs', - // JSON.stringify(results), - // JSON.stringify(filter), - // JSON.stringify(threshold) - // ); - - const transformedHits = results.aggregations.threshold.buckets.map(({ key, doc_count }) => ({ - // ...rest, - _index: '', - _id: uuid.v4(), - _source: { - '@timestamp': new Date().toISOString(), - [threshold.field.split('.')[0]]: { - [threshold.field.split('.')[1]]: key, - }, - threshold_count: doc_count, - }, - })); + console.log( + 'transformThresholdResultsToEcs', + // JSON.stringify(results), + JSON.stringify(filter) + // JSON.stringify(threshold) + ); + + const signalQueryFields = getThresholdSignalQueryFields(filter); - return { + console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); + + const transformedHits = getTransformedHits(results, threshold, signalQueryFields); + + const thresholdResults = { ...results, hits: { ...results.hits, hits: transformedHits, }, }; + + set(thresholdResults, 'this.total.value', transformedHits.length); + + return thresholdResults; }; export const bulkCreateThresholdSignals = async ( - params: BulkCreateMlSignalsParams + params: BulkCreateThresholdSignalsParams ): Promise => { const thresholdResults = params.someResult; const ecsResults = transformThresholdResultsToEcs( @@ -110,7 +166,7 @@ export const bulkCreateThresholdSignals = async ( params.threshold ); - // console.log('ecsResults', JSON.stringify(ecsResults, null, 2)); + console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index e4590e25eb714..42e7869ace353 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -4,10 +4,25 @@ * you may not use this file except in compliance with the Elastic License. */ -import dateMath from '@elastic/datemath'; +import { isEmpty } from 'lodash/fp'; +import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; import { singleSearchAfter } from './single_search_after'; +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { SignalSearchResponse } from './types'; + +interface FindThresholdSignalsParams { + from: string; + to: string; + inputIndexPattern: string[]; + services: AlertServices; + logger: Logger; + filter: unknown; + threshold: Threshold; +} + export const findThresholdSignals = async ({ from, to, @@ -16,11 +31,28 @@ export const findThresholdSignals = async ({ logger, filter, threshold, -}: {}) => { +}: FindThresholdSignalsParams) => { + const aggregations = + threshold && !isEmpty(threshold.field) + ? { + threshold: { + terms: { + field: threshold.field, + min_doc_count: threshold.value, + }, + }, + } + : {}; + const { searchResult, searchDuration, - }: { searchResult: SignalSearchResponse; searchDuration: string } = await singleSearchAfter({ + }: { + searchResult: SignalSearchResponse; + searchDuration: string; + } = await singleSearchAfter({ + aggregations, + searchAfterSortId: undefined, index: inputIndexPattern, from, to, @@ -28,7 +60,6 @@ export const findThresholdSignals = async ({ logger, filter, pageSize: 0, - threshold, }); return searchResult; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 75a3af692f236..3742202fea6e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -7,7 +7,6 @@ /* eslint-disable complexity */ import { Logger, KibanaRequest } from 'src/core/server'; -import deepMerge from 'deepmerge'; import { SIGNALS_ID, @@ -228,8 +227,7 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } - } else if (type === 'threshold') { - // console.log('ruleTytpe', 'threshold'); + } else if (type === 'threshold' && threshold) { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 91f5765cade21..daea277f14368 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -12,6 +12,7 @@ import { buildEventsSearchQuery } from './build_events_query'; import { makeFloatString } from './utils'; interface SingleSearchAfterParams { + aggregations?: unknown; searchAfterSortId: string | undefined; index: string[]; from: string; @@ -24,6 +25,7 @@ interface SingleSearchAfterParams { // utilize search_after for paging results into bulk. export const singleSearchAfter = async ({ + aggregations, searchAfterSortId, index, from, @@ -32,20 +34,19 @@ export const singleSearchAfter = async ({ filter, logger, pageSize, - threshold, }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; }> => { try { const searchAfterQuery = buildEventsSearchQuery({ + aggregations, index, from, to, filter, size: pageSize, searchAfterSortId, - threshold, }); const start = performance.now(); const nextSearchAfterResult: SignalSearchResponse = await services.callCluster( From 82538f044073e5878d746843ae8ce5cf72040fc9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 10:43:50 +0200 Subject: [PATCH 08/30] WIP --- .../request/create_rules_type_dependents.ts | 3 +-- .../rules/threshold_input/index.tsx | 9 ------- .../rules/create_rules.mock.ts | 2 ++ .../rules/patch_rules.mock.ts | 2 ++ .../rules/update_rules.mock.ts | 2 ++ .../lib/detection_engine/rules/utils.test.ts | 3 +++ .../signals/__mocks__/es_results.ts | 1 + .../signals/build_events_query.ts | 2 +- .../detection_engine/signals/build_signal.ts | 4 +-- .../signals/bulk_create_threshold_signals.ts | 25 +++++++++++-------- .../signals/find_threshold_signals.ts | 14 ++++------- .../signals/signal_params_schema.mock.ts | 1 + .../signals/signal_rule_alert_type.ts | 3 +-- .../signals/single_bulk_create.ts | 5 ---- .../lib/detection_engine/signals/types.ts | 1 + 15 files changed, 37 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index 12ebca3fddbba..af665ff8c81d2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty } from 'lodash/fp'; import { CreateRulesSchema } from './create_rules_schema'; export const validateAnomalyThreshold = (rule: CreateRulesSchema): string[] => { @@ -96,7 +95,7 @@ export const validateTimelineTitle = (rule: CreateRulesSchema): string[] => { export const validateThreshold = (rule: CreateRulesSchema): string[] => { if (rule.type === 'threshold') { if (!rule.threshold) { - return ['when "type" is "threshold", "threshold" ois required']; + return ['when "type" is "threshold", "threshold" is required']; } else if (rule.threshold.value <= 0) { return ['"threshold.value" has to be bigger than 0']; } else { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 34c1a1cdfabb8..7d9385312c115 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -42,15 +42,6 @@ export const ThresholdInput = ({ field }: ThresholdInputProps) => { }, }} /> - {/* */} {'>='} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index a7e24a1ac1609..1117f34b6f8c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -39,6 +39,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -81,6 +82,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index f3102a5ad2cf3..cfb40056eb85d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -143,6 +143,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -185,6 +186,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index fdc0a61274e75..650b59fb85bc0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -41,6 +41,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -84,6 +85,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threshold: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index aa0512678b073..17505a4478261 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -53,6 +53,7 @@ describe('utils', () => { severityMapping: undefined, tags: undefined, threat: undefined, + threshold: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -94,6 +95,7 @@ describe('utils', () => { severityMapping: undefined, tags: undefined, threat: undefined, + threshold: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -135,6 +137,7 @@ describe('utils', () => { severityMapping: undefined, tags: undefined, threat: undefined, + threshold: undefined, to: undefined, timestampOverride: undefined, type: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 7492422968062..17e05109b9a87 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -46,6 +46,7 @@ export const sampleRuleAlertParams = ( machineLearningJobId: undefined, filters: undefined, savedId: undefined, + threshold: undefined, timelineId: undefined, timelineTitle: undefined, timestampOverride: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 3db012bced953..341a930bf8d19 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -5,7 +5,7 @@ */ interface BuildEventsSearchQuery { - aggregations: unknown; + aggregations?: unknown; index: string[]; from: string; to: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index b97b86495f188..563c416faaae4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -56,8 +56,8 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): S if (doc._source.event != null) { return { ...signal, original_event: doc._source.event }; } - if (doc.threshold_count != null) { - return { ...signal, threshold_count: doc.threshold_count }; + if (doc._source.threshold_count != null) { + return { ...signal, threshold_count: doc._source.threshold_count }; } return signal; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 2dac1ecac9a54..b328ccdc6d713 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -5,24 +5,25 @@ */ import uuid from 'uuid'; -import { flow, omit, reduce, get, isEmpty } from 'lodash/fp'; +import { reduce, get, isEmpty } from 'lodash/fp'; import set from 'set-value'; import { SearchResponse } from 'elasticsearch'; +import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { AnomalyResults, Anomaly } from '../../machine_learning'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; - someResult: AnomalyResults; + someResult: SingleSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; logger: Logger; id: string; + filter: unknown; signalsIndex: string; name: string; createdAt: string; @@ -38,7 +39,7 @@ interface BulkCreateThresholdSignalsParams { interface ThresholdResults {} -const getNestedQueryFilters = (filtersObj) => { +const getNestedQueryFilters = (filtersObj: unknown) => { if (Array.isArray(filtersObj.bool?.filter)) { return reduce( (acc, filterItem) => { @@ -58,7 +59,7 @@ const getNestedQueryFilters = (filtersObj) => { } }; -const getThresholdSignalQueryFields = (filter) => { +const getThresholdSignalQueryFields = (filter: unknown) => { const filters = get('bool.filter', filter); return reduce( @@ -82,7 +83,11 @@ const getThresholdSignalQueryFields = (filter) => { ); }; -const getTransformedHits = (results, threshold, signalQueryFields) => { +const getTransformedHits = ( + results, + threshold: Threshold, + signalQueryFields: Record +) => { if (isEmpty(threshold.field)) { if (results.hits.total.value < threshold.value) { return []; @@ -110,6 +115,7 @@ const getTransformedHits = (results, threshold, signalQueryFields) => { return results.aggregations.threshold.buckets.map(({ key, doc_count }) => { const source = { '@timestamp': new Date().toISOString(), + threshold_count: doc_count, ...signalQueryFields, }; @@ -120,15 +126,14 @@ const getTransformedHits = (results, threshold, signalQueryFields) => { _index: '', _id: uuid.v4(), _source: source, - threshold_count: doc_count, }; }); }; const transformThresholdResultsToEcs = ( results: ThresholdResults, - filter, - threshold + filter: unknown, + threshold: Threshold ): SearchResponse => { console.log( 'transformThresholdResultsToEcs', @@ -163,7 +168,7 @@ export const bulkCreateThresholdSignals = async ( const ecsResults = transformThresholdResultsToEcs( thresholdResults, params.filter, - params.threshold + params.ruleParams.threshold ); console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 42e7869ace353..a9a199f210da0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -31,7 +31,10 @@ export const findThresholdSignals = async ({ logger, filter, threshold, -}: FindThresholdSignalsParams) => { +}: FindThresholdSignalsParams): Promise<{ + searchResult: SignalSearchResponse; + searchDuration: string; +}> => { const aggregations = threshold && !isEmpty(threshold.field) ? { @@ -44,13 +47,7 @@ export const findThresholdSignals = async ({ } : {}; - const { - searchResult, - searchDuration, - }: { - searchResult: SignalSearchResponse; - searchDuration: string; - } = await singleSearchAfter({ + return singleSearchAfter({ aggregations, searchAfterSortId: undefined, index: inputIndexPattern, @@ -61,5 +58,4 @@ export const findThresholdSignals = async ({ filter, pageSize: 0, }); - return searchResult; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index 0c56ed300cb48..3e8865d107979 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -43,6 +43,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ severity: 'high', severityMapping: null, threat: null, + threshold: null, timelineId: null, timelineTitle: null, timestampOverride: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 3742202fea6e3..9add39673ca49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -240,7 +240,7 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); - const thresholdResults = await findThresholdSignals({ + const { searchResult: thresholdResults } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, to, @@ -269,7 +269,6 @@ export const signalRulesAlertType = ({ } = await bulkCreateThresholdSignals({ actions, throttle, - threshold, someResult: thresholdResults, ruleParams: params, filter: esFilter, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index b01d1a6380639..3d4e7384714eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -82,9 +82,7 @@ export const singleBulkCreate = async ({ tags, throttle, }: SingleBulkCreateParams): Promise => { - // console.log('singleBulkCreate', ruleParams.threshold); filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); - // console.log('filterDuplicateRules', filteredEvents.hits.hits.length); if (filteredEvents.hits.hits.length === 0) { logger.debug(`all events were duplicates`); return { success: true, createdItemsCount: 0 }; @@ -98,7 +96,6 @@ export const singleBulkCreate = async ({ // while preventing duplicates from being added to the // signals index if rules are re-run over the same time // span. Also allow for versioning. - // console.log('singleBulkCreate2', JSON.stringify(ruleParams, null, 2)); const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ { create: { @@ -127,14 +124,12 @@ export const singleBulkCreate = async ({ throttle, }), ]); - // console.log('singleBulkCreate3'); const start = performance.now(); const response: BulkResponse = await services.callCluster('bulk', { index: signalsIndex, refresh, body: bulkBody, }); - // console.log('singleBulkCreate4'); const end = performance.now(); logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); logger.debug(`took property says bulk took: ${response.took} milliseconds`); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 082211df28320..5d6bafc5a6d09 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -121,6 +121,7 @@ export interface Signal { original_time: string; original_event?: SearchTypes; status: Status; + threshold_count?: SearchTypes; } export interface SignalHit { From 39a7ffa669a5fce46d9b128b8e801fd156a74c62 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 11:10:25 +0200 Subject: [PATCH 09/30] WIP --- .../add_prepackaged_rules_type_dependents.ts | 14 ++++++++++++ .../request/import_rules_type_dependents.ts | 14 ++++++++++++ .../request/update_rules_type_dependents.ts | 14 ++++++++++++ .../rules/step_define_rule/index.tsx | 22 +++++++++++++------ .../pages/detection_engine/rules/helpers.tsx | 5 ++++- .../routes/rules/create_rules_route.ts | 2 -- .../routes/rules/find_rules_route.ts | 3 --- .../signals/bulk_create_threshold_signals.ts | 3 ++- .../signals/signal_params_schema.ts | 2 +- 9 files changed, 64 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.ts index 2788c331154d2..6a51f724fc9e6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.ts @@ -92,6 +92,19 @@ export const validateTimelineTitle = (rule: AddPrepackagedRulesSchema): string[] return []; }; +export const validateThreshold = (rule: AddPrepackagedRulesSchema): string[] => { + if (rule.type === 'threshold') { + if (!rule.threshold) { + return ['when "type" is "threshold", "threshold" is required']; + } else if (rule.threshold.value <= 0) { + return ['"threshold.value" has to be bigger than 0']; + } else { + return []; + } + } + return []; +}; + export const addPrepackagedRuleValidateTypeDependents = ( schema: AddPrepackagedRulesSchema ): string[] => { @@ -103,5 +116,6 @@ export const addPrepackagedRuleValidateTypeDependents = ( ...validateMachineLearningJobId(schema), ...validateTimelineId(schema), ...validateTimelineTitle(schema), + ...validateThreshold(schema), ]; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.ts index 59191a4fe3121..269181449e9e9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.ts @@ -92,6 +92,19 @@ export const validateTimelineTitle = (rule: ImportRulesSchema): string[] => { return []; }; +export const validateThreshold = (rule: ImportRulesSchema): string[] => { + if (rule.type === 'threshold') { + if (!rule.threshold) { + return ['when "type" is "threshold", "threshold" is required']; + } else if (rule.threshold.value <= 0) { + return ['"threshold.value" has to be bigger than 0']; + } else { + return []; + } + } + return []; +}; + export const importRuleValidateTypeDependents = (schema: ImportRulesSchema): string[] => { return [ ...validateAnomalyThreshold(schema), @@ -101,5 +114,6 @@ export const importRuleValidateTypeDependents = (schema: ImportRulesSchema): str ...validateMachineLearningJobId(schema), ...validateTimelineId(schema), ...validateTimelineTitle(schema), + ...validateThreshold(schema), ]; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.ts index 9204f727b2660..44182d250c801 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.ts @@ -102,6 +102,19 @@ export const validateId = (rule: UpdateRulesSchema): string[] => { } }; +export const validateThreshold = (rule: UpdateRulesSchema): string[] => { + if (rule.type === 'threshold') { + if (!rule.threshold) { + return ['when "type" is "threshold", "threshold" is required']; + } else if (rule.threshold.value <= 0) { + return ['"threshold.value" has to be bigger than 0']; + } else { + return []; + } + } + return []; +}; + export const updateRuleValidateTypeDependents = (schema: UpdateRulesSchema): string[] => { return [ ...validateId(schema), @@ -112,5 +125,6 @@ export const updateRuleValidateTypeDependents = (schema: UpdateRulesSchema): str ...validateMachineLearningJobId(schema), ...validateTimelineId(schema), ...validateTimelineTitle(schema), + ...validateThreshold(schema), ]; }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 5a0f0a96bbbdc..aac6b3f8d5ee7 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -67,8 +67,8 @@ const stepDefineDefaultValue: DefineStepRule = { saved_id: undefined, }, threshold: { - field: ['host.name'], - value: 1000, + field: [''], + value: 100, }, timeline: { id: null, @@ -90,6 +90,12 @@ MyLabelButton.defaultProps = { flush: 'right', }; +const RuleTypeEuiFormRow = styled(EuiFormRow).attrs<{ isVisible: boolean }>(({ isVisible }) => ({ + style: { + display: isVisible ? 'flex' : 'none', + }, +}))<{ isVisible: boolean }>``; + const StepDefineRuleComponent: FC = ({ addPadding = false, defaultValues, @@ -236,7 +242,7 @@ const StepDefineRuleComponent: FC = ({ /> - + <> = ({ }} /> - - + @@ -265,6 +271,7 @@ const StepDefineRuleComponent: FC = ({ componentProps={{ idAria: 'detectionEngineStepAboutRuleName', 'data-test-subj': 'detectionEngineStepAboutRuleName', + describedByIds: ['detectionEngineStepDefineRuleThresholdField'], euiFieldProps: { fullWidth: true, disabled: isLoading, @@ -285,6 +292,7 @@ const StepDefineRuleComponent: FC = ({ componentProps={{ idAria: 'detectionEngineStepAboutRuleName', 'data-test-subj': 'detectionEngineStepAboutRuleName', + describedByIds: ['detectionEngineStepDefineRuleThresholdValue'], type: 'number', euiFieldProps: { fullWidth: true, @@ -295,7 +303,7 @@ const StepDefineRuleComponent: FC = ({ /> - + ({ field: [rule.threshold.field], value: parseInt(rule.threshold.value, 10), } - : undefined, + : { + field: [''], + value: 100, + }, }); export const getScheduleStepsData = (rule: Rule): ScheduleStepRule => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 89d6a1c62fa13..b9a942e6f3dd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -130,8 +130,6 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void } } - console.log('aaa'); - const createdRule = await createRules({ alertsClient, anomalyThreshold, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index fb7d2596344a9..eceb953762090 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -80,10 +80,7 @@ export const findRulesRoute = (router: IRouter) => { }) ); - console.log('aaa'); - const [validated, errors] = transformValidateFindAlerts(rules, ruleActions, ruleStatuses); - console.log('bbb', errors); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index b328ccdc6d713..fea3ee59b3a4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -15,10 +15,11 @@ import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; +import { SignalSearchResponse } from './types'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; - someResult: SingleSearchResponse; + someResult: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; logger: Logger; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 574bc2dfec7f6..d08ca90f3e353 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -37,7 +37,7 @@ const signalSchema = schema.object({ severity: schema.string(), severityMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), - threshold: schema.nullable( + threshold: schema.maybe( schema.object({ field: schema.nullable(schema.string()), value: schema.number() }) ), timestampOverride: schema.nullable(schema.string()), From 4974834ed1437b7da1895c03777c9dff01ec4771 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 11:15:52 +0200 Subject: [PATCH 10/30] WIP --- .../pages/detection_engine/rules/create/helpers.ts | 2 +- .../pages/detection_engine/rules/create/index.tsx | 2 -- .../detections/pages/detection_engine/rules/helpers.tsx | 6 ++++-- .../lib/detection_engine/routes/rules/create_rules_route.ts | 6 ------ .../server/lib/detection_engine/signals/build_bulk_body.ts | 2 -- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 3a5258e92de3d..3805a14995e7f 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -96,7 +96,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ...(ruleType === 'threshold' && { threshold: { field: ruleFields.threshold.field[0] ?? '', - value: parseInt(ruleFields.threshold.value, 10), + value: parseInt(ruleFields.threshold.value, 10) ?? 100, }, }), }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx index 749475c371d5f..6475b6f6b6b54 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx @@ -286,8 +286,6 @@ const CreateRulePageComponent: React.FC = () => { return null; } - console.error('aaa', stepsData); - return ( <> diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index c02cb498900fb..9bc7acbe91e66 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -253,15 +253,17 @@ export const redirectToDetections = ( (!isSignalIndexExists || !isAuthenticated || !hasEncryptionKey); const getRuleSpecificRuleParamKeys = (ruleType: RuleType) => { + const queryRuleParams = ['index', 'filters', 'language', 'query', 'saved_id']; + if (isMlRule(ruleType)) { return ['anomaly_threshold', 'machine_learning_job_id']; } if (ruleType === 'threshold') { - return ['index', 'filters', 'language', 'query', 'saved_id', 'threshold']; + return ['threshold', ...queryRuleParams]; } - return ['index', 'filters', 'language', 'query', 'saved_id']; + return queryRuleParams; }; export const getActionMessageRuleParams = (ruleType: RuleType): string[] => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index b9a942e6f3dd5..edad3dd8a4f21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -45,8 +45,6 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void return siemResponse.error({ statusCode: 400, body: validationErrors }); } - console.log(JSON.stringify(request.body, null, 2)); - const { actions: actionsRest, anomaly_threshold: anomalyThreshold, @@ -173,7 +171,6 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void actions: throttle === 'rule' ? actions : [], // Only enable actions if throttle is rule, otherwise we are a notification and should not enable it, }); - console.log('bbb'); const ruleActions = await updateRulesNotifications({ ruleAlertId: createdRule.id, alertsClient, @@ -184,7 +181,6 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void name, }); - console.log('ccc'); const ruleStatuses = await ruleStatusSavedObjectsClientFactory(savedObjectsClient).find({ perPage: 1, sortField: 'statusDate', @@ -192,13 +188,11 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void search: `${createdRule.id}`, searchFields: ['alertId'], }); - console.log('ddd'); const [validated, errors] = transformValidate( createdRule, ruleActions, ruleStatuses.saved_objects[0] ); - console.log('eee', errors); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index b7e4aa446205d..75c4d75cedf1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -57,9 +57,7 @@ export const buildBulkBody = ({ tags, throttle, }); - // console.log('buildRule', JSON.stringify(rule, null, 2)); const signal = buildSignal(doc, rule); - // console.log('buildSignal', JSON.stringify(signal, null, 2)); const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, From 38942cd34b5645023ab1298059bba289b1542a8e Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 13:45:34 +0200 Subject: [PATCH 11/30] WIP --- ..._prepackaged_rules_type_dependents.test.ts | 24 +++++- .../schemas/request/create_rules_schema.ts | 1 - .../create_rules_type_dependents.test.ts | 22 +++++ .../import_rules_type_dependents.test.ts | 22 +++++ .../patch_rule_type_dependents.test.ts | 22 +++++ .../update_rules_type_dependents.test.ts | 22 +++++ .../schemas/response/rules_schema.ts | 1 + .../components/alerts_table/actions.tsx | 5 +- .../rules/step_define_rule/index.tsx | 42 +++++----- .../detection_engine/rules/create/helpers.ts | 4 +- .../signals/bulk_create_threshold_signals.ts | 4 +- .../detection_engine/signals/get_filter.ts | 82 +++++++++++-------- .../signals/signal_params_schema.mock.ts | 1 - 13 files changed, 187 insertions(+), 65 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.test.ts index 793d4b04ed0e5..f844d0e86e1f9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_type_dependents.test.ts @@ -8,7 +8,7 @@ import { AddPrepackagedRulesSchema } from './add_prepackaged_rules_schema'; import { addPrepackagedRuleValidateTypeDependents } from './add_prepackaged_rules_type_dependents'; import { getAddPrepackagedRulesSchemaMock } from './add_prepackaged_rules_schema.mock'; -describe('create_rules_type_dependents', () => { +describe('add_prepackaged_rules_type_dependents', () => { test('saved_id is required when type is saved_query and will not validate without out', () => { const schema: AddPrepackagedRulesSchema = { ...getAddPrepackagedRulesSchemaMock(), @@ -68,4 +68,26 @@ describe('create_rules_type_dependents', () => { const errors = addPrepackagedRuleValidateTypeDependents(schema); expect(errors).toEqual(['when "timeline_title" exists, "timeline_id" must also exist']); }); + + test('threshold is required when type is threshold and validates with it', () => { + const schema: AddPrepackagedRulesSchema = { + ...getAddPrepackagedRulesSchemaMock(), + type: 'threshold', + }; + const errors = addPrepackagedRuleValidateTypeDependents(schema); + expect(errors).toEqual(['when "type" is "threshold", "threshold" is required']); + }); + + test('threshold.value is required and has to be bigger than 0 when type is threshold and validates with it', () => { + const schema: AddPrepackagedRulesSchema = { + ...getAddPrepackagedRulesSchemaMock(), + type: 'threshold', + threshold: { + field: '', + value: -1, + }, + }; + const errors = addPrepackagedRuleValidateTypeDependents(schema); + expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index fc07ab1d912c4..308b3c24010fb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -29,7 +29,6 @@ import { type, Threat, threshold, - Threshold, ThrottleOrNull, note, Version, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts index ebf0b2e591ca9..43f0901912271 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts @@ -65,4 +65,26 @@ describe('create_rules_type_dependents', () => { const errors = createRuleValidateTypeDependents(schema); expect(errors).toEqual(['when "timeline_title" exists, "timeline_id" must also exist']); }); + + test('threshold is required when type is threshold and validates with it', () => { + const schema: CreateRulesSchema = { + ...getCreateRulesSchemaMock(), + type: 'threshold', + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual(['when "type" is "threshold", "threshold" is required']); + }); + + test('threshold.value is required and has to be bigger than 0 when type is threshold and validates with it', () => { + const schema: CreateRulesSchema = { + ...getCreateRulesSchemaMock(), + type: 'threshold', + threshold: { + field: '', + value: -1, + }, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.test.ts index f9b989c81e533..4b047ee6b7198 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_type_dependents.test.ts @@ -65,4 +65,26 @@ describe('import_rules_type_dependents', () => { const errors = importRuleValidateTypeDependents(schema); expect(errors).toEqual(['when "timeline_title" exists, "timeline_id" must also exist']); }); + + test('threshold is required when type is threshold and validates with it', () => { + const schema: ImportRulesSchema = { + ...getImportRulesSchemaMock(), + type: 'threshold', + }; + const errors = importRuleValidateTypeDependents(schema); + expect(errors).toEqual(['when "type" is "threshold", "threshold" is required']); + }); + + test('threshold.value is required and has to be bigger than 0 when type is threshold and validates with it', () => { + const schema: ImportRulesSchema = { + ...getImportRulesSchemaMock(), + type: 'threshold', + threshold: { + field: '', + value: -1, + }, + }; + const errors = importRuleValidateTypeDependents(schema); + expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts index a388e69332072..bafaf6f9e2203 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts @@ -78,4 +78,26 @@ describe('patch_rules_type_dependents', () => { const errors = patchRuleValidateTypeDependents(schema); expect(errors).toEqual(['either "id" or "rule_id" must be set']); }); + + test('threshold is required when type is threshold and validates with it', () => { + const schema: PatchRulesSchema = { + ...getPatchRulesSchemaMock(), + type: 'threshold', + }; + const errors = patchRuleValidateTypeDependents(schema); + expect(errors).toEqual(['when "type" is "threshold", "threshold" is required']); + }); + + test('threshold.value is required and has to be bigger than 0 when type is threshold and validates with it', () => { + const schema: PatchRulesSchema = { + ...getPatchRulesSchemaMock(), + type: 'threshold', + threshold: { + field: '', + value: -1, + }, + }; + const errors = patchRuleValidateTypeDependents(schema); + expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.test.ts index a63c8243cb5f1..91b11ea758e93 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_type_dependents.test.ts @@ -85,4 +85,26 @@ describe('update_rules_type_dependents', () => { const errors = updateRuleValidateTypeDependents(schema); expect(errors).toEqual(['either "id" or "rule_id" must be set']); }); + + test('threshold is required when type is threshold and validates with it', () => { + const schema: UpdateRulesSchema = { + ...getUpdateRulesSchemaMock(), + type: 'threshold', + }; + const errors = updateRuleValidateTypeDependents(schema); + expect(errors).toEqual(['when "type" is "threshold", "threshold" is required']); + }); + + test('threshold.value is required and has to be bigger than 0 when type is threshold and validates with it', () => { + const schema: UpdateRulesSchema = { + ...getUpdateRulesSchemaMock(), + type: 'threshold', + threshold: { + field: '', + value: -1, + }, + }; + const errors = updateRuleValidateTypeDependents(schema); + expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 942abb370e949..9ea24e651f6ab 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -233,6 +233,7 @@ export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { if (typeAndTimelineOnly.type === 'threshold') { return [ + // t.exact(t.type({ saved_id: dependentRulesSchema.props.saved_id })), t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), t.exact(t.type({ threshold: dependentRulesSchema.props.threshold })), diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 875b978101bf3..8a54d7041ce62 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -32,6 +32,7 @@ import { replaceTemplateFieldFromMatchFilters, replaceTemplateFieldFromDataProviders, } from './helpers'; +import { KueryFilterQueryKind } from '../../../common/store'; export const getUpdateAlertsQuery = (eventIds: Readonly) => { return { @@ -238,7 +239,7 @@ export const sendAlertToTimelineAction = async ({ filterQuery: { kuery: { kind: ecsData.signal?.rule?.language?.length - ? ecsData.signal?.rule?.language[0] + ? (ecsData.signal?.rule?.language[0] as KueryFilterQueryKind) : 'kuery', expression: ecsData.signal?.rule?.query?.length ? ecsData.signal?.rule?.query[0] : '', }, @@ -248,7 +249,7 @@ export const sendAlertToTimelineAction = async ({ }, filterQueryDraft: { kind: ecsData.signal?.rule?.language?.length - ? ecsData.signal?.rule?.language[0] + ? (ecsData.signal?.rule?.language[0] as KueryFilterQueryKind) : 'kuery', expression: ecsData.signal?.rule?.query?.length ? ecsData.signal?.rule?.query[0] : '', }, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index aac6b3f8d5ee7..28b6e4f3a5b23 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -39,7 +39,6 @@ import { getCategorizedFieldNames } from '../../../../timelines/components/edit_ import { Field, Form, - FormDataProvider, getUseField, UseField, useForm, @@ -157,6 +156,27 @@ const StepDefineRuleComponent: FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [form]); + useEffect(() => { + const subscription = form.subscribe(({ data, isValid, ...rest }) => { + const { index, ruleType } = data.raw; + + if (index != null) { + if (deepEqual(index, indicesConfig) && indexModified) { + setIndexModified(false); + } else if (!deepEqual(index, indicesConfig) && !indexModified) { + setMyStepData((currentValue) => ({ ...currentValue, index })); + setIndexModified(true); + } + } + + if (ruleType !== localRuleType) { + setLocalRuleType(ruleType); + clearErrors(); + } + }); + return subscription.unsubscribe; + }, [clearErrors, form, indexModified, indicesConfig, localRuleType]); + const handleResetIndices = useCallback(() => { const indexField = form.getFields().index; indexField.setValue(indicesConfig); @@ -313,26 +333,6 @@ const StepDefineRuleComponent: FC = ({ dataTestSubj: 'detectionEngineStepDefineRuleTimeline', }} /> - - {({ index, ruleType }) => { - if (index != null) { - if (deepEqual(index, indicesConfig) && indexModified) { - setIndexModified(false); - } else if (!deepEqual(index, indicesConfig) && !indexModified) { - // TODO: refactor to form.subscribe() - setMyStepData((currentValue) => ({ ...currentValue, index })); - setIndexModified(true); - } - } - - if (ruleType !== localRuleType) { - setLocalRuleType(ruleType); - clearErrors(); - } - - return null; - }} - diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 3805a14995e7f..bd39d7b2ce4f4 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -95,8 +95,8 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold.field[0] ?? '', - value: parseInt(ruleFields.threshold.value, 10) ?? 100, + field: ruleFields.threshold?.field[0] ?? '', + value: parseInt(ruleFields.threshold?.value, 10) ?? 100, }, }), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index fea3ee59b3a4b..9203b7fcd1f2d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -135,7 +135,7 @@ const transformThresholdResultsToEcs = ( results: ThresholdResults, filter: unknown, threshold: Threshold -): SearchResponse => { +): SearchResponse => { console.log( 'transformThresholdResultsToEcs', // JSON.stringify(results), @@ -169,7 +169,7 @@ export const bulkCreateThresholdSignals = async ( const ecsResults = transformThresholdResultsToEcs( thresholdResults, params.filter, - params.ruleParams.threshold + params.ruleParams.threshold! ); console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index c5fd42e12a075..67dc1d50eefcd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -49,50 +49,62 @@ export const getFilter = async ({ query, lists, }: GetFilterArgs): Promise => { + const queryFilter = () => { + if (query != null && language != null && index != null) { + return getQueryFilter(query, language, filters || [], index, lists); + } else { + throw new BadRequestError('query, filters, and index parameter should be defined'); + } + }; + + const savedQueryFilter = async () => { + if (savedId != null && index != null) { + try { + // try to get the saved object first + const savedObject = await services.savedObjectsClient.get( + 'query', + savedId + ); + return getQueryFilter( + savedObject.attributes.query.query, + savedObject.attributes.query.language, + savedObject.attributes.filters, + index, + lists + ); + } catch (err) { + // saved object does not exist, so try and fall back if the user pushed + // any additional language, query, filters, etc... + if (query != null && language != null && index != null) { + return getQueryFilter(query, language, filters || [], index, lists); + } else { + // user did not give any additional fall back mechanism for generating a rule + // rethrow error for activity monitoring + throw err; + } + } + } else { + throw new BadRequestError('savedId parameter should be defined'); + } + }; + switch (type) { - case 'threshold': + case 'threshold': { + return savedId != null ? savedQueryFilter() : queryFilter(); + } case 'query': { - if (query != null && language != null && index != null) { - return getQueryFilter(query, language, filters || [], index, lists); - } else { - throw new BadRequestError('query, filters, and index parameter should be defined'); - } + return queryFilter(); } case 'saved_query': { - if (savedId != null && index != null) { - try { - // try to get the saved object first - const savedObject = await services.savedObjectsClient.get( - 'query', - savedId - ); - return getQueryFilter( - savedObject.attributes.query.query, - savedObject.attributes.query.language, - savedObject.attributes.filters, - index, - lists - ); - } catch (err) { - // saved object does not exist, so try and fall back if the user pushed - // any additional language, query, filters, etc... - if (query != null && language != null && index != null) { - return getQueryFilter(query, language, filters || [], index, lists); - } else { - // user did not give any additional fall back mechanism for generating a rule - // rethrow error for activity monitoring - throw err; - } - } - } else { - throw new BadRequestError('savedId parameter should be defined'); - } + return savedQueryFilter(); } case 'machine_learning': { throw new BadRequestError( 'Unsupported Rule of type "machine_learning" supplied to getFilter' ); } + default: { + return assertUnreachable(type); + } } - return assertUnreachable(type); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index 3e8865d107979..0c56ed300cb48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -43,7 +43,6 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ severity: 'high', severityMapping: null, threat: null, - threshold: null, timelineId: null, timelineTitle: null, timestampOverride: null, From e92e003abf9c0d9b13eb709cf9bcc4ef7d5abeb7 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 21:41:30 +0200 Subject: [PATCH 12/30] WIP --- .../schemas/response/rules_schema.ts | 6 +- .../rules/description_step/helpers.tsx | 9 ++ .../rules/description_step/index.tsx | 5 +- .../rules/description_step/translations.tsx | 14 ++ .../rules/step_define_rule/index.tsx | 131 ++++++++---------- .../rules/step_define_rule/schema.tsx | 7 + .../rules/threshold_input/index.tsx | 106 +++++++------- .../rules/threshold_input/translations.ts | 14 ++ .../detection_engine/rules/create/helpers.ts | 5 +- .../pages/detection_engine/rules/helpers.tsx | 2 +- .../public/shared_imports.ts | 1 + 11 files changed, 172 insertions(+), 128 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/translations.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 9ea24e651f6ab..dedb798a8e239 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -207,7 +207,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (typeAndTimelineOnly.type === 'query' || typeAndTimelineOnly.type === 'saved_query') { + if (['query', 'saved_query', 'threshold'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), @@ -233,10 +233,8 @@ export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { if (typeAndTimelineOnly.type === 'threshold') { return [ - // t.exact(t.type({ saved_id: dependentRulesSchema.props.saved_id })), - t.exact(t.type({ query: dependentRulesSchema.props.query })), - t.exact(t.type({ language: dependentRulesSchema.props.language })), t.exact(t.type({ threshold: dependentRulesSchema.props.threshold })), + t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), ]; } else { return []; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 28117d7ba976e..b381d20715468 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -304,3 +304,12 @@ export const buildRuleTypeDescription = (label: string, ruleType: RuleType): Lis return assertUnreachable(ruleType); } }; + +export const buildThresholdDescription = (label: string, threshold: Threshold): ListItems[] => [ + { + title: label, + description: isEmpty(threshold.field[0]) + ? `${i18n.THRESHOLD_RESULTS_ALL} >= ${threshold.value}` + : `${i18n.THRESHOLD_RESULTS_AGGREGATED_BY} ${threshold.field[0]} >= ${threshold.value}`, + }, +]; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 8d6fb1e3bc7b1..d1911b14113f4 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -35,6 +35,7 @@ import { buildUrlsDescription, buildNoteDescription, buildRuleTypeDescription, + buildThresholdDescription, } from './helpers'; import { useSiemJobs } from '../../../../common/components/ml_popover/hooks/use_siem_jobs'; import { buildMlJobDescription } from './ml_job_description'; @@ -180,9 +181,9 @@ export const getDescriptionItem = ( ); return buildThreatDescription({ label, threat }); } else if (field === 'threshold') { - console.error('thresholffi', field); + const threshold = get(field, data); - return []; + return buildThresholdDescription(label, threshold); } else if (field === 'references') { const urls: string[] = get(field, data); return buildUrlsDescription(label, urls); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx index 4611d7bb8e319..76217964a87cb 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx @@ -61,3 +61,17 @@ export const ML_JOB_STOPPED = i18n.translate( defaultMessage: 'Stopped', } ); + +export const THRESHOLD_RESULTS_ALL = i18n.translate( + 'xpack.securitySolution.detectionEngine.ruleDescription.thresholdResultsAllDescription', + { + defaultMessage: 'All results', + } +); + +export const THRESHOLD_RESULTS_AGGREGATED_BY = i18n.translate( + 'xpack.securitySolution.detectionEngine.ruleDescription.thresholdResultsAggregatedByDescription', + { + defaultMessage: 'Results aggregated by', + } +); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 28b6e4f3a5b23..942c0d547769b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiButtonEmpty, EuiFormRow, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiButtonEmpty, EuiFormRow } from '@elastic/eui'; import React, { FC, memo, useCallback, useState, useEffect } from 'react'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; @@ -35,12 +35,14 @@ import { MlJobSelect } from '../ml_job_select'; import { PickTimeline } from '../pick_timeline'; import { StepContentWrapper } from '../step_content_wrapper'; import { NextStep } from '../next_step'; -import { getCategorizedFieldNames } from '../../../../timelines/components/edit_data_provider/helpers'; +import { ThresholdInput } from '../threshold_input'; import { Field, Form, getUseField, UseField, + UseMultiFields, + FormDataProvider, useForm, FormSchema, } from '../../../../shared_imports'; @@ -52,7 +54,6 @@ const CommonUseField = getUseField({ component: Field }); interface StepDefineRuleProps extends RuleStepProps { defaultValues?: DefineStepRule | null; } -const FIELD_COMBO_BOX_WIDTH = 195; const stepDefineDefaultValue: DefineStepRule = { anomalyThreshold: 50, @@ -66,8 +67,8 @@ const stepDefineDefaultValue: DefineStepRule = { saved_id: undefined, }, threshold: { - field: [''], - value: 100, + field: [], + value: 200, }, timeline: { id: null, @@ -89,11 +90,11 @@ MyLabelButton.defaultProps = { flush: 'right', }; -const RuleTypeEuiFormRow = styled(EuiFormRow).attrs<{ isVisible: boolean }>(({ isVisible }) => ({ +const RuleTypeEuiFormRow = styled(EuiFormRow).attrs<{ $isVisible: boolean }>(({ $isVisible }) => ({ style: { - display: isVisible ? 'flex' : 'none', + display: $isVisible ? 'flex' : 'none', }, -}))<{ isVisible: boolean }>``; +}))<{ $isVisible: boolean }>``; const StepDefineRuleComponent: FC = ({ addPadding = false, @@ -156,27 +157,6 @@ const StepDefineRuleComponent: FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [form]); - useEffect(() => { - const subscription = form.subscribe(({ data, isValid, ...rest }) => { - const { index, ruleType } = data.raw; - - if (index != null) { - if (deepEqual(index, indicesConfig) && indexModified) { - setIndexModified(false); - } else if (!deepEqual(index, indicesConfig) && !indexModified) { - setMyStepData((currentValue) => ({ ...currentValue, index })); - setIndexModified(true); - } - } - - if (ruleType !== localRuleType) { - setLocalRuleType(ruleType); - clearErrors(); - } - }); - return subscription.unsubscribe; - }, [clearErrors, form, indexModified, indicesConfig, localRuleType]); - const handleResetIndices = useCallback(() => { const indexField = form.getFields().index; indexField.setValue(indicesConfig); @@ -190,6 +170,17 @@ const StepDefineRuleComponent: FC = ({ setOpenTimelineSearch(false); }, []); + const ThresholdInputChildren = useCallback( + ({ thresholdField, thresholdValue }) => ( + + ), + [browserFields] + ); + return isReadOnlyView ? ( = ({ isMlAdmin: hasMlAdminPermissions(mlCapabilities), }} /> - + <> = ({ }} /> - - + + <> = ({ - - - - - {'>='} - - - - + <> + + {ThresholdInputChildren} + + = ({ dataTestSubj: 'detectionEngineStepDefineRuleTimeline', }} /> + + {({ index, ruleType }) => { + if (index != null) { + if (deepEqual(index, indicesConfig) && indexModified) { + setIndexModified(false); + } else if (!deepEqual(index, indicesConfig) && !indexModified) { + setMyStepData((currentValue) => ({ ...currentValue, index })); + setIndexModified(true); + } + } + + if (ruleType !== localRuleType) { + setLocalRuleType(ruleType); + clearErrors(); + } + return null; + }} + diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index 0899784830ec0..67d795ccf90f0 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -173,6 +173,12 @@ export const schema: FormSchema = { ), }, threshold: { + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldThresholdLabel', + { + defaultMessage: 'Threshold', + } + ), field: { type: FIELD_TYPES.COMBO_BOX, label: i18n.translate( @@ -189,6 +195,7 @@ export const schema: FormSchema = { ), }, value: { + type: FIELD_TYPES.NUMBER, label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdValueLabel', { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 7d9385312c115..8a61d19efba45 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -4,60 +4,72 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useCallback } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiRange, EuiFormRow } from '@elastic/eui'; +import React, { useMemo } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import styled from 'styled-components'; -import { FieldHook, getUseField, Field } from '../../../../shared_imports'; +import { BrowserFields } from '../../../../common/containers/source'; +import { getCategorizedFieldNames } from '../../../../timelines/components/edit_data_provider/helpers'; +import { FieldHook, Field } from '../../../../shared_imports'; +import { THRESHOLD_FIELD_PLACEHOLDER } from './translations'; + +const FIELD_COMBO_BOX_WIDTH = 460; interface ThresholdInputProps { - field: FieldHook; + thresholdField: FieldHook; + thresholdValue: FieldHook; + browserFields: BrowserFields; } -// type Event = React.ChangeEvent; -// type EventArg = Event | React.MouseEvent; -const CommonUseField = getUseField({ component: Field }); +const OperatorWrapper = styled(EuiFlexItem)` + align-self: center; +`; + +const fieldDescribedByIds = ['detectionEngineStepDefineRuleThresholdField']; +const valueDescribedByIds = ['detectionEngineStepDefineRuleThresholdValue']; -export const ThresholdInput = ({ field }: ThresholdInputProps) => { - // const threshold = field.value as number; - // const onThresholdChange = useCallback( - // (event: EventArg) => { - // const thresholdValue = Number((event as Event).target.value); - // field.setValue(thresholdValue); - // }, - // [field] - // ); +const ThresholdInputComponent: React.FC = ({ + thresholdField, + thresholdValue, + browserFields, +}: ThresholdInputProps) => { + const fieldEuiFieldProps = useMemo( + () => ({ + fullWidth: true, + singleSelection: { asPlainText: true }, + noSuggestions: false, + options: getCategorizedFieldNames(browserFields), + placeholder: THRESHOLD_FIELD_PLACEHOLDER, + onCreateOption: undefined, + style: { width: `${FIELD_COMBO_BOX_WIDTH}px` }, + }), + [browserFields] + ); return ( - - - - - - {'>='} - - - - - + + + + + {'>='} + + + + ); }; + +export const ThresholdInput = React.memo(ThresholdInputComponent); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/translations.ts b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/translations.ts new file mode 100644 index 0000000000000..228848ef12130 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/translations.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const THRESHOLD_FIELD_PLACEHOLDER = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.thresholdField.thresholdFieldPlaceholderText', + { + defaultMessage: 'All results', + } +); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index bd39d7b2ce4f4..6178beb7652b1 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -70,6 +70,7 @@ export const filterRuleFieldsForType = (fields: T, type: R export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => { const ruleFields = filterRuleFieldsForType(defineStepData, defineStepData.ruleType); + console.error('formatDefineStepData', ruleFields); const { ruleType, timeline } = ruleFields; const baseFields = { type: ruleType, @@ -95,8 +96,8 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold?.field[0] ?? '', - value: parseInt(ruleFields.threshold?.value, 10) ?? 100, + field: ruleFields.threshold?.field, + value: ruleFields.threshold?.value, }, }), }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 9bc7acbe91e66..f92c2a56042bb 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -90,7 +90,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ value: parseInt(rule.threshold.value, 10), } : { - field: [''], + field: [], value: 100, }, }); diff --git a/x-pack/plugins/security_solution/public/shared_imports.ts b/x-pack/plugins/security_solution/public/shared_imports.ts index 472006a9e55b1..79778e4c54d96 100644 --- a/x-pack/plugins/security_solution/public/shared_imports.ts +++ b/x-pack/plugins/security_solution/public/shared_imports.ts @@ -16,6 +16,7 @@ export { FormHook, FormSchema, UseField, + UseMultiFields, useForm, ValidationFunc, VALIDATION_TYPES, From feeae46bb47bfea0a8d2542e70a8539aa715e367 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 11 Jul 2020 21:46:12 +0200 Subject: [PATCH 13/30] test --- .../detections/pages/detection_engine/rules/helpers.test.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index f8969f06c8ef6..5de3fb874841e 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -74,6 +74,10 @@ describe('rule helpers', () => { ], saved_id: 'test123', }, + threshold: { + field: [], + value: 100, + }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', title: 'Titled timeline', From fe19fa10bbd83e2be81b34453aa2e98624bffc48 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 11:50:28 +0200 Subject: [PATCH 14/30] WIP --- .../schemas/response/rules_schema.ts | 3 +- .../rules/description_step/helpers.tsx | 1 + .../detection_engine/rules/create/helpers.ts | 37 ++++++++---- .../detection_engine/rules/helpers.test.tsx | 10 +++- .../pages/detection_engine/rules/helpers.tsx | 13 ++-- .../pages/detection_engine/rules/types.ts | 7 ++- .../routes/__mocks__/request_responses.ts | 1 + .../signals/bulk_create_threshold_signals.ts | 59 +++++++++++-------- .../signals/signal_rule_alert_type.ts | 1 - 9 files changed, 84 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index dedb798a8e239..4bd18a13e4ebb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -45,7 +45,6 @@ import { type, threat, threshold, - thresholdOrUndefined, throttle, job_status, status_date, @@ -127,7 +126,7 @@ export const dependentRulesSchema = t.partial({ machine_learning_job_id, // Threshold fields - threshold: thresholdOrUndefined, + threshold, }); /** diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index b381d20715468..24e6b468b6a92 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -19,6 +19,7 @@ import { isEmpty } from 'lodash/fp'; import React from 'react'; import styled from 'styled-components'; +import { Threshold } from '../../../../../common/detection_engine/schemas/common/schemas'; import { RuleType } from '../../../../../common/detection_engine/types'; import { esFilters } from '../../../../../../../../src/plugins/data/public'; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 6178beb7652b1..e7e8c5062d354 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -51,26 +51,35 @@ export interface RuleFields { queryBar: unknown; index: unknown; ruleType: unknown; + threshold?: unknown; } -type QueryRuleFields = Omit; +type QueryRuleFields = Omit; +type ThresholdRuleFields = Omit; type MlRuleFields = Omit; -const isMlFields = (fields: QueryRuleFields | MlRuleFields): fields is MlRuleFields => - has('anomalyThreshold', fields); +const isMlFields = ( + fields: QueryRuleFields | MlRuleFields | ThresholdRuleFields +): fields is MlRuleFields => has('anomalyThreshold', fields); + +const isThresholdFields = ( + fields: QueryRuleFields | MlRuleFields | ThresholdRuleFields +): fields is ThresholdRuleFields => has('threshold', fields); export const filterRuleFieldsForType = (fields: T, type: RuleType) => { if (isMlRule(type)) { const { index, queryBar, ...mlRuleFields } = fields; return mlRuleFields; + } else if (type === 'threshold') { + const { anomalyThreshold, machineLearningJobId, ...thresholdRuleFields } = fields; + return thresholdRuleFields; } else { - const { anomalyThreshold, machineLearningJobId, ...queryRuleFields } = fields; + const { anomalyThreshold, machineLearningJobId, threshold, ...queryRuleFields } = fields; return queryRuleFields; } }; export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => { const ruleFields = filterRuleFieldsForType(defineStepData, defineStepData.ruleType); - console.error('formatDefineStepData', ruleFields); const { ruleType, timeline } = ruleFields; const baseFields = { type: ruleType, @@ -86,20 +95,28 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep anomaly_threshold: ruleFields.anomalyThreshold, machine_learning_job_id: ruleFields.machineLearningJobId, } - : { + : isThresholdFields(ruleFields) + ? { index: ruleFields.index, filters: ruleFields.queryBar?.filters, language: ruleFields.queryBar?.query?.language, query: ruleFields.queryBar?.query?.query as string, saved_id: ruleFields.queryBar?.saved_id, - ...(ruleType === 'query' && - ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold?.field, - value: ruleFields.threshold?.value, + field: ruleFields.threshold?.field ?? [], + value: ruleFields.threshold?.value ?? 0, }, }), + } + : { + index: ruleFields.index, + filters: ruleFields.queryBar?.filters, + language: ruleFields.queryBar?.query?.language, + query: ruleFields.queryBar?.query?.query as string, + saved_id: ruleFields.queryBar?.saved_id, + ...(ruleType === 'query' && + ruleFields.queryBar?.saved_id && { type: 'saved_query' as RuleType }), }; return { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 5de3fb874841e..f46d6f99c8b2c 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -76,7 +76,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: 100, + value: undefined, }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', @@ -210,6 +210,10 @@ describe('rule helpers', () => { filters: [], saved_id: "Garrett's IP", }, + threshold: { + field: [], + value: undefined, + }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', title: 'Untitled timeline', @@ -239,6 +243,10 @@ describe('rule helpers', () => { filters: [], saved_id: undefined, }, + threshold: { + field: [], + value: undefined, + }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', title: 'Untitled timeline', diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index f92c2a56042bb..f5a3884408143 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -84,15 +84,10 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ id: rule.timeline_id ?? null, title: rule.timeline_title ?? null, }, - threshold: rule.threshold - ? { - field: [rule.threshold.field], - value: parseInt(rule.threshold.value, 10), - } - : { - field: [], - value: 100, - }, + threshold: { + field: rule.threshold?.field ? [rule.threshold.field] : [], + value: rule.threshold?.value, + }, }); export const getScheduleStepsData = (rule: Rule): ScheduleStepRule => { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index c6eab9c6f5c05..cadd9c908bd98 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -102,7 +102,7 @@ export interface DefineStepRule extends StepRuleData { timeline: FieldValueTimeline; threshold?: { field: string[]; - value: number; + value?: number; }; } @@ -127,7 +127,10 @@ export interface DefineStepRuleJson { saved_id?: string; query?: string; language?: string; - threshold?: Threshold; + threshold?: { + field: string; + value: number; + }; timeline_id?: string; timeline_title?: string; type: RuleType; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 9ca102b437511..29c56e8ed80b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -394,6 +394,7 @@ export const getResult = (): RuleAlertType => ({ ], }, ], + threshold: undefined, timestampOverride: undefined, references: ['http://www.example.com', 'https://ww.example.com'], note: '# Investigative notes', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 9203b7fcd1f2d..4e99995331d56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -7,7 +7,6 @@ import uuid from 'uuid'; import { reduce, get, isEmpty } from 'lodash/fp'; import set from 'set-value'; -import { SearchResponse } from 'elasticsearch'; import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../src/core/server'; @@ -16,6 +15,7 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { SignalSearchResponse } from './types'; +import { SearchResponse } from '../../types'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; @@ -38,9 +38,16 @@ interface BulkCreateThresholdSignalsParams { throttle: string; } -interface ThresholdResults {} +type ThresholdResults = SignalSearchResponse; -const getNestedQueryFilters = (filtersObj: unknown) => { +interface FilterObject { + bool?: { + filter?: FilterObject | FilterObject[]; + should?: Array>>; + }; +} + +const getNestedQueryFilters = (filtersObj: FilterObject): Record => { if (Array.isArray(filtersObj.bool?.filter)) { return reduce( (acc, filterItem) => { @@ -53,10 +60,13 @@ const getNestedQueryFilters = (filtersObj: unknown) => { return acc; }, {}, - filtersObj.bool.filter + filtersObj.bool?.filter ); } else { - return filtersObj.bool.should && filtersObj.bool.should[0].match; + return ( + (filtersObj.bool?.should && filtersObj.bool?.should[0] && filtersObj.bool.should[0].match) ?? + {} + ); } }; @@ -85,12 +95,14 @@ const getThresholdSignalQueryFields = (filter: unknown) => { }; const getTransformedHits = ( - results, + results: ThresholdResults, threshold: Threshold, signalQueryFields: Record ) => { if (isEmpty(threshold.field)) { - if (results.hits.total.value < threshold.value) { + const totalResults = results.hits.total; + + if (totalResults < threshold.value) { return []; } @@ -104,7 +116,7 @@ const getTransformedHits = ( _index: '', _id: uuid.v4(), _source: source, - threshold_count: results.hits.total.value, + threshold_count: totalResults, }, ]; } @@ -113,22 +125,23 @@ const getTransformedHits = ( return []; } - return results.aggregations.threshold.buckets.map(({ key, doc_count }) => { - const source = { - '@timestamp': new Date().toISOString(), - threshold_count: doc_count, - ...signalQueryFields, - }; + return results.aggregations.threshold.buckets.map( + ({ key, doc_count }: { key: string; doc_count: number }) => { + const source = { + '@timestamp': new Date().toISOString(), + threshold_count: doc_count, + ...signalQueryFields, + }; - set(source, threshold.field, key); + set(source, threshold.field, key); - return { - // ...rest, - _index: '', - _id: uuid.v4(), - _source: source, - }; - }); + return { + _index: '', + _id: uuid.v4(), + _source: source, + }; + } + ); }; const transformThresholdResultsToEcs = ( @@ -157,7 +170,7 @@ const transformThresholdResultsToEcs = ( }, }; - set(thresholdResults, 'this.total.value', transformedHits.length); + set(thresholdResults, 'results.hits.total', transformedHits.length); return thresholdResults; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 9add39673ca49..6312bd590b7d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -291,7 +291,6 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } - // console.log('resulttttt', result); } else { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ From ed681d6f25e07f86bb07f1577e4b3dc8c186e89c Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 14:20:35 +0200 Subject: [PATCH 15/30] WIP --- .../rules/description_step/helpers.test.tsx | 12 ++ .../rules/description_step/helpers.tsx | 14 +- .../rules/description_step/index.test.tsx | 48 +++++- .../rules/description_step/index.tsx | 20 ++- .../rules/select_rule_type/index.tsx | 88 ++++------ .../select_rule_type/ml_card_description.tsx | 48 ++++++ .../rules/threshold_input/index.tsx | 5 + .../rules/all/__mocks__/mock.ts | 4 + .../detection_engine/rules/create/helpers.ts | 2 +- .../pages/detection_engine/rules/helpers.tsx | 2 +- .../pages/detection_engine/rules/types.ts | 7 +- .../detection_engine/routes/rules/utils.ts | 2 +- .../signals/build_events_query.test.ts | 154 ++++++++++++++++++ .../signals/build_events_query.ts | 2 - .../signals/bulk_create_threshold_signals.ts | 25 +-- .../signals/signal_rule_alert_type.ts | 7 +- 16 files changed, 342 insertions(+), 98 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx index b82d1c0a36ab2..41ee91845a8ec 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx @@ -403,5 +403,17 @@ describe('helpers', () => { expect(result.description).toEqual('Query'); }); + + it('returns the label for a threshold type', () => { + const [result]: ListItems[] = buildRuleTypeDescription('Test label', 'threshold'); + + expect(result.title).toEqual('Test label'); + }); + + it('returns a humanized description for a threshold type', () => { + const [result]: ListItems[] = buildRuleTypeDescription('Test label', 'threshold'); + + expect(result.description).toEqual('Threshold'); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 24e6b468b6a92..8393f2230dcfe 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -133,10 +133,10 @@ export const buildThreatDescription = ({ label, threat }: BuildThreatDescription {tactic != null ? tactic.text : ''} - {singleThreat.technique.map((technique) => { + {singleThreat.technique.map((technique, listIndex) => { const myTechnique = techniquesOptions.find((t) => t.id === technique.id); return ( - + [ { title: label, - description: isEmpty(threshold.field[0]) - ? `${i18n.THRESHOLD_RESULTS_ALL} >= ${threshold.value}` - : `${i18n.THRESHOLD_RESULTS_AGGREGATED_BY} ${threshold.field[0]} >= ${threshold.value}`, + description: ( + <> + {isEmpty(threshold.field[0]) + ? `${i18n.THRESHOLD_RESULTS_ALL} >= ${threshold.value}` + : `${i18n.THRESHOLD_RESULTS_AGGREGATED_BY} ${threshold.field[0]} >= ${threshold.value}`} + + ), }, ]; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx index 0a7e666d65aef..5a2a44a284e3b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.test.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import React from 'react'; -import { shallow } from 'enzyme'; +import { shallow, mount } from 'enzyme'; import { StepRuleDescriptionComponent, @@ -367,6 +367,52 @@ describe('description_step', () => { }); }); + describe('threshold', () => { + test('returns threshold description when threshold exist and field is empty', () => { + const mockThreshold = { + isNew: false, + threshold: { + field: [''], + value: 100, + }, + }; + const result: ListItems[] = getDescriptionItem( + 'threshold', + 'Threshold label', + mockThreshold, + mockFilterManager + ); + + expect(result[0].title).toEqual('Threshold label'); + expect(React.isValidElement(result[0].description)).toBeTruthy(); + expect(mount(result[0].description as React.ReactElement).html()).toContain( + 'All results >= 100' + ); + }); + + test('returns threshold description when threshold exist and field is set', () => { + const mockThreshold = { + isNew: false, + threshold: { + field: ['user.name'], + value: 100, + }, + }; + const result: ListItems[] = getDescriptionItem( + 'threshold', + 'Threshold label', + mockThreshold, + mockFilterManager + ); + + expect(result[0].title).toEqual('Threshold label'); + expect(React.isValidElement(result[0].description)).toBeTruthy(); + expect(mount(result[0].description as React.ReactElement).html()).toContain( + 'Results aggregated by user.name >= 100' + ); + }); + }); + describe('references', () => { test('returns array of ListItems when references exist', () => { const result: ListItems[] = getDescriptionItem( diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index d1911b14113f4..350d9b13c3bc4 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -95,14 +95,17 @@ export const StepRuleDescriptionComponent: React.FC = if (columns === 'multi') { return ( - {chunk(Math.ceil(listItems.length / 2), listItems).map((chunkListItems, index) => ( - - - - ))} + {chunk(Math.ceil(listItems.length / 2), listItems).map((chunkListItems, index) => { + console.log('chunkListItems', chunkListItems); + return ( + + + + ); + })} ); } @@ -182,7 +185,6 @@ export const getDescriptionItem = ( return buildThreatDescription({ label, threat }); } else if (field === 'threshold') { const threshold = get(field, data); - return buildThresholdDescription(label, threshold); } else if (field === 'references') { const urls: string[] = get(field, data); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx index 11d06a5451f65..a8d72f5545c93 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx @@ -4,55 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useCallback } from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { - EuiCard, - EuiFlexGrid, - EuiFlexItem, - EuiFormRow, - EuiIcon, - EuiLink, - EuiText, -} from '@elastic/eui'; +import React, { useCallback, useMemo } from 'react'; +import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiFormRow, EuiIcon } from '@elastic/eui'; import { isMlRule } from '../../../../../common/machine_learning/helpers'; import { RuleType } from '../../../../../common/detection_engine/types'; import { FieldHook } from '../../../../shared_imports'; import { useKibana } from '../../../../common/lib/kibana'; import * as i18n from './translations'; +import { MlCardDescription } from './ml_card_description'; const isThresholdRule = (ruleType: RuleType) => ruleType === 'threshold'; -const MlCardDescription = ({ - subscriptionUrl, - hasValidLicense = false, -}: { - subscriptionUrl: string; - hasValidLicense?: boolean; -}) => ( - - {hasValidLicense ? ( - i18n.ML_TYPE_DESCRIPTION - ) : ( - - - - ), - }} - /> - )} - -); - interface SelectRuleTypeProps { describedByIds?: string[]; field: FieldHook; @@ -83,6 +46,33 @@ export const SelectRuleType: React.FC = ({ path: '#/management/stack/license_management', }); + const querySelectableConfig = useMemo( + () => ({ + isDisabled: isReadOnly, + onClick: setQuery, + isSelected: !isMlRule(ruleType) && !isThresholdRule(ruleType), + }), + [isReadOnly, ruleType, setQuery] + ); + + const mlSelectableConfig = useMemo( + () => ({ + isDisabled: mlCardDisabled, + onClick: setMl, + isSelected: isMlRule(ruleType), + }), + [mlCardDisabled, ruleType, setMl] + ); + + const thresholdSelectableConfig = useMemo( + () => ({ + isDisabled: isReadOnly, + onClick: setThreshold, + isSelected: isThresholdRule(ruleType), + }), + [isReadOnly, ruleType, setThreshold] + ); + return ( = ({ title={i18n.QUERY_TYPE_TITLE} description={i18n.QUERY_TYPE_DESCRIPTION} icon={} - selectable={{ - isDisabled: isReadOnly, - onClick: setQuery, - isSelected: !isMlRule(ruleType) && !isThresholdRule(ruleType), - }} + selectable={querySelectableConfig} /> @@ -113,11 +99,7 @@ export const SelectRuleType: React.FC = ({ } icon={} isDisabled={mlCardDisabled} - selectable={{ - isDisabled: mlCardDisabled, - onClick: setMl, - isSelected: isMlRule(ruleType), - }} + selectable={mlSelectableConfig} /> @@ -126,11 +108,7 @@ export const SelectRuleType: React.FC = ({ title={i18n.THRESHOLD_TYPE_TITLE} description={i18n.THRESHOLD_TYPE_DESCRIPTION} icon={} - selectable={{ - isDisabled: isReadOnly, - onClick: setThreshold, - isSelected: isThresholdRule(ruleType), - }} + selectable={thresholdSelectableConfig} /> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx new file mode 100644 index 0000000000000..2171c93e47d63 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiText, EuiLink } from '@elastic/eui'; +import React from 'react'; + +import { ML_TYPE_DESCRIPTION } from './translations'; + +interface MlCardDescriptionProps { + subscriptionUrl: string; + hasValidLicense?: boolean; +} + +const MlCardDescriptionComponent: React.FC = ({ + subscriptionUrl, + hasValidLicense = false, +}) => ( + + {hasValidLicense ? ( + ML_TYPE_DESCRIPTION + ) : ( + + + + ), + }} + /> + )} + +); + +MlCardDescriptionComponent.displayName = 'MlCardDescriptionComponent'; + +export const MlCardDescription = React.memo(MlCardDescriptionComponent); + +MlCardDescription.displayName = 'MlCardDescription'; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 8a61d19efba45..89ca8c9526ecf 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -15,6 +15,11 @@ import { THRESHOLD_FIELD_PLACEHOLDER } from './translations'; const FIELD_COMBO_BOX_WIDTH = 460; +export interface FieldValueThreshold { + field: string[]; + value: number; +} + interface ThresholdInputProps { thresholdField: FieldHook; thresholdValue: FieldHook; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 2b86abf4255c6..22704e2a70501 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -213,6 +213,10 @@ export const mockDefineStepRule = (isNew = false): DefineStepRule => ({ id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', title: 'Titled timeline', }, + threshold: { + field: [''], + value: 100, + }, }); export const mockScheduleStepRule = (isNew = false): ScheduleStepRule => ({ diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index e7e8c5062d354..6c3474185d9f6 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -104,7 +104,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep saved_id: ruleFields.queryBar?.saved_id, ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold?.field ?? [], + field: ruleFields.threshold?.field[0] ?? '', value: ruleFields.threshold?.value ?? 0, }, }), diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index f5a3884408143..2478305b69992 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -86,7 +86,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ }, threshold: { field: rule.threshold?.field ? [rule.threshold.field] : [], - value: rule.threshold?.value, + value: rule.threshold?.value ?? 100, }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index cadd9c908bd98..e7daff0947b0d 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -10,6 +10,7 @@ import { Filter } from '../../../../../../../../src/plugins/data/common'; import { FormData, FormHook } from '../../../../shared_imports'; import { FieldValueQueryBar } from '../../../components/rules/query_bar'; import { FieldValueTimeline } from '../../../components/rules/pick_timeline'; +import { FieldValueThreshold } from '../../../components/rules/threshold_input'; import { Author, BuildingBlockType, @@ -17,7 +18,6 @@ import { RiskScoreMapping, RuleNameOverride, SeverityMapping, - Threshold, TimestampOverride, } from '../../../../../common/detection_engine/schemas/common/schemas'; @@ -100,10 +100,7 @@ export interface DefineStepRule extends StepRuleData { queryBar: FieldValueQueryBar; ruleType: RuleType; timeline: FieldValueTimeline; - threshold?: { - field: string[]; - value?: number; - }; + threshold: FieldValueThreshold; } export interface ScheduleStepRule extends StepRuleData { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 900a8ccfb4c23..ee83ea91578c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -144,7 +144,7 @@ export const transformAlertToRule = ( to: alert.params.to, type: alert.params.type, threat: alert.params.threat ?? [], - threshold: alert.params.threshold ?? undefined, + threshold: alert.params.threshold, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index b368c8fe36054..452ba958876d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -291,4 +291,158 @@ describe('create_signals', () => { }, }); }); + test('if aggregations is not provided it should not be included', () => { + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: undefined, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + }, + }); + }); + + test('if aggregations is provided it should be included', () => { + const query = buildEventsSearchQuery({ + aggregations: { + tags: { + terms: { + field: 'tag', + }, + }, + }, + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: undefined, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + aggregations: { + tags: { + terms: { + field: 'tag', + }, + }, + }, + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + }, + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 341a930bf8d19..dcf3a90364a40 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -87,8 +87,6 @@ export const buildEventsSearchQuery = ({ }, }; - console.log('searchQuery', JSON.stringify(searchQuery, null, 2)); - if (searchAfterSortId) { return { ...searchQuery, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 4e99995331d56..88b0ee5b3313f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -22,6 +22,7 @@ interface BulkCreateThresholdSignalsParams { someResult: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; + inputIndexPattern: string[]; logger: Logger; id: string; filter: unknown; @@ -38,7 +39,9 @@ interface BulkCreateThresholdSignalsParams { throttle: string; } -type ThresholdResults = SignalSearchResponse; +interface ThresholdResults extends SignalSearchResponse { + '@timestamp': string; +} interface FilterObject { bool?: { @@ -96,6 +99,7 @@ const getThresholdSignalQueryFields = (filter: unknown) => { const getTransformedHits = ( results: ThresholdResults, + inputIndex: string, threshold: Threshold, signalQueryFields: Record ) => { @@ -108,15 +112,15 @@ const getTransformedHits = ( const source = { '@timestamp': new Date().toISOString(), + threshold_count: totalResults, ...signalQueryFields, }; return [ { - _index: '', + _index: inputIndex, _id: uuid.v4(), _source: source, - threshold_count: totalResults, }, ]; } @@ -136,7 +140,7 @@ const getTransformedHits = ( set(source, threshold.field, key); return { - _index: '', + _index: inputIndex, _id: uuid.v4(), _source: source, }; @@ -146,21 +150,15 @@ const getTransformedHits = ( const transformThresholdResultsToEcs = ( results: ThresholdResults, + inputIndex: string, filter: unknown, threshold: Threshold ): SearchResponse => { - console.log( - 'transformThresholdResultsToEcs', - // JSON.stringify(results), - JSON.stringify(filter) - // JSON.stringify(threshold) - ); - const signalQueryFields = getThresholdSignalQueryFields(filter); console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); - const transformedHits = getTransformedHits(results, threshold, signalQueryFields); + const transformedHits = getTransformedHits(results, inputIndex, threshold, signalQueryFields); const thresholdResults = { ...results, @@ -181,10 +179,13 @@ export const bulkCreateThresholdSignals = async ( const thresholdResults = params.someResult; const ecsResults = transformThresholdResultsToEcs( thresholdResults, + params.inputIndexPattern.join(','), params.filter, params.ruleParams.threshold! ); + console.log('inputIndexPattern', params.inputIndexPattern.join(',')); + console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 6312bd590b7d0..039a3bc0e8361 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -256,12 +256,6 @@ export const signalRulesAlertType = ({ // JSON.stringify(thresholdResults.aggregations?.threshold.buckets, null, 2) ); - const thresholdCount = 0; - // const thresholdCount = thresholdResults.aggregations.threshold.buckets.length; - if (thresholdCount) { - logger.info(buildRuleMessage(`Found ${thresholdCount} signals from Threshold aggs.`)); - } - const { success, bulkCreateDuration, @@ -275,6 +269,7 @@ export const signalRulesAlertType = ({ services, logger, id: alertId, + inputIndexPattern: inputIndex, signalsIndex: outputIndex, name, createdBy, From 797232778ba28d446949462b61f9a7f6f778e989 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 14:26:32 +0200 Subject: [PATCH 16/30] types --- .../signals/bulk_create_threshold_signals.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 88b0ee5b3313f..2048708273cba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -15,7 +15,6 @@ import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { SignalSearchResponse } from './types'; -import { SearchResponse } from '../../types'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; @@ -39,10 +38,6 @@ interface BulkCreateThresholdSignalsParams { throttle: string; } -interface ThresholdResults extends SignalSearchResponse { - '@timestamp': string; -} - interface FilterObject { bool?: { filter?: FilterObject | FilterObject[]; @@ -98,7 +93,7 @@ const getThresholdSignalQueryFields = (filter: unknown) => { }; const getTransformedHits = ( - results: ThresholdResults, + results: SignalSearchResponse, inputIndex: string, threshold: Threshold, signalQueryFields: Record @@ -149,11 +144,11 @@ const getTransformedHits = ( }; const transformThresholdResultsToEcs = ( - results: ThresholdResults, + results: SignalSearchResponse, inputIndex: string, filter: unknown, threshold: Threshold -): SearchResponse => { +): SignalSearchResponse => { const signalQueryFields = getThresholdSignalQueryFields(filter); console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); From 81c1adc40ababf378954abed1cb2870a201be7a4 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 18:26:55 +0200 Subject: [PATCH 17/30] cleanup --- ...ts => patch_rules_type_dependents.test.ts} | 0 .../request/patch_rules_type_dependents.ts | 14 ++++++++++++++ .../rules/description_step/index.tsx | 19 ++++++++----------- .../detection_engine/rules/helpers.test.tsx | 6 +++--- .../signals/bulk_create_threshold_signals.ts | 6 ++---- .../signals/signal_rule_alert_type.ts | 10 +++++----- 6 files changed, 32 insertions(+), 23 deletions(-) rename x-pack/plugins/security_solution/common/detection_engine/schemas/request/{patch_rule_type_dependents.test.ts => patch_rules_type_dependents.test.ts} (100%) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.test.ts similarity index 100% rename from x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rule_type_dependents.test.ts rename to x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.test.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts index 554cdb822762f..de2679d774d08 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts @@ -66,6 +66,19 @@ export const validateId = (rule: PatchRulesSchema): string[] => { } }; +export const validateThreshold = (rule: ImportRulesSchema): string[] => { + if (rule.type === 'threshold') { + if (!rule.threshold) { + return ['when "type" is "threshold", "threshold" is required']; + } else if (rule.threshold.value <= 0) { + return ['"threshold.value" has to be bigger than 0']; + } else { + return []; + } + } + return []; +}; + export const patchRuleValidateTypeDependents = (schema: PatchRulesSchema): string[] => { return [ ...validateId(schema), @@ -73,5 +86,6 @@ export const patchRuleValidateTypeDependents = (schema: PatchRulesSchema): strin ...validateLanguage(schema), ...validateTimelineId(schema), ...validateTimelineTitle(schema), + ...validateThreshold(schema), ]; }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 350d9b13c3bc4..51624d04cb58b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -95,17 +95,14 @@ export const StepRuleDescriptionComponent: React.FC = if (columns === 'multi') { return ( - {chunk(Math.ceil(listItems.length / 2), listItems).map((chunkListItems, index) => { - console.log('chunkListItems', chunkListItems); - return ( - - - - ); - })} + {chunk(Math.ceil(listItems.length / 2), listItems).map((chunkListItems, index) => ( + + + + ))} ); } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index f46d6f99c8b2c..3820815938101 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -76,7 +76,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: undefined, + value: 100, }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', @@ -212,7 +212,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: undefined, + value: 100, }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', @@ -245,7 +245,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: undefined, + value: 100, }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 2048708273cba..46de08bed113f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -151,7 +151,7 @@ const transformThresholdResultsToEcs = ( ): SignalSearchResponse => { const signalQueryFields = getThresholdSignalQueryFields(filter); - console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); + // console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); const transformedHits = getTransformedHits(results, inputIndex, threshold, signalQueryFields); @@ -179,9 +179,7 @@ export const bulkCreateThresholdSignals = async ( params.ruleParams.threshold! ); - console.log('inputIndexPattern', params.inputIndexPattern.join(',')); - - console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); + // console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 039a3bc0e8361..66899215bb59a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -250,11 +250,11 @@ export const signalRulesAlertType = ({ threshold, }); - console.log( - 'thresholdResults', - thresholdResults - // JSON.stringify(thresholdResults.aggregations?.threshold.buckets, null, 2) - ); + // console.log( + // 'thresholdResults', + // thresholdResults + // // JSON.stringify(thresholdResults.aggregations?.threshold.buckets, null, 2) + // ); const { success, From 35d5bc34f71201d5d391e426e003ab05ab9e5f02 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 19:22:43 +0200 Subject: [PATCH 18/30] WIP --- .../detection_engine/signals/build_signal.ts | 7 +- .../bulk_create_threshold_signals.test.ts | 196 ++++++++++++++++++ .../signals/bulk_create_threshold_signals.ts | 18 +- 3 files changed, 207 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 563c416faaae4..e7098c015c165 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -46,7 +46,7 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): S const ruleWithoutInternalTags = removeInternalTagsFromRule(rule); const parent = buildAncestor(doc, rule); const ancestors = buildAncestorsSignal(doc, rule); - const signal: Signal = { + let signal: Signal = { parent, ancestors, original_time: doc._source['@timestamp'], @@ -54,10 +54,11 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): S rule: ruleWithoutInternalTags, }; if (doc._source.event != null) { - return { ...signal, original_event: doc._source.event }; + signal = { ...signal, original_event: doc._source.event }; } if (doc._source.threshold_count != null) { - return { ...signal, threshold_count: doc._source.threshold_count }; + signal = { ...signal, threshold_count: doc._source.threshold_count }; + delete doc._source.threshold_count; } return signal; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts new file mode 100644 index 0000000000000..744e2b0c06efe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getThresholdSignalQueryFields } from './bulk_create_threshold_signals'; + +describe('getThresholdSignalQueryFields', () => { + it('should return proper fields for match_phrase filters', () => { + const mockFilters = { + bool: { + must: [], + filter: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + match_phrase: { + 'event.module': 'traefik', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + filter: [ + { + bool: { + should: [ + { + match_phrase: { + 'event.dataset': 'traefik.access', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + match_phrase: { + 'traefik.access.entryPointName': 'web-secure', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + ], + }, + }, + { + match_phrase: { + 'url.domain': 'kibana.siem.estc.dev', + }, + }, + ], + should: [], + must_not: [], + }, + }; + + expect(getThresholdSignalQueryFields(mockFilters)).toEqual({ + 'event.module': 'traefik', + 'event.dataset': 'traefik.access', + 'traefik.access.entryPointName': 'web-secure', + 'url.domain': 'kibana.siem.estc.dev', + }); + }); + + it('should return proper fields object for nested match filters', () => { + const filters = { + bool: { + must: [], + filter: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + match_phrase: { + 'event.module': 'traefik', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + match: { + 'event.dataset': 'traefik.access', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + ], + should: [], + must_not: [], + }, + }; + + expect(getThresholdSignalQueryFields(filters)).toEqual({ + 'event.module': 'traefik', + 'event.dataset': 'traefik.access', + }); + }); + + it('should return proper object for simple match filters', () => { + const filters = { + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'event.module': 'traefik', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'event.dataset': 'traefik.access', + }, + }, + ], + should: [], + must_not: [], + }, + }; + + expect(getThresholdSignalQueryFields(filters)).toEqual({ + 'event.module': 'traefik', + 'event.dataset': 'traefik.access', + }); + }); + + it('should return proper object for simple match_phrase filters', () => { + const filters = { + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match_phrase: { + 'event.module': 'traefik', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'event.dataset': 'traefik.access', + }, + }, + ], + should: [], + must_not: [], + }, + }; + + expect(getThresholdSignalQueryFields(filters)).toEqual({ + 'event.module': 'traefik', + 'event.dataset': 'traefik.access', + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 46de08bed113f..acdb36c61d1b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -62,13 +62,15 @@ const getNestedQueryFilters = (filtersObj: FilterObject): Record ); } else { return ( - (filtersObj.bool?.should && filtersObj.bool?.should[0] && filtersObj.bool.should[0].match) ?? + (filtersObj.bool?.should && + filtersObj.bool?.should[0] && + (filtersObj.bool.should[0].match || filtersObj.bool.should[0].match_phrase)) ?? {} ); } }; -const getThresholdSignalQueryFields = (filter: unknown) => { +export const getThresholdSignalQueryFields = (filter: unknown) => { const filters = get('bool.filter', filter); return reduce( @@ -77,8 +79,8 @@ const getThresholdSignalQueryFields = (filter: unknown) => { return { ...acc, ...item.match_phrase }; } - if (item.bool.should && item.bool.should[0].match) { - return { ...acc, ...item.bool.should[0].match }; + if (item.bool.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { + return { ...acc, ...(item.bool.should[0].match || item.bool.should[0].match_phrase) }; } if (item.bool?.filter) { @@ -99,7 +101,7 @@ const getTransformedHits = ( signalQueryFields: Record ) => { if (isEmpty(threshold.field)) { - const totalResults = results.hits.total; + const totalResults = results.hits.total.value; if (totalResults < threshold.value) { return []; @@ -150,11 +152,7 @@ const transformThresholdResultsToEcs = ( threshold: Threshold ): SignalSearchResponse => { const signalQueryFields = getThresholdSignalQueryFields(filter); - - // console.log('signalQueryFields', JSON.stringify(signalQueryFields, null, 2)); - const transformedHits = getTransformedHits(results, inputIndex, threshold, signalQueryFields); - const thresholdResults = { ...results, hits: { @@ -179,7 +177,5 @@ export const bulkCreateThresholdSignals = async ( params.ruleParams.threshold! ); - // console.log('ruleParams', JSON.stringify(params.ruleParams, null, 2)); - return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; From b422977c476df47e99d88ed8e7287eb86f313880 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 12 Jul 2020 19:32:30 +0200 Subject: [PATCH 19/30] types --- .../schemas/request/patch_rules_type_dependents.ts | 2 +- .../detection_engine/signals/bulk_create_threshold_signals.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts index de2679d774d08..a229771a7c05c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_type_dependents.ts @@ -66,7 +66,7 @@ export const validateId = (rule: PatchRulesSchema): string[] => { } }; -export const validateThreshold = (rule: ImportRulesSchema): string[] => { +export const validateThreshold = (rule: PatchRulesSchema): string[] => { if (rule.type === 'threshold') { if (!rule.threshold) { return ['when "type" is "threshold", "threshold" is required']; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index acdb36c61d1b4..076f3f97aff23 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -101,7 +101,8 @@ const getTransformedHits = ( signalQueryFields: Record ) => { if (isEmpty(threshold.field)) { - const totalResults = results.hits.total.value; + const totalResults = + typeof results.hits.total === 'number' ? results.hits.total : results.hits.total.value; if (totalResults < threshold.value) { return []; From fc125b555caf2b6c79c12261ba00491fb5ff154c Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 09:03:40 +0200 Subject: [PATCH 20/30] fix input --- .../detections/components/rules/threshold_input/index.tsx | 2 +- .../detections/pages/detection_engine/rules/create/helpers.ts | 2 +- .../public/detections/pages/detection_engine/rules/helpers.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 89ca8c9526ecf..952340749b2a9 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -17,7 +17,7 @@ const FIELD_COMBO_BOX_WIDTH = 460; export interface FieldValueThreshold { field: string[]; - value: number; + value: string; } interface ThresholdInputProps { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 6c3474185d9f6..4bb7196e17db5 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -105,7 +105,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ...(ruleType === 'threshold' && { threshold: { field: ruleFields.threshold?.field[0] ?? '', - value: ruleFields.threshold?.value ?? 0, + value: parseInt(ruleFields.threshold?.value, 10) ?? 0, }, }), } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 2478305b69992..0e025190921c8 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -86,7 +86,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ }, threshold: { field: rule.threshold?.field ? [rule.threshold.field] : [], - value: rule.threshold?.value ?? 100, + value: `${rule.threshold?.value}` ?? '100', }, }); From d0bfd94127d81983846a04d245239e1c1e25e0d9 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 10:50:25 +0200 Subject: [PATCH 21/30] constant results --- .../rules/step_define_rule/index.tsx | 2 +- .../rules/all/__mocks__/mock.ts | 6 +++- .../detection_engine/rules/helpers.test.tsx | 8 ++--- .../pages/detection_engine/rules/helpers.tsx | 2 +- .../signals/bulk_create_threshold_signals.ts | 31 ++++++++++++++----- .../signals/signal_rule_alert_type.ts | 2 ++ 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 942c0d547769b..081b9cd9550e4 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -68,7 +68,7 @@ const stepDefineDefaultValue: DefineStepRule = { }, threshold: { field: [], - value: 200, + value: '200', }, timeline: { id: null, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 22704e2a70501..5d84cf5314029 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -153,6 +153,10 @@ export const mockRuleWithEverything = (id: string): Rule => ({ ], }, ], + threshold: { + field: 'host.name', + value: 50, + }, throttle: 'no_actions', timestamp_override: 'event.ingested', note: '# this is some markdown documentation', @@ -215,7 +219,7 @@ export const mockDefineStepRule = (isNew = false): DefineStepRule => ({ }, threshold: { field: [''], - value: 100, + value: '100', }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 3820815938101..590643f8236ee 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -75,8 +75,8 @@ describe('rule helpers', () => { saved_id: 'test123', }, threshold: { - field: [], - value: 100, + field: ['host.name'], + value: '50', }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', @@ -212,7 +212,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: 100, + value: '100', }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', @@ -245,7 +245,7 @@ describe('rule helpers', () => { }, threshold: { field: [], - value: 100, + value: '100', }, timeline: { id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 0e025190921c8..3865bb69dbe7f 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -86,7 +86,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ }, threshold: { field: rule.threshold?.field ? [rule.threshold.field] : [], - value: `${rule.threshold?.value}` ?? '100', + value: `${rule.threshold?.value || 100}`, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 076f3f97aff23..ef9fbe485b92f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import uuid from 'uuid'; +import uuidv5 from 'uuid/v5'; import { reduce, get, isEmpty } from 'lodash/fp'; import set from 'set-value'; @@ -16,6 +16,9 @@ import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { SignalSearchResponse } from './types'; +// used to generate constant Threshold Signals ID when run with the same params +const NAMESPACE_ID = '0684ec03-7201-4ee0-8ee0-3a3f6b2479b2'; + interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; someResult: SignalSearchResponse; @@ -36,6 +39,7 @@ interface BulkCreateThresholdSignalsParams { refresh: RefreshTypes; tags: string[]; throttle: string; + startedAt: Date; } interface FilterObject { @@ -97,7 +101,9 @@ export const getThresholdSignalQueryFields = (filter: unknown) => { const getTransformedHits = ( results: SignalSearchResponse, inputIndex: string, + startedAt: Date, threshold: Threshold, + ruleId: string, signalQueryFields: Record ) => { if (isEmpty(threshold.field)) { @@ -117,7 +123,7 @@ const getTransformedHits = ( return [ { _index: inputIndex, - _id: uuid.v4(), + _id: uuidv5(`${ruleId}${startedAt}${threshold.field}`, NAMESPACE_ID), _source: source, }, ]; @@ -139,21 +145,30 @@ const getTransformedHits = ( return { _index: inputIndex, - _id: uuid.v4(), + _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), _source: source, }; } ); }; -const transformThresholdResultsToEcs = ( +export const transformThresholdResultsToEcs = ( results: SignalSearchResponse, inputIndex: string, + startedAt: Date, filter: unknown, - threshold: Threshold + threshold: Threshold, + ruleId: string ): SignalSearchResponse => { const signalQueryFields = getThresholdSignalQueryFields(filter); - const transformedHits = getTransformedHits(results, inputIndex, threshold, signalQueryFields); + const transformedHits = getTransformedHits( + results, + inputIndex, + startedAt, + threshold, + ruleId, + signalQueryFields + ); const thresholdResults = { ...results, hits: { @@ -174,8 +189,10 @@ export const bulkCreateThresholdSignals = async ( const ecsResults = transformThresholdResultsToEcs( thresholdResults, params.inputIndexPattern.join(','), + params.startedAt, params.filter, - params.ruleParams.threshold! + params.ruleParams.threshold!, + params.ruleParams.ruleId ); return singleBulkCreate({ ...params, filteredEvents: ecsResults }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 66899215bb59a..9859854cdf2d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -60,6 +60,7 @@ export const signalRulesAlertType = ({ producer: SERVER_APP_ID, async executor({ previousStartedAt, + startedAt, alertId, services, params, @@ -271,6 +272,7 @@ export const signalRulesAlertType = ({ id: alertId, inputIndexPattern: inputIndex, signalsIndex: outputIndex, + startedAt, name, createdBy, createdAt, From 8e4c5ef2a4cd931a6bd7cddc68ab76e62bbc8072 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 14:02:02 +0200 Subject: [PATCH 22/30] Fix layout --- .../detections/components/rules/threshold_input/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 952340749b2a9..81e771ce4dc5b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -13,7 +13,7 @@ import { getCategorizedFieldNames } from '../../../../timelines/components/edit_ import { FieldHook, Field } from '../../../../shared_imports'; import { THRESHOLD_FIELD_PLACEHOLDER } from './translations'; -const FIELD_COMBO_BOX_WIDTH = 460; +const FIELD_COMBO_BOX_WIDTH = 410; export interface FieldValueThreshold { field: string[]; From d57926acd5bf43fea5aa2cf1a71267295511ed19 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 19:52:22 +0200 Subject: [PATCH 23/30] Fix form state --- .../rules/select_rule_type/index.tsx | 6 ++- .../select_rule_type/ml_card_description.tsx | 53 +++++++++++-------- .../rules/step_define_rule/index.tsx | 4 +- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx index a8d72f5545c93..6546c1ba59d84 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/index.tsx @@ -87,6 +87,7 @@ export const SelectRuleType: React.FC = ({ title={i18n.QUERY_TYPE_TITLE} description={i18n.QUERY_TYPE_DESCRIPTION} icon={} + isDisabled={querySelectableConfig.isDisabled && !querySelectableConfig.isSelected} selectable={querySelectableConfig} /> @@ -98,7 +99,7 @@ export const SelectRuleType: React.FC = ({ } icon={} - isDisabled={mlCardDisabled} + isDisabled={mlSelectableConfig.isDisabled && !mlSelectableConfig.isSelected} selectable={mlSelectableConfig} /> @@ -108,6 +109,9 @@ export const SelectRuleType: React.FC = ({ title={i18n.THRESHOLD_TYPE_TITLE} description={i18n.THRESHOLD_TYPE_DESCRIPTION} icon={} + isDisabled={ + thresholdSelectableConfig.isDisabled && !thresholdSelectableConfig.isSelected + } selectable={thresholdSelectableConfig} /> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx index 2171c93e47d63..710efd8025517 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx @@ -6,7 +6,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText, EuiLink } from '@elastic/eui'; -import React from 'react'; +import React, { useMemo } from 'react'; import { ML_TYPE_DESCRIPTION } from './translations'; @@ -18,28 +18,35 @@ interface MlCardDescriptionProps { const MlCardDescriptionComponent: React.FC = ({ subscriptionUrl, hasValidLicense = false, -}) => ( - - {hasValidLicense ? ( - ML_TYPE_DESCRIPTION - ) : ( - - - - ), - }} - /> - )} - -); +}) => { + const subscriptionsLinkConfig = useMemo( + () => ({ + subscriptionsLink: ( + + + + ), + }), + [subscriptionUrl] + ); + + return ( + + {hasValidLicense ? ( + ML_TYPE_DESCRIPTION + ) : ( + + )} + + ); +}; MlCardDescriptionComponent.displayName = 'MlCardDescriptionComponent'; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 081b9cd9550e4..c7d70684b34cf 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -308,9 +308,11 @@ const StepDefineRuleComponent: FC = ({ if (deepEqual(index, indicesConfig) && indexModified) { setIndexModified(false); } else if (!deepEqual(index, indicesConfig) && !indexModified) { - setMyStepData((currentValue) => ({ ...currentValue, index })); setIndexModified(true); } + if (myStepData.index !== index) { + setMyStepData((prevValue) => ({ ...prevValue, index })); + } } if (ruleType !== localRuleType) { From df98cf9c492b43af0d2528c12f5066003f892e73 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 21:21:21 +0200 Subject: [PATCH 24/30] Fix sendAlertToTimelineAction --- .../components/alerts_table/actions.tsx | 46 ++++++++++++------- .../body/events/event_column_view.tsx | 20 +++++--- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 8a54d7041ce62..b7f78992dd294 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -102,6 +102,35 @@ export const determineToAndFrom = ({ ecsData }: { ecsData: Ecs }) => { return { to, from }; }; +export const getThresholdAggregationDataProvider = (ecsData: Ecs) => { + const aggregationField = ecsData.signal?.rule?.threshold.field; + const aggregationValue = get(aggregationField, ecsData); + const dataProviderValue = + (Array.isArray(aggregationValue) && aggregationValue[0]) ?? aggregationValue; + + if (!dataProviderValue) { + return []; + } + + const aggregationFieldId = aggregationField.replace('.', '-'); + + return [ + { + and: [], + id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-timeline-1-${aggregationFieldId}-${dataProviderValue}`, + name: ecsData.signal?.rule?.threshold.field, + enabled: true, + excluded: false, + kqlQuery: '', + queryMatch: { + field: aggregationField, + value: dataProviderValue, + operator: ':', + }, + }, + ]; +}; + export const sendAlertToTimelineAction = async ({ apolloClient, createTimeline, @@ -212,22 +241,7 @@ export const sendAlertToTimelineAction = async ({ operator: ':', }, }, - { - and: [], - id: `event-details-value-default-draggable-plain-column-renderer-formatted-field-value-timeline-1-cc2e514d80f3d7abad5b167f6a9efb2761efea2bae4a54834ef807d69daefefd-host_name-${get( - ecsData.signal?.rule?.threshold.field, - ecsData - )}`, - name: ecsData.signal?.rule?.threshold.field, - enabled: true, - excluded: false, - kqlQuery: '', - queryMatch: { - field: ecsData.signal?.rule?.threshold.field, - value: get(ecsData.signal?.rule?.threshold.field, ecsData), - operator: ':', - }, - }, + ...getThresholdAggregationDataProvider(ecsData), ], id: 'timeline-1', dateRange: { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx index 23f7aad049215..14d4747d2036d 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx @@ -201,6 +201,18 @@ export const EventColumnView = React.memo( : grouped.icon; }, [button, closePopover, id, onClickCb, data, ecsData, timelineActions, isPopoverOpen]); + const handlePinClicked = useCallback( + () => + getPinOnClick({ + allowUnpinning: !eventHasNotes(eventIdToNoteIds[id]), + eventId: id, + onPinEvent, + onUnPinEvent, + isEventPinned, + }), + [eventIdToNoteIds, id, isEventPinned, onPinEvent, onUnPinEvent] + ); + return ( ( loadingEventIds={loadingEventIds} noteIds={eventIdToNoteIds[id] || emptyNotes} onEventToggled={onEventToggled} - onPinClicked={getPinOnClick({ - allowUnpinning: !eventHasNotes(eventIdToNoteIds[id]), - eventId: id, - onPinEvent, - onUnPinEvent, - isEventPinned, - })} + onPinClicked={handlePinClicked} showCheckboxes={showCheckboxes} showNotes={showNotes} toggleShowNotes={toggleShowNotes} From 9a67b4d5cf63392e50996d12971c8c92c5b09155 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 13 Jul 2020 21:41:53 +0200 Subject: [PATCH 25/30] Fix react warnings --- .../components/timeline/body/actions/index.tsx | 8 ++++---- .../timeline/body/events/event_column_view.tsx | 12 ++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx index 125ba23a5c5a5..c9c8250922161 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx @@ -96,7 +96,7 @@ export const Actions = React.memo( data-test-subj="event-actions-container" > {showCheckboxes && ( - + {loadingEventIds.includes(eventId) ? ( @@ -117,7 +117,7 @@ export const Actions = React.memo( )} - + {loading ? ( @@ -137,7 +137,7 @@ export const Actions = React.memo( {!isEventViewer && ( <> - + ( - + ( ...acc, icon: [ ...acc.icon, - - + + ( return grouped.contextMenu.length > 0 ? [ ...grouped.icon, - - + + Date: Mon, 13 Jul 2020 22:48:28 +0200 Subject: [PATCH 26/30] Add support for nonEcsData in sendAlertToTimelineAction --- .../components/alerts_table/actions.tsx | 18 ++++++++++++------ .../components/alerts_table/default_config.tsx | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index b7f78992dd294..feac3df314e8d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -7,7 +7,7 @@ /* eslint-disable complexity */ import dateMath from '@elastic/datemath'; -import { get, getOr, isEmpty } from 'lodash/fp'; +import { get, getOr, isEmpty, find } from 'lodash/fp'; import moment from 'moment'; import { updateAlertStatus } from '../../containers/detection_engine/alerts/api'; @@ -102,11 +102,16 @@ export const determineToAndFrom = ({ ecsData }: { ecsData: Ecs }) => { return { to, from }; }; -export const getThresholdAggregationDataProvider = (ecsData: Ecs) => { +export const getThresholdAggregationDataProvider = ( + ecsData: Ecs, + nonEcsData: TimelineNonEcsData[] +) => { const aggregationField = ecsData.signal?.rule?.threshold.field; - const aggregationValue = get(aggregationField, ecsData); - const dataProviderValue = - (Array.isArray(aggregationValue) && aggregationValue[0]) ?? aggregationValue; + const aggregationValue = + get(aggregationField, ecsData) ?? find(['field', aggregationField], nonEcsData)?.value; + const dataProviderValue = Array.isArray(aggregationValue) + ? aggregationValue[0] + : aggregationValue; if (!dataProviderValue) { return []; @@ -135,6 +140,7 @@ export const sendAlertToTimelineAction = async ({ apolloClient, createTimeline, ecsData, + nonEcsData, updateTimelineIsLoading, }: SendAlertToTimelineActionProps) => { let openAlertInBasicTimeline = true; @@ -241,7 +247,7 @@ export const sendAlertToTimelineAction = async ({ operator: ':', }, }, - ...getThresholdAggregationDataProvider(ecsData), + ...getThresholdAggregationDataProvider(ecsData, nonEcsData), ], id: 'timeline-1', dateRange: { diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx index 319575c9c307f..6f1f2e46dce3d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx @@ -309,11 +309,12 @@ export const getAlertActions = ({ displayType: 'icon', iconType: 'timeline', id: 'sendAlertToTimeline', - onClick: ({ ecsData }: TimelineRowActionOnClick) => + onClick: ({ ecsData, data }: TimelineRowActionOnClick) => sendAlertToTimelineAction({ apolloClient, createTimeline, ecsData, + nonEcsData: data, updateTimelineIsLoading, }), width: DEFAULT_ICON_BUTTON_WIDTH, From 8dbe7ab94e16ebb37be14dd21cc655f8f43cba26 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 14 Jul 2020 10:06:02 +0200 Subject: [PATCH 27/30] PR comments --- .../detections/components/alerts_table/actions.test.tsx | 7 +++++++ .../public/detections/components/alerts_table/actions.tsx | 3 ++- .../public/detections/components/alerts_table/types.ts | 3 ++- .../lib/detection_engine/routes/index/signals_mapping.json | 3 +++ .../lib/detection_engine/signals/signal_rule_alert_type.ts | 6 ------ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx index 1213312e2a22c..24bfeaa4dae1a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx @@ -53,6 +53,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: mockEcsDataWithAlert, + nonEcsData: [], updateTimelineIsLoading, }); @@ -65,6 +66,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: mockEcsDataWithAlert, + nonEcsData: [], updateTimelineIsLoading, }); const expected = { @@ -250,6 +252,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: mockEcsDataWithAlert, + nonEcsData: [], updateTimelineIsLoading, }); // @ts-ignore @@ -279,6 +282,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: mockEcsDataWithAlert, + nonEcsData: [], updateTimelineIsLoading, }); // @ts-ignore @@ -297,6 +301,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: mockEcsDataWithAlert, + nonEcsData: [], updateTimelineIsLoading, }); @@ -326,6 +331,7 @@ describe('alert actions', () => { apolloClient, createTimeline, ecsData: ecsDataMock, + nonEcsData: [], updateTimelineIsLoading, }); @@ -350,6 +356,7 @@ describe('alert actions', () => { await sendAlertToTimelineAction({ createTimeline, ecsData: ecsDataMock, + nonEcsData: [], updateTimelineIsLoading, }); diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index feac3df314e8d..11c13c2358e94 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -33,6 +33,7 @@ import { replaceTemplateFieldFromDataProviders, } from './helpers'; import { KueryFilterQueryKind } from '../../../common/store'; +import { DataProvider } from '../../../timelines/components/timeline/data_providers/data_provider'; export const getUpdateAlertsQuery = (eventIds: Readonly) => { return { @@ -105,7 +106,7 @@ export const determineToAndFrom = ({ ecsData }: { ecsData: Ecs }) => { export const getThresholdAggregationDataProvider = ( ecsData: Ecs, nonEcsData: TimelineNonEcsData[] -) => { +): DataProvider[] => { const aggregationField = ecsData.signal?.rule?.threshold.field; const aggregationValue = get(aggregationField, ecsData) ?? find(['field', aggregationField], nonEcsData)?.value; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts index b127ff04eca46..34d18b4dedba6 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts @@ -7,7 +7,7 @@ import ApolloClient from 'apollo-client'; import { Status } from '../../../../common/detection_engine/schemas/common/schemas'; -import { Ecs } from '../../../graphql/types'; +import { Ecs, TimelineNonEcsData } from '../../../graphql/types'; import { TimelineModel } from '../../../timelines/store/timeline/model'; import { inputsModel } from '../../../common/store'; @@ -53,6 +53,7 @@ export interface SendAlertToTimelineActionProps { apolloClient?: ApolloClient<{}>; createTimeline: CreateTimeline; ecsData: Ecs; + nonEcsData: TimelineNonEcsData[]; updateTimelineIsLoading: UpdateTimelineLoading; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index d600bae2746d9..82996c4852316 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -286,6 +286,9 @@ }, "status": { "type": "keyword" + }, + "threshold_count": { + "type": "keyword" } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 9859854cdf2d6..9e77165063a3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -251,12 +251,6 @@ export const signalRulesAlertType = ({ threshold, }); - // console.log( - // 'thresholdResults', - // thresholdResults - // // JSON.stringify(thresholdResults.aggregations?.threshold.buckets, null, 2) - // ); - const { success, bulkCreateDuration, From 51780f96d4d9414fc9be9c10505b6e529bd99630 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 14 Jul 2020 11:28:39 +0200 Subject: [PATCH 28/30] Fix i18n --- .../select_rule_type/ml_card_description.tsx | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx index 710efd8025517..88bf388c26bc2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx @@ -6,7 +6,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText, EuiLink } from '@elastic/eui'; -import React, { useMemo } from 'react'; +import React from 'react'; import { ML_TYPE_DESCRIPTION } from './translations'; @@ -19,19 +19,16 @@ const MlCardDescriptionComponent: React.FC = ({ subscriptionUrl, hasValidLicense = false, }) => { - const subscriptionsLinkConfig = useMemo( - () => ({ - subscriptionsLink: ( - - - - ), - }), - [subscriptionUrl] - ); + const subscriptionsLinkConfig = { + subscriptionsLink: ( + + + + ), + }; return ( From 7679e3221f2ec020b068b5a992d27bd94e79c6da Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 14 Jul 2020 12:54:55 +0200 Subject: [PATCH 29/30] i18n --- .../select_rule_type/ml_card_description.tsx | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx index 88bf388c26bc2..2171c93e47d63 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/select_rule_type/ml_card_description.tsx @@ -18,32 +18,28 @@ interface MlCardDescriptionProps { const MlCardDescriptionComponent: React.FC = ({ subscriptionUrl, hasValidLicense = false, -}) => { - const subscriptionsLinkConfig = { - subscriptionsLink: ( - - - - ), - }; - - return ( - - {hasValidLicense ? ( - ML_TYPE_DESCRIPTION - ) : ( - - )} - - ); -}; +}) => ( + + {hasValidLicense ? ( + ML_TYPE_DESCRIPTION + ) : ( + + + + ), + }} + /> + )} + +); MlCardDescriptionComponent.displayName = 'MlCardDescriptionComponent'; From 1b2662b67ba681addc9198fc97b1244047703f0f Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Tue, 14 Jul 2020 16:53:17 +0200 Subject: [PATCH 30/30] Fix mapping --- .../routes/index/signals_mapping.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 82996c4852316..7d80a319e9e52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -174,6 +174,16 @@ } } }, + "threshold": { + "properties": { + "field": { + "type": "keyword" + }, + "value": { + "type": "float" + } + } + }, "note": { "type": "text" }, @@ -288,7 +298,7 @@ "type": "keyword" }, "threshold_count": { - "type": "keyword" + "type": "float" } } }