diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index f002e13a07cf1..5fbba84467ecf 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -299,6 +299,7 @@ export const type = t.keyof({ query: null, saved_query: null, threshold: null, + threat_match: null, }); export type Type = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts index b666b95ea1e97..777256ff961f9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts @@ -48,3 +48,104 @@ export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSc exceptions_list: [], rule_id: 'rule-1', }); + +export const getAddPrepackagedThreatMatchRulesSchemaMock = (): AddPrepackagedRulesSchema => ({ + description: 'some description', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + version: 1, + threat_query: '*:*', + threat_index: 'list-index', + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); + +export const getAddPrepackagedThreatMatchRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({ + author: [], + description: 'some description', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + severity_mapping: [], + type: 'threat_match', + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + references: [], + actions: [], + enabled: false, + false_positives: [], + from: 'now-6m', + interval: '5m', + max_signals: DEFAULT_MAX_SIGNALS, + tags: [], + to: 'now', + threat: [], + throttle: null, + version: 1, + exceptions_list: [], + rule_id: 'rule-1', + threat_query: '*:*', + threat_index: 'list-index', + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 9b90cf9fdf782..69538f025d95d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -45,6 +45,12 @@ import { RiskScoreMapping, SeverityMapping, } from '../common/schemas'; +import { + threat_index, + threat_query, + threat_filters, + threat_mapping, +} from '../types/threat_mapping'; import { DefaultStringArray, @@ -116,6 +122,10 @@ export const addPrepackagedRulesSchema = t.intersection([ references: DefaultStringArray, // defaults to empty array of strings if not set during decode note, // defaults to "undefined" if not set during decode exceptions_list: DefaultListArray, // defaults to empty array if not set during decode + threat_filters, // defaults to "undefined" if not set during decode + threat_mapping, // defaults to "undefined" if not set during decode + threat_query, // defaults to "undefined" if not set during decode + threat_index, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts index 137b40eb648ba..8c916e4f013b4 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts @@ -17,6 +17,8 @@ import { left } from 'fp-ts/lib/Either'; import { getAddPrepackagedRulesSchemaMock, getAddPrepackagedRulesSchemaDecodedMock, + getAddPrepackagedThreatMatchRulesSchemaMock, + getAddPrepackagedThreatMatchRulesSchemaDecodedMock, } from './add_prepackaged_rules_schema.mock'; import { DEFAULT_MAX_SIGNALS } from '../../../constants'; import { getListArrayMock } from '../types/lists.mock'; @@ -1597,4 +1599,16 @@ describe('add prepackaged rules schema', () => { expect(message.schema).toEqual(expected); }); }); + + describe('threat_mapping', () => { + test('You can set a threat query, index, mapping, filters on a pre-packaged rule', () => { + const payload = getAddPrepackagedThreatMatchRulesSchemaMock(); + const decoded = addPrepackagedRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getAddPrepackagedThreatMatchRulesSchemaDecodedMock(); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts index f1e87bdb11e75..32299be500b45 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts @@ -55,3 +55,103 @@ export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ( exceptions_list: [], rule_id: 'rule-1', }); + +export const getCreateThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): CreateRulesSchema => ({ + description: 'Detecting root and admin users', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: ruleId, + threat_query: '*:*', + threat_index: 'list-index', + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); + +export const getCreateThreatMatchRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({ + author: [], + severity_mapping: [], + risk_score_mapping: [], + description: 'Detecting root and admin users', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'threat_match', + risk_score: 55, + language: 'kuery', + references: [], + actions: [], + enabled: true, + false_positives: [], + from: 'now-6m', + interval: '5m', + max_signals: DEFAULT_MAX_SIGNALS, + tags: [], + to: 'now', + threat: [], + throttle: null, + version: 1, + exceptions_list: [], + rule_id: 'rule-1', + threat_query: '*:*', + threat_index: 'list-index', + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index 56bc68a275ee4..19517017743f1 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -16,6 +16,8 @@ import { left } from 'fp-ts/lib/Either'; import { getCreateRulesSchemaMock, getCreateRulesSchemaDecodedMock, + getCreateThreatMatchRulesSchemaMock, + getCreateThreatMatchRulesSchemaDecodedMock, } from './create_rules_schema.mock'; import { DEFAULT_MAX_SIGNALS } from '../../../constants'; import { getListArrayMock } from '../types/lists.mock'; @@ -1661,4 +1663,16 @@ describe('create rules schema', () => { expect(message.schema).toEqual(expected); }); }); + + describe('threat_mapping', () => { + test('You can set a threat query, index, mapping, filters when creating a rule', () => { + const payload = getCreateThreatMatchRulesSchemaMock(); + const decoded = createRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getCreateThreatMatchRulesSchemaDecodedMock(); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index 7b6b98383cc33..c024ba1c48f8d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -46,6 +46,12 @@ import { RiskScoreMapping, SeverityMapping, } from '../common/schemas'; +import { + threat_index, + threat_query, + threat_filters, + threat_mapping, +} from '../types/threat_mapping'; import { DefaultStringArray, @@ -112,6 +118,10 @@ export const createRulesSchema = t.intersection([ note, // defaults to "undefined" if not set during decode version: DefaultVersionNumber, // defaults to 1 if not set during decode exceptions_list: DefaultListArray, // defaults to empty array if not set during decode + threat_mapping, // defaults to "undefined" if not set during decode + threat_query, // defaults to "undefined" if not set during decode + threat_filters, // defaults to "undefined" if not set during decode + threat_index, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts index 43f0901912271..75ad92578318c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getCreateRulesSchemaMock } from './create_rules_schema.mock'; +import { + getCreateRulesSchemaMock, + getCreateThreatMatchRulesSchemaMock, +} from './create_rules_schema.mock'; import { CreateRulesSchema } from './create_rules_schema'; import { createRuleValidateTypeDependents } from './create_rules_type_dependents'; @@ -87,4 +90,39 @@ describe('create_rules_type_dependents', () => { const errors = createRuleValidateTypeDependents(schema); expect(errors).toEqual(['"threshold.value" has to be bigger than 0']); }); + + test('threat_index, threat_query, and threat_mapping are required when type is "threat_match" and validates with it', () => { + const schema: CreateRulesSchema = { + ...getCreateRulesSchemaMock(), + type: 'threat_match', + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "type" is "threat_match", "threat_index" is required', + 'when "type" is "threat_match", "threat_query" is required', + 'when "type" is "threat_match", "threat_mapping" is required', + ]); + }); + + test('validates with threat_index, threat_query, and threat_mapping when type is "threat_match"', () => { + const schema = getCreateThreatMatchRulesSchemaMock(); + const { threat_filters: threatFilters, ...noThreatFilters } = schema; + const errors = createRuleValidateTypeDependents(noThreatFilters); + expect(errors).toEqual([]); + }); + + test('does NOT validate when threat_mapping is an empty array', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + threat_mapping: [], + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual(['threat_mapping" must have at least one element']); + }); + + test('validates with threat_index, threat_query, threat_mapping, and an optional threat_filters, when type is "threat_match"', () => { + const schema = getCreateThreatMatchRulesSchemaMock(); + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([]); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index 91b14fa9b999c..c2a41005ebf4d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -5,7 +5,7 @@ */ import { isMlRule } from '../../../machine_learning/helpers'; -import { isThresholdRule } from '../../utils'; +import { isThreatMatchRule, isThresholdRule } from '../../utils'; import { CreateRulesSchema } from './create_rules_schema'; export const validateAnomalyThreshold = (rule: CreateRulesSchema): string[] => { @@ -107,6 +107,24 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => { return []; }; +export const validateThreatMapping = (rule: CreateRulesSchema): string[] => { + let errors: string[] = []; + if (isThreatMatchRule(rule.type)) { + if (!rule.threat_mapping) { + errors = ['when "type" is "threat_match", "threat_mapping" is required', ...errors]; + } else if (rule.threat_mapping.length === 0) { + errors = ['threat_mapping" must have at least one element', ...errors]; + } + if (!rule.threat_query) { + errors = ['when "type" is "threat_match", "threat_query" is required', ...errors]; + } + if (!rule.threat_index) { + errors = ['when "type" is "threat_match", "threat_index" is required', ...errors]; + } + } + return errors; +}; + export const createRuleValidateTypeDependents = (schema: CreateRulesSchema): string[] => { return [ ...validateAnomalyThreshold(schema), @@ -117,5 +135,6 @@ export const createRuleValidateTypeDependents = (schema: CreateRulesSchema): str ...validateTimelineId(schema), ...validateTimelineTitle(schema), ...validateThreshold(schema), + ...validateThreatMapping(schema), ]; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts index e3b4196c90c6c..160dbb92b74cd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts @@ -76,3 +76,94 @@ export const ruleIdsToNdJsonString = (ruleIds: string[]) => { const rules = ruleIds.map((ruleId) => getImportRulesSchemaMock(ruleId)); return rulesToNdJsonString(rules); }; + +export const getImportThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): ImportRulesSchema => ({ + description: 'some description', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: ruleId, + threat_index: 'index-123', + threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }], + threat_query: '*:*', + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); + +export const getImportThreatMatchRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => ({ + author: [], + description: 'some description', + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + severity: 'high', + severity_mapping: [], + type: 'threat_match', + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + references: [], + actions: [], + enabled: true, + false_positives: [], + from: 'now-6m', + interval: '5m', + max_signals: DEFAULT_MAX_SIGNALS, + tags: [], + to: 'now', + threat: [], + throttle: null, + version: 1, + exceptions_list: [], + rule_id: 'rule-1', + immutable: false, + threat_query: '*:*', + threat_index: 'index-123', + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts index 0515bee0052d7..bd25a63e153dd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts @@ -20,6 +20,8 @@ import { import { getImportRulesSchemaMock, getImportRulesSchemaDecodedMock, + getImportThreatMatchRulesSchemaMock, + getImportThreatMatchRulesSchemaDecodedMock, } from './import_rules_schema.mock'; import { DEFAULT_MAX_SIGNALS } from '../../../constants'; import { getListArrayMock } from '../types/lists.mock'; @@ -1792,4 +1794,16 @@ describe('import rules schema', () => { expect(message.schema).toEqual(expected); }); }); + + describe('threat_mapping', () => { + test('You can set a threat query, index, mapping, filters on an imported rule', () => { + const payload = getImportThreatMatchRulesSchemaMock(); + const decoded = importRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getImportThreatMatchRulesSchemaDecodedMock(); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 698716fea696e..b63d70783b7b5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -52,6 +52,12 @@ import { RiskScoreMapping, SeverityMapping, } from '../common/schemas'; +import { + threat_index, + threat_query, + threat_filters, + threat_mapping, +} from '../types/threat_mapping'; import { DefaultStringArray, @@ -135,6 +141,10 @@ export const importRulesSchema = t.intersection([ updated_at, // defaults "undefined" if not set during decode created_by, // defaults "undefined" if not set during decode updated_by, // defaults "undefined" if not set during decode + threat_filters, // defaults to "undefined" if not set during decode + threat_mapping, // defaults to "undefined" if not set during decode + threat_query, // defaults to "undefined" if not set during decode + threat_index, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index ed9fb8930ea1b..a462b297d37f8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -82,3 +82,31 @@ export const getRulesMlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSch machine_learning_job_id: 'some_machine_learning_job_id', }; }; + +export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => { + return { + ...getRulesSchemaMock(anchorDate), + type: 'threat_match', + threat_index: 'index-123', + threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }], + threat_query: '*:*', + threat_filters: [ + { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 36fc063761840..3a47d4af6ac14 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -17,11 +17,16 @@ import { addQueryFields, addTimelineTitle, addMlFields, + addThreatMatchFields, } from './rules_schema'; import { exactCheck } from '../../../exact_check'; import { foldLeftRight, getPaths } from '../../../test_utils'; import { TypeAndTimelineOnly } from './type_timeline_only_schema'; -import { getRulesSchemaMock, getRulesMlSchemaMock } from './rules_schema.mocks'; +import { + getRulesSchemaMock, + getRulesMlSchemaMock, + getThreatMatchingSchemaMock, +} from './rules_schema.mocks'; import { ListArray } from '../types/lists'; export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z'; @@ -593,6 +598,36 @@ describe('rules_schema', () => { expect(getPaths(left(message.errors))).toEqual(['invalid keys "query,language"']); expect(message.schema).toEqual({}); }); + + test('it validates a threat_match response', () => { + const payload = getThreatMatchingSchemaMock(); + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getThreatMatchingSchemaMock(); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + + test('it rejects a response with threat_match properties but type of "query"', () => { + const payload: RulesSchema = { + ...getThreatMatchingSchemaMock(), + type: 'query', + }; + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'invalid keys "threat_index,threat_mapping,[{"entries":[{"field":"host.name","type":"mapping","value":"host.name"}]}],threat_query,threat_filters,[{"bool":{"must":[{"query_string":{"query":"host.name: linux","analyze_wildcard":true,"time_zone":"Zulu"}}],"filter":[],"should":[],"must_not":[]}}]"', + ]); + expect(message.schema).toEqual({}); + }); }); describe('addSavedId', () => { @@ -647,6 +682,11 @@ describe('rules_schema', () => { const fields = addQueryFields({ type: 'saved_query' }); expect(fields.length).toEqual(2); }); + + test('should return two fields for a rule of type "threat_match"', () => { + const fields = addQueryFields({ type: 'threat_match' }); + expect(fields.length).toEqual(2); + }); }); describe('addMlFields', () => { @@ -704,4 +744,17 @@ describe('rules_schema', () => { expect(message.schema).toEqual({ ...payload, exceptions_list: [] }); }); }); + + describe('addThreatMatchFields', () => { + test('should return empty array if type is not "threat_match"', () => { + const fields = addThreatMatchFields({ type: 'query' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return 5 fields for a rule of type "threat_match"', () => { + const fields = addThreatMatchFields({ type: 'threat_match' }); + expect(fields.length).toEqual(5); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index c26a7efb0c288..1c2254f9f8f09 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -60,6 +60,13 @@ import { rule_name_override, timestamp_override, } from '../common/schemas'; +import { + threat_index, + threat_query, + threat_filters, + threat_mapping, +} from '../types/threat_mapping'; + import { DefaultListArray } from '../types/lists_default_array'; import { DefaultStringArray, @@ -114,7 +121,7 @@ export const dependentRulesSchema = t.partial({ language, query, - // when type = saved_query, saved_is is required + // when type = saved_query, saved_id is required saved_id, // These two are required together or not at all. @@ -127,6 +134,12 @@ export const dependentRulesSchema = t.partial({ // Threshold fields threshold, + + // Threat Match fields + threat_filters, + threat_index, + threat_query, + threat_mapping, }); /** @@ -206,7 +219,9 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (['eql', 'query', 'saved_query', 'threshold'].includes(typeAndTimelineOnly.type)) { + if ( + ['eql', 'query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type) + ) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), @@ -240,6 +255,20 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t. } }; +export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'threat_match') { + return [ + t.exact(t.type({ threat_query: dependentRulesSchema.props.threat_query })), + t.exact(t.type({ threat_index: dependentRulesSchema.props.threat_index })), + t.exact(t.type({ threat_mapping: dependentRulesSchema.props.threat_mapping })), + t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })), + t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), + ]; + } else { + return []; + } +}; + export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed => { const dependents: t.Mixed[] = [ t.exact(requiredRulesSchema), @@ -249,6 +278,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed ...addQueryFields(typeAndTimelineOnly), ...addMlFields(typeAndTimelineOnly), ...addThresholdFields(typeAndTimelineOnly), + ...addThreatMatchFields(typeAndTimelineOnly), ]; if (dependents.length > 1) { diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts new file mode 100644 index 0000000000000..63d593ea84e67 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ThreatMapping, + threatMappingEntries, + ThreatMappingEntries, + threat_mapping, +} from './threat_mapping'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; +import { foldLeftRight, getPaths } from '../../../test_utils'; +import { exactCheck } from '../../../exact_check'; + +describe('threat_mapping', () => { + describe('threatMappingEntries', () => { + test('it should validate an entry', () => { + const payload: ThreatMappingEntries = [ + { + field: 'field.one', + type: 'mapping', + value: 'field.one', + }, + ]; + const decoded = threatMappingEntries.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should NOT validate an extra entry item', () => { + const payload: ThreatMappingEntries & Array<{ extra: string }> = [ + { + field: 'field.one', + type: 'mapping', + value: 'field.one', + extra: 'blah', + }, + ]; + const decoded = threatMappingEntries.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['invalid keys "extra"']); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate a non string', () => { + const payload = ([ + { + field: 5, + type: 'mapping', + value: 'field.one', + }, + ] as unknown) as ThreatMappingEntries[]; + const decoded = threatMappingEntries.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['Invalid value "5" supplied to "field"']); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate a wrong type', () => { + const payload = ([ + { + field: 'field.one', + type: 'invalid', + value: 'field.one', + }, + ] as unknown) as ThreatMappingEntries[]; + const decoded = threatMappingEntries.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "invalid" supplied to "type"', + ]); + expect(message.schema).toEqual({}); + }); + }); + + describe('threat_mapping', () => { + test('it should validate a threat mapping', () => { + const payload: ThreatMapping = [ + { + entries: [ + { + field: 'field.one', + type: 'mapping', + value: 'field.one', + }, + ], + }, + ]; + const decoded = threat_mapping.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + }); + + test('it should NOT validate an extra key', () => { + const payload: ThreatMapping & Array<{ extra: string }> = [ + { + entries: [ + { + field: 'field.one', + type: 'mapping', + value: 'field.one', + }, + ], + extra: 'invalid', + }, + ]; + + const decoded = threat_mapping.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['invalid keys "extra"']); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate an extra inner entry', () => { + const payload: ThreatMapping & Array<{ entries: Array<{ extra: string }> }> = [ + { + entries: [ + { + field: 'field.one', + type: 'mapping', + value: 'field.one', + extra: 'blah', + }, + ], + }, + ]; + + const decoded = threat_mapping.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['invalid keys "extra"']); + expect(message.schema).toEqual({}); + }); + + test('it should NOT validate an extra inner entry with the wrong data type', () => { + const payload = ([ + { + entries: [ + { + field: 5, + type: 'mapping', + value: 'field.one', + }, + ], + }, + ] as unknown) as ThreatMapping; + + const decoded = threat_mapping.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "5" supplied to "entries,field"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts new file mode 100644 index 0000000000000..f2b4754c2d113 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import * as t from 'io-ts'; +import { NonEmptyString } from './non_empty_string'; + +export const threat_query = t.string; +export type ThreatQuery = t.TypeOf; +export const threatQueryOrUndefined = t.union([threat_query, t.undefined]); +export type ThreatQueryOrUndefined = t.TypeOf; + +export const threat_filters = t.array(t.unknown); // Filters are not easily type-able yet +export type ThreatFilters = t.TypeOf; +export const threatFiltersOrUndefined = t.union([threat_filters, t.undefined]); +export type ThreatFiltersOrUndefined = t.TypeOf; + +export const threatMappingEntries = t.array( + t.exact( + t.type({ + field: NonEmptyString, + type: t.keyof({ mapping: null }), + value: NonEmptyString, + }) + ) +); +export type ThreatMappingEntries = t.TypeOf; + +export const threat_mapping = t.array( + t.exact( + t.type({ + entries: threatMappingEntries, + }) + ) +); +export type ThreatMapping = t.TypeOf; + +export const threatMappingOrUndefined = t.union([threat_mapping, t.undefined]); +export type ThreatMappingOrUndefined = t.TypeOf; + +export const threat_index = t.string; +export const threatIndexOrUndefined = t.union([threat_index, t.undefined]); +export type ThreatIndexOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 99680ffe41d44..ea50acc9b46be 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { hasLargeValueList, hasNestedEntry } from './utils'; +import { hasLargeValueList, hasNestedEntry, isThreatMatchRule } from './utils'; import { EntriesArray } from '../shared_imports'; describe('#hasLargeValueList', () => { @@ -102,4 +102,14 @@ describe('#hasNestedEntry', () => { expect(hasLists).toBeFalsy(); }); + + describe('isThreatMatchRule', () => { + test('it returns true if a threat match rule', () => { + expect(isThreatMatchRule('threat_match')).toEqual(true); + }); + + test('it returns false if not a threat match rule', () => { + expect(isThreatMatchRule('query')).toEqual(false); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 170d28cb5a725..f76417099bb17 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -17,6 +17,7 @@ export const hasNestedEntry = (entries: EntriesArray): boolean => { return found.length > 0; }; -export const isEqlRule = (ruleType: Type | undefined) => ruleType === 'eql'; -export const isThresholdRule = (ruleType: Type | undefined) => ruleType === 'threshold'; -export const isQueryRule = (ruleType: Type | undefined) => ruleType === 'query'; +export const isEqlRule = (ruleType: Type | undefined): boolean => ruleType === 'eql'; +export const isThresholdRule = (ruleType: Type | undefined): boolean => ruleType === 'threshold'; +export const isQueryRule = (ruleType: Type | undefined): boolean => ruleType === 'query'; +export const isThreatMatchRule = (ruleType: Type): boolean => ruleType === 'threat_match'; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx index 073cb46d3949a..f2eb5cf5b94f3 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.test.tsx @@ -429,5 +429,11 @@ describe('helpers', () => { expect(result.description).toEqual('Threshold'); }); + + it('returns a humanized description for a threat_match type', () => { + const [result]: ListItems[] = buildRuleTypeDescription('Test label', 'threat_match'); + + expect(result.description).toEqual('Threat Match'); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 0c866ae0bd926..4d46d4dc86846 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -391,6 +391,14 @@ export const buildRuleTypeDescription = (label: string, ruleType: Type): ListIte }, ]; } + case 'threat_match': { + return [ + { + title: label, + description: i18n.THREAT_MATCH_TYPE_DESCRIPTION, + }, + ]; + } default: return assertUnreachable(ruleType); } diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx index 124ef9e648403..d714f04f519d4 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.tsx @@ -55,6 +55,13 @@ export const THRESHOLD_TYPE_DESCRIPTION = i18n.translate( } ); +export const THREAT_MATCH_TYPE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.threatMatchRuleTypeDescription', + { + defaultMessage: 'Threat Match', + } +); + export const ML_JOB_STARTED = i18n.translate( 'xpack.securitySolution.detectionEngine.ruleDescription.mlJobStartedDescription', { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 2acb3e57c5a3b..65a5c6aca0050 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -94,6 +94,7 @@ export const filterRuleFieldsForType = (fields: T, type: T case 'threshold': const { anomalyThreshold, machineLearningJobId, ...thresholdRuleFields } = fields; return thresholdRuleFields; + case 'threat_match': case 'query': case 'saved_query': case 'eql': diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index dbfb0333e48ee..42fbe40d690ea 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -315,6 +315,7 @@ const getRuleSpecificRuleParamKeys = (ruleType: Type) => { return ['anomaly_threshold', 'machine_learning_job_id']; case 'threshold': return ['threshold', ...queryRuleParams]; + case 'threat_match': case 'query': case 'saved_query': case 'eql': diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 29c56e8ed80b1..fb01f92255516 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -396,6 +396,10 @@ export const getResult = (): RuleAlertType => ({ ], threshold: undefined, timestampOverride: undefined, + threatFilters: undefined, + threatMapping: undefined, + threatIndex: undefined, + threatQuery: undefined, references: ['http://www.example.com', 'https://ww.example.com'], note: '# Investigative notes', version: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 959bf3186f136..dd887233c36a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -91,6 +91,10 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => severity_mapping: severityMapping, tags, threat, + threat_filters: threatFilters, + threat_index: threatIndex, + threat_mapping: threatMapping, + threat_query: threatQuery, threshold, throttle, timestamp_override: timestampOverride, @@ -176,6 +180,10 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => to, type, threat, + threatFilters, + threatMapping, + threatQuery, + threatIndex, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 701e5b5e706ed..26ab89ad8ea7c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -78,6 +78,10 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void tags, threat, threshold, + threat_filters: threatFilters, + threat_index: threatIndex, + threat_query: threatQuery, + threat_mapping: threatMapping, throttle, timestamp_override: timestampOverride, to, @@ -162,6 +166,10 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void type, threat, threshold, + threatFilters, + threatIndex, + threatQuery, + threatMapping, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 0f44b50d4bc74..0f5d0304f5ca0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -158,6 +158,10 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP severity_mapping: severityMapping, tags, threat, + threat_filters: threatFilters, + threat_index: threatIndex, + threat_query: threatQuery, + threat_mapping: threatMapping, threshold, timestamp_override: timestampOverride, to, @@ -217,7 +221,11 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP to, type, threat, + threatFilters, + threatIndex, + threatQuery, threshold, + threatMapping, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts index 11f74c264ae0c..2159245f2f735 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts @@ -19,7 +19,7 @@ import { } from './utils'; import { getResult } from '../__mocks__/request_responses'; import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; -import { RuleTypeParams } from '../../types'; +import { PartialFilter, RuleTypeParams } from '../../types'; import { BulkError, ImportSuccessError } from '../utils'; import { getOutputRuleAlertForRest } from '../__mocks__/utils'; import { createPromiseFromStreams } from '../../../../../../../../src/core/server/utils'; @@ -30,6 +30,7 @@ import { RuleAlertType } from '../../rules/types'; import { CreateRulesBulkSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/create_rules_bulk_schema'; import { ImportRulesSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/import_rules_schema'; import { getCreateRulesSchemaMock } from '../../../../../common/detection_engine/schemas/request/create_rules_schema.mock'; +import { ThreatMapping } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; type PromiseFromStreams = ImportRulesSchemaDecoded | Error; @@ -122,6 +123,55 @@ describe('utils', () => { ); }); + test('transforms threat_matching fields', () => { + const threatRule = getResult(); + const threatFilters: PartialFilter[] = [ + { + query: { + bool: { + must: [ + { + query_string: { + query: 'host.name: linux', + analyze_wildcard: true, + time_zone: 'Zulu', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + }, + }, + ]; + const threatMapping: ThreatMapping = [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ]; + threatRule.params.threatIndex = 'index-123'; + threatRule.params.threatFilters = threatFilters; + threatRule.params.threatMapping = threatMapping; + threatRule.params.threatQuery = '*:*'; + + const rule = transformAlertToRule(threatRule); + expect(rule).toEqual( + expect.objectContaining({ + threat_index: 'index-123', + threat_filters: threatFilters, + threat_mapping: threatMapping, + threat_query: '*:*', + }) + ); + }); + // This has to stay here until we do data migration of saved objects and lists is removed from: // signal_params_schema.ts test('does not leak a lists structure in the transform which would cause validation issues', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index ee83ea91578c5..556ea209152e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -145,6 +145,10 @@ export const transformAlertToRule = ( type: alert.params.type, threat: alert.params.threat ?? [], threshold: alert.params.threshold, + threat_filters: alert.params.threatFilters, + threat_index: alert.params.threatIndex, + threat_query: alert.params.threatQuery, + threat_mapping: alert.params.threatMapping, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 1117f34b6f8c5..95067e57868d1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -39,6 +39,10 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threatFilters: undefined, + threatMapping: undefined, + threatQuery: undefined, + threatIndex: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', @@ -82,6 +86,10 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ severityMapping: [], tags: [], threat: [], + threatFilters: undefined, + threatIndex: undefined, + threatMapping: undefined, + threatQuery: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 0c67d9ca77146..9ed94cd7bff2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -42,6 +42,10 @@ export const createRules = async ({ severityMapping, tags, threat, + threatFilters, + threatIndex, + threatQuery, + threatMapping, threshold, timestampOverride, to, @@ -86,6 +90,10 @@ export const createRules = async ({ severityMapping, threat, threshold, + threatFilters, + threatIndex, + threatQuery, + threatMapping, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 3af0c3f55b485..59e14dcffc3c0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -47,6 +47,10 @@ export const installPrepackagedRules = ( to, type, threat, + threat_filters: threatFilters, + threat_mapping: threatMapping, + threat_query: threatQuery, + threat_index: threatIndex, threshold, timestamp_override: timestampOverride, references, @@ -93,6 +97,10 @@ export const installPrepackagedRules = ( to, type, threat, + threatFilters, + threatMapping, + threatQuery, + threatIndex, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index b845990fd94ef..6b851351f27f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -85,6 +85,13 @@ import { BuildingBlockTypeOrUndefined, RuleNameOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; +import { + ThreatIndexOrUndefined, + ThreatQueryOrUndefined, + ThreatMappingOrUndefined, + ThreatFiltersOrUndefined, +} from '../../../../common/detection_engine/schemas/types/threat_mapping'; + import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; import { Alert, SanitizedAlert } from '../../../../../alerts/common'; import { SIGNALS_ID } from '../../../../common/constants'; @@ -206,6 +213,10 @@ export interface CreateRulesOptions { tags: Tags; threat: Threat; threshold: ThresholdOrUndefined; + threatFilters: ThreatFiltersOrUndefined; + threatIndex: ThreatIndexOrUndefined; + threatQuery: ThreatQueryOrUndefined; + threatMapping: ThreatMappingOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh new file mode 100755 index 0000000000000..23c1914387c44 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + + +# Adds port mock data to a threat list for testing. +# Example: ./create_threat_data.sh +# Example: ./create_threat_data.sh 1000 2000 + +START=${1:-1} +END=${2:-1000} + +for (( i=$START; i<=$END; i++ )) +do { +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \ + --data " +{ + \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\", + \"source\": { \"ip\": \"127.0.0.1\", \"port\": \"${i}\" }, + \"destination\": { \"ip\": \"127.0.0.1\", \"port\": \"${i}\" } +}" +} > /dev/null +done diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_mapping.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_mapping.sh new file mode 100755 index 0000000000000..b0ec2973b2dd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_mapping.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Add a small partial ECS based mapping of just source.ip, source.port, destination.ip, destination.port +# dnd then adds a large volume of threat lists to it + +# Example: .create_threat_mapping.sh + +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list \ + --data ' +{ + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "source": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "host": { + "properties": { + "name": { + "type": "keyword" + }, + "ip" : { + "type" : "ip" + } + } + } + } + } +}' | jq . diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/delete_threat_list.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/delete_threat_list.sh new file mode 100755 index 0000000000000..85eac94a2991f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/delete_threat_list.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Deletes a mock threat list +# Example: ./delete_threat_list.sh + +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${ELASTICSEARCH_URL}/mock-threat-list \ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping.json new file mode 100644 index 0000000000000..c914e568048a1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping.json @@ -0,0 +1,60 @@ +{ + "name": "Query with a threat mapping", + "description": "Query with a threat mapping", + "rule_id": "threat-mapping", + "risk_score": 1, + "severity": "high", + "type": "threat_match", + "query": "*:*", + "tags": ["tag_1", "tag_2"], + "threat_index": "mock-threat-list", + "threat_query": "*:*", + "threat_mapping": [ + { + "entries": [ + { + "field": "host.name", + "type": "mapping", + "value": "host.name" + }, + { + "field": "host.ip", + "type": "mapping", + "value": "host.ip" + } + ] + }, + { + "entries": [ + { + "field": "destination.ip", + "type": "mapping", + "value": "destination.ip" + }, + { + "field": "destination.port", + "type": "mapping", + "value": "destination.port" + } + ] + }, + { + "entries": [ + { + "field": "source.port", + "type": "mapping", + "value": "source.port" + } + ] + }, + { + "entries": [ + { + "field": "source.ip", + "type": "mapping", + "value": "source.ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 9d3eb29be08dd..bbdb8ea0a36ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -47,6 +47,10 @@ export const sampleRuleAlertParams = ( filters: undefined, savedId: undefined, threshold: undefined, + threatFilters: undefined, + threatQuery: undefined, + threatMapping: undefined, + threatIndex: undefined, timelineId: undefined, timelineTitle: undefined, timestampOverride: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 6323938d6903b..6ce0be54a9e7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -18,6 +18,7 @@ import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { AlertServices } from '../../../../../alerts/server'; import { PartialFilter } from '../types'; import { BadRequestError } from '../errors/bad_request_error'; +import { QueryFilter } from './types'; interface GetFilterArgs { type: Type; @@ -48,7 +49,7 @@ export const getFilter = async ({ type, query, lists, -}: GetFilterArgs): Promise => { +}: GetFilterArgs): Promise => { const queryFilter = () => { if (query != null && language != null && index != null) { return getQueryFilter(query, language, filters || [], index, lists); @@ -90,6 +91,7 @@ export const getFilter = async ({ switch (type) { case 'eql': + case 'threat_match': case 'threshold': { return savedId != null ? savedQueryFilter() : queryFilter(); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index 0c56ed300cb48..c8f8341392553 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -42,6 +42,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ savedId: null, severity: 'high', severityMapping: null, + threatFilters: null, threat: null, timelineId: null, timelineTitle: null, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index d08ca90f3e353..dbb48d59d3a3f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -48,6 +48,10 @@ const signalSchema = schema.object({ lists: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), // For backwards compatibility with customers that had a data bug in 7.7. Once we use a migration script please remove this. exceptions_list: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), // For backwards compatibility with customers that had a data bug in 7.8. Once we use a migration script please remove this. exceptionsList: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + threatFilters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + threatIndex: schema.maybe(schema.string()), + threatQuery: schema.maybe(schema.string()), + threatMapping: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), }); /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 5f9e0a08065c4..3ff5d5d2a6e13 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -40,7 +40,6 @@ jest.mock('./utils'); jest.mock('../notifications/schedule_notification_actions'); jest.mock('./find_ml_signals'); jest.mock('./bulk_create_ml_signals'); -jest.mock('./../../../../common/detection_engine/utils'); jest.mock('../../../../common/detection_engine/parse_schedule_dates'); const getPayload = (ruleAlert: RuleAlertType, services: AlertServicesMock) => ({ @@ -480,6 +479,19 @@ describe('rules_notification_alert_type', () => { ); }); }); + + describe('threat match', () => { + it('should throw an error if threatQuery or threatIndex or threatMapping was not null', async () => { + const result = getResult(); + result.params.type = 'threat_match'; + payload = getPayload(result, alertServices) as jest.Mocked; + await alert.executor(payload); + expect(logger.error).toHaveBeenCalled(); + expect(logger.error.mock.calls[0][0]).toContain( + 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' + ); + }); + }); }); describe('should catch error', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8ea94f943336e..196c17b42221b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -14,7 +14,11 @@ import { SERVER_APP_ID, } from '../../../../common/constants'; import { isJobStarted, isMlRule } from '../../../../common/machine_learning/helpers'; -import { isThresholdRule, isEqlRule } from '../../../../common/detection_engine/utils'; +import { + isThresholdRule, + isEqlRule, + isThreatMatchRule, +} from '../../../../common/detection_engine/utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; @@ -45,6 +49,7 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ logger, @@ -90,6 +95,10 @@ export const signalRulesAlertType = ({ query, to, threshold, + threatFilters, + threatQuery, + threatIndex, + threatMapping, type, exceptionsList, } = params; @@ -310,6 +319,57 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } + } else if (isThreatMatchRule(type)) { + if ( + threatQuery == null || + threatIndex == null || + threatMapping == null || + query == null + ) { + throw new Error( + [ + 'Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping:', + `threatQuery: "${threatQuery}"`, + `threatIndex: "${threatIndex}"`, + `threatMapping: "${threatMapping}"`, + ].join(' ') + ); + } + const inputIndex = await getInputIndex(services, version, index); + result = await createThreatSignals({ + threatMapping, + query, + inputIndex, + type, + filters: filters ?? [], + language, + name, + savedId, + services, + exceptionItems: exceptionItems ?? [], + gap, + previousStartedAt, + listClient, + logger, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + interval, + updatedAt, + enabled, + refresh, + tags, + throttle, + threatFilters: threatFilters ?? [], + threatQuery, + buildRuleMessage, + threatIndex, + }); } else { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts new file mode 100644 index 0000000000000..b1fab34d66ab8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ThreatMapping } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; +import { Filter } from 'src/plugins/data/common'; + +import { SearchResponse } from 'elasticsearch'; +import { ThreatListItem } from './types'; + +export const getThreatMappingMock = (): ThreatMapping => { + return [ + { + entries: [ + { + field: 'host.name', + type: 'mapping', + value: 'host.name', + }, + { + field: 'host.ip', + type: 'mapping', + value: 'host.ip', + }, + ], + }, + { + entries: [ + { + field: 'destination.ip', + type: 'mapping', + value: 'destination.ip', + }, + { + field: 'destination.port', + type: 'mapping', + value: 'destination.port', + }, + ], + }, + { + entries: [ + { + field: 'source.port', + type: 'mapping', + value: 'source.port', + }, + ], + }, + { + entries: [ + { + field: 'source.ip', + type: 'mapping', + value: 'source.ip', + }, + ], + }, + ]; +}; + +export const getThreatListSearchResponseMock = (): SearchResponse => ({ + took: 0, + timed_out: false, + _shards: { + total: 1, + successful: 1, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 0, + hits: [ + { + _index: 'index', + _type: 'type', + _id: '123', + _score: 0, + _source: getThreatListItemMock(), + }, + ], + }, +}); + +export const getThreatListItemMock = (): ThreatListItem => ({ + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + ip: '192.168.0.0.1', + }, + source: { + ip: '127.0.0.1', + port: 1, + }, + destination: { + ip: '127.0.0.1', + port: 1, + }, +}); + +export const getFilterThreatMapping = (): ThreatMapping => [ + { + entries: [ + { + field: 'host.name', + type: 'mapping', + value: 'host.name', + }, + { + field: 'host.ip', + type: 'mapping', + value: 'host.ip', + }, + ], + }, + { + entries: [ + { + field: 'destination.ip', + type: 'mapping', + value: 'destination.ip', + }, + { + field: 'destination.port', + type: 'mapping', + value: 'destination.port', + }, + ], + }, + { + entries: [ + { + field: 'source.port', + type: 'mapping', + value: 'source.port', + }, + ], + }, + { + entries: [ + { + field: 'source.ip', + type: 'mapping', + value: 'source.ip', + }, + ], + }, +]; + +export const getThreatMappingFilterMock = (): Filter => ({ + meta: { + alias: null, + negate: false, + disabled: false, + }, + query: { + bool: { + should: getThreatMappingFiltersShouldMock(), + minimum_should_match: 1, + }, + }, +}); + +export const getThreatMappingFiltersShouldMock = (count = 1) => { + return new Array(count).fill(null).map((_, index) => getThreatMappingFilterShouldMock(index + 1)); +}; + +export const getThreatMappingFilterShouldMock = (port = 1) => ({ + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [{ match: { 'host.name': 'host-1' } }], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [{ match: { 'host.ip': '192.168.0.0.1' } }], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + bool: { + should: [{ match: { 'destination.ip': '127.0.0.1' } }], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [{ match: { 'destination.port': port } }], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + bool: { + should: [{ match: { 'source.port': port } }], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + bool: { + should: [{ match: { 'source.ip': '127.0.0.1' } }], + minimum_should_match: 1, + }, + }, + ], + }, + }, + ], + minimum_should_match: 1, + }, +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts new file mode 100644 index 0000000000000..cf4a570248c99 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -0,0 +1,457 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ThreatMapping, + ThreatMappingEntries, +} from '../../../../../common/detection_engine/schemas/types/threat_mapping'; + +import { + filterThreatMapping, + buildThreatMappingFilter, + splitShouldClauses, + createInnerAndClauses, + createAndOrClauses, + buildEntriesMappingFilter, +} from './build_threat_mapping_filter'; +import { + getThreatMappingMock, + getThreatListSearchResponseMock, + getThreatListItemMock, + getThreatMappingFilterMock, + getFilterThreatMapping, + getThreatMappingFiltersShouldMock, + getThreatMappingFilterShouldMock, +} from './build_threat_mapping_filter.mock'; +import { BooleanFilter } from './types'; + +describe('build_threat_mapping_filter', () => { + describe('buildThreatMappingFilter', () => { + test('it should throw if given a chunk over 1024 in size', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + expect(() => + buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1025 }) + ).toThrow('chunk sizes cannot exceed 1024 in size'); + }); + + test('it should NOT throw if given a chunk under 1024 in size', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + expect(() => + buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023 }) + ).not.toThrow(); + }); + + test('it should create the correct entries when using the default mocks', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + const filter = buildThreatMappingFilter({ threatMapping, threatList }); + expect(filter).toEqual(getThreatMappingFilterMock()); + }); + + test('it should not mutate the original threatMapping', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + buildThreatMappingFilter({ threatMapping, threatList }); + expect(threatMapping).toEqual(getThreatMappingMock()); + }); + + test('it should not mutate the original threatListItem', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + buildThreatMappingFilter({ threatMapping, threatList }); + expect(threatList).toEqual(getThreatListSearchResponseMock()); + }); + }); + + describe('filterThreatMapping', () => { + test('it should not remove any entries when using the default mocks', () => { + const threatMapping = getThreatMappingMock(); + const threatListItem = getThreatListItemMock(); + + const item = filterThreatMapping({ threatMapping, threatListItem }); + const expected = getFilterThreatMapping(); + expect(item).toEqual(expected); + }); + + test('it should only give one filtered element if only 1 element is defined', () => { + const [firstElement] = getThreatMappingMock(); // get only the first element + const threatListItem = getThreatListItemMock(); + + const item = filterThreatMapping({ threatMapping: [firstElement], threatListItem }); + const [firstElementFilter] = getFilterThreatMapping(); // get only the first element to compare + expect(item).toEqual([firstElementFilter]); + }); + + test('it should not mutate the original threatMapping', () => { + const threatMapping = getThreatMappingMock(); + const threatListItem = getThreatListItemMock(); + + filterThreatMapping({ + threatMapping, + threatListItem, + }); + expect(threatMapping).toEqual(getThreatMappingMock()); + }); + + test('it should not mutate the original threatListItem', () => { + const threatMapping = getThreatMappingMock(); + const threatListItem = getThreatListItemMock(); + + filterThreatMapping({ + threatMapping, + threatListItem, + }); + expect(threatListItem).toEqual(getThreatListItemMock()); + }); + }); + + describe('createInnerAndClauses', () => { + test('it should return two clauses given a single entry', () => { + const [{ entries: threatMappingEntries }] = getThreatMappingMock(); // get the first element + const threatListItem = getThreatListItemMock(); + const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); + const { + bool: { + should: [ + { + bool: { filter }, + }, + ], + }, + } = getThreatMappingFilterShouldMock(); // get the first element + expect(innerClause).toEqual(filter); + }); + + test('it should return an empty array given an empty array', () => { + const threatListItem = getThreatListItemMock(); + const innerClause = createInnerAndClauses({ threatMappingEntries: [], threatListItem }); + expect(innerClause).toEqual([]); + }); + + test('it should filter out a single unknown value', () => { + const [{ entries }] = getThreatMappingMock(); // get the first element + const threatMappingEntries: ThreatMappingEntries = [ + ...entries, + { + field: 'host.name', // add second invalid entry which should be filtered away + value: 'invalid', + type: 'mapping', + }, + ]; + const threatListItem = getThreatListItemMock(); + const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); + const { + bool: { + should: [ + { + bool: { filter }, + }, + ], + }, + } = getThreatMappingFilterShouldMock(); // get the first element + expect(innerClause).toEqual(filter); + }); + + test('it should filter out 2 unknown values', () => { + const [{ entries }] = getThreatMappingMock(); // get the first element + const threatMappingEntries: ThreatMappingEntries = [ + ...entries, + { + field: 'host.name', // add second invalid entry which should be filtered away + value: 'invalid', + type: 'mapping', + }, + { + field: 'host.ip', // add second invalid entry which should be filtered away + value: 'invalid', + type: 'mapping', + }, + ]; + const threatListItem = getThreatListItemMock(); + const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); + const { + bool: { + should: [ + { + bool: { filter }, + }, + ], + }, + } = getThreatMappingFilterShouldMock(); // get the first element + expect(innerClause).toEqual(filter); + }); + + test('it should filter out all unknown values as an empty array', () => { + const threatMappingEntries: ThreatMappingEntries = [ + { + field: 'host.name', // add second invalid entry which should be filtered away + value: 'invalid', + type: 'mapping', + }, + { + field: 'host.ip', // add second invalid entry which should be filtered away + value: 'invalid', + type: 'mapping', + }, + ]; + const threatListItem = getThreatListItemMock(); + const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); + expect(innerClause).toEqual([]); + }); + }); + + describe('createAndOrClauses', () => { + test('it should return all clauses given the entries', () => { + const threatMapping = getThreatMappingMock(); + const threatListItem = getThreatListItemMock(); + const innerClause = createAndOrClauses({ threatMapping, threatListItem }); + expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); + }); + + test('it should filter out data from entries that do not have mappings', () => { + const threatMapping = getThreatMappingMock(); + const threatListItem = { ...getThreatListItemMock(), foo: 'bar' }; + const innerClause = createAndOrClauses({ threatMapping, threatListItem }); + expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); + }); + + test('it should return an empty boolean given an empty array', () => { + const threatListItem = getThreatListItemMock(); + const innerClause = createAndOrClauses({ threatMapping: [], threatListItem }); + expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); + }); + + test('it should return an empty boolean clause given an empty object for a threat list item', () => { + const threatMapping = getThreatMappingMock(); + const innerClause = createAndOrClauses({ threatMapping, threatListItem: {} }); + expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); + }); + }); + + describe('buildEntriesMappingFilter', () => { + test('it should return all clauses given the entries', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + const mapping = buildEntriesMappingFilter({ + threatMapping, + threatList, + chunkSize: 1024, + }); + const expected: BooleanFilter = { + bool: { should: [getThreatMappingFilterShouldMock()], minimum_should_match: 1 }, + }; + expect(mapping).toEqual(expected); + }); + + test('it should return empty "should" given an empty threat list', () => { + const threatMapping = getThreatMappingMock(); + const threatList = getThreatListSearchResponseMock(); + threatList.hits.hits = []; + const mapping = buildEntriesMappingFilter({ + threatMapping, + threatList, + chunkSize: 1024, + }); + const expected: BooleanFilter = { + bool: { should: [], minimum_should_match: 1 }, + }; + expect(mapping).toEqual(expected); + }); + + test('it should return empty "should" given an empty threat mapping', () => { + const threatList = getThreatListSearchResponseMock(); + const mapping = buildEntriesMappingFilter({ + threatMapping: [], + threatList, + chunkSize: 1024, + }); + const expected: BooleanFilter = { + bool: { should: [], minimum_should_match: 1 }, + }; + expect(mapping).toEqual(expected); + }); + + test('it should ignore entries that are invalid', () => { + const entries: ThreatMappingEntries = [ + { + field: 'host.name', + type: 'mapping', + value: 'invalid', + }, + { + field: 'host.ip', + type: 'mapping', + value: 'invalid', + }, + ]; + + const threatMapping: ThreatMapping = [ + ...getThreatMappingMock(), + ...[ + { + entries, + }, + ], + ]; + const threatList = getThreatListSearchResponseMock(); + const mapping = buildEntriesMappingFilter({ + threatMapping, + threatList, + chunkSize: 1024, + }); + const expected: BooleanFilter = { + bool: { should: [getThreatMappingFilterShouldMock()], minimum_should_match: 1 }, + }; + expect(mapping).toEqual(expected); + }); + }); + + describe('splitShouldClauses', () => { + test('it should NOT split a single should clause as there is nothing to split on with chunkSize 1', () => { + const should = getThreatMappingFiltersShouldMock(); + const clauses = splitShouldClauses({ should, chunkSize: 1 }); + expect(clauses).toEqual(getThreatMappingFiltersShouldMock()); + }); + + test('it should NOT mutate the original should clause passed in', () => { + const should = getThreatMappingFiltersShouldMock(); + expect(should).toEqual(getThreatMappingFiltersShouldMock()); + }); + + test('it should NOT split a single should clause as there is nothing to split on with chunkSize 2', () => { + const should = getThreatMappingFiltersShouldMock(); + const clauses = splitShouldClauses({ should, chunkSize: 2 }); + expect(clauses).toEqual(getThreatMappingFiltersShouldMock()); + }); + + test('it should return an empty array given an empty array', () => { + const clauses = splitShouldClauses({ should: [], chunkSize: 2 }); + expect(clauses).toEqual([]); + }); + + test('it should split an array of size 2 into a length 2 array with chunks on "chunkSize: 1"', () => { + const should = getThreatMappingFiltersShouldMock(2); + const clauses = splitShouldClauses({ should, chunkSize: 1 }); + expect(clauses.length).toEqual(2); + }); + + test('it should not mutate the original when splitting on chunks', () => { + const should = getThreatMappingFiltersShouldMock(2); + splitShouldClauses({ should, chunkSize: 1 }); + expect(should).toEqual(getThreatMappingFiltersShouldMock(2)); + }); + + test('it should split an array of size 2 into 2 different chunks on "chunkSize: 1"', () => { + const should = getThreatMappingFiltersShouldMock(2); + const clauses = splitShouldClauses({ should, chunkSize: 1 }); + const expected: BooleanFilter[] = [ + { + bool: { + should: [getThreatMappingFilterShouldMock(1)], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(2)], + minimum_should_match: 1, + }, + }, + ]; + expect(clauses).toEqual(expected); + }); + + test('it should split an array of size 4 into 4 groups of 4 chunks on "chunkSize: 1"', () => { + const should = getThreatMappingFiltersShouldMock(4); + const clauses = splitShouldClauses({ should, chunkSize: 1 }); + const expected: BooleanFilter[] = [ + { + bool: { + should: [getThreatMappingFilterShouldMock(1)], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(2)], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(3)], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(4)], + minimum_should_match: 1, + }, + }, + ]; + expect(clauses).toEqual(expected); + }); + + test('it should split an array of size 4 into 2 groups of 2 chunks on "chunkSize: 2"', () => { + const should = getThreatMappingFiltersShouldMock(4); + const clauses = splitShouldClauses({ should, chunkSize: 2 }); + const expected: BooleanFilter[] = [ + { + bool: { + should: [getThreatMappingFilterShouldMock(1), getThreatMappingFilterShouldMock(2)], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(3), getThreatMappingFilterShouldMock(4)], + minimum_should_match: 1, + }, + }, + ]; + expect(clauses).toEqual(expected); + }); + + test('it should NOT split an array of size 4 into any groups on "chunkSize: 5"', () => { + const should = getThreatMappingFiltersShouldMock(4); + const clauses = splitShouldClauses({ should, chunkSize: 5 }); + const expected: BooleanFilter[] = [ + getThreatMappingFilterShouldMock(1), + getThreatMappingFilterShouldMock(2), + getThreatMappingFilterShouldMock(3), + getThreatMappingFilterShouldMock(4), + ]; + expect(clauses).toEqual(expected); + }); + + test('it should split an array of size 4 into 2 groups on "chunkSize: 3"', () => { + const should = getThreatMappingFiltersShouldMock(4); + const clauses = splitShouldClauses({ should, chunkSize: 3 }); + const expected: BooleanFilter[] = [ + { + bool: { + should: [ + getThreatMappingFilterShouldMock(1), + getThreatMappingFilterShouldMock(2), + getThreatMappingFilterShouldMock(3), + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [getThreatMappingFilterShouldMock(4)], + minimum_should_match: 1, + }, + }, + ]; + expect(clauses).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts new file mode 100644 index 0000000000000..3299b6ae34e4d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import get from 'lodash/fp/get'; +import { Filter } from 'src/plugins/data/common'; +import { ThreatMapping } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; +import { + BooleanFilter, + BuildEntriesMappingFilterOptions, + BuildThreatMappingFilterOptions, + CreateAndOrClausesOptions, + CreateInnerAndClausesOptions, + FilterThreatMappingOptions, + SplitShouldClausesOptions, +} from './types'; + +export const MAX_CHUNK_SIZE = 1024; + +export const buildThreatMappingFilter = ({ + threatMapping, + threatList, + chunkSize, +}: BuildThreatMappingFilterOptions): Filter => { + const computedChunkSize = chunkSize ?? MAX_CHUNK_SIZE; + if (computedChunkSize > 1024) { + throw new TypeError('chunk sizes cannot exceed 1024 in size'); + } + const query = buildEntriesMappingFilter({ + threatMapping, + threatList, + chunkSize: computedChunkSize, + }); + const filterChunk: Filter = { + meta: { + alias: null, + negate: false, + disabled: false, + }, + query, + }; + return filterChunk; +}; + +/** + * Filters out any entries which do not include the threat list item. + */ +export const filterThreatMapping = ({ + threatMapping, + threatListItem, +}: FilterThreatMappingOptions): ThreatMapping => + threatMapping + .map((threatMap) => { + const entries = threatMap.entries.filter((entry) => get(entry.value, threatListItem) != null); + return { ...threatMap, entries }; + }) + .filter((threatMap) => threatMap.entries.length !== 0); + +export const createInnerAndClauses = ({ + threatMappingEntries, + threatListItem, +}: CreateInnerAndClausesOptions): BooleanFilter[] => { + return threatMappingEntries.reduce((accum, threatMappingEntry) => { + const value = get(threatMappingEntry.value, threatListItem); + if (value != null) { + // These values could be potentially 10k+ large so mutating the array intentionally + accum.push({ + bool: { + should: [ + { + match: { + [threatMappingEntry.field]: value, + }, + }, + ], + minimum_should_match: 1, + }, + }); + } + return accum; + }, []); +}; + +export const createAndOrClauses = ({ + threatMapping, + threatListItem, +}: CreateAndOrClausesOptions): BooleanFilter => { + const should = threatMapping.reduce((accum, threatMap) => { + const innerAndClauses = createInnerAndClauses({ + threatMappingEntries: threatMap.entries, + threatListItem, + }); + if (innerAndClauses.length !== 0) { + // These values could be potentially 10k+ large so mutating the array intentionally + accum.push({ + bool: { filter: innerAndClauses }, + }); + } + return accum; + }, []); + return { bool: { should, minimum_should_match: 1 } }; +}; + +export const buildEntriesMappingFilter = ({ + threatMapping, + threatList, + chunkSize, +}: BuildEntriesMappingFilterOptions): BooleanFilter => { + const combinedShould = threatList.hits.hits.reduce( + (accum, threatListSearchItem) => { + const filteredEntries = filterThreatMapping({ + threatMapping, + threatListItem: threatListSearchItem._source, + }); + const queryWithAndOrClause = createAndOrClauses({ + threatMapping: filteredEntries, + threatListItem: threatListSearchItem._source, + }); + if (queryWithAndOrClause.bool.should.length !== 0) { + // These values can be 10k+ large, so using a push here for performance + accum.push(queryWithAndOrClause); + } + return accum; + }, + [] + ); + const should = splitShouldClauses({ should: combinedShould, chunkSize }); + return { bool: { should, minimum_should_match: 1 } }; +}; + +export const splitShouldClauses = ({ + should, + chunkSize, +}: SplitShouldClausesOptions): BooleanFilter[] => { + if (should.length <= chunkSize) { + return should; + } else { + return should.reduce((accum, item, index) => { + const chunkIndex = Math.floor(index / chunkSize); + const currentChunk = accum[chunkIndex]; + if (!currentChunk) { + // create a new element in the array at the correct spot + accum[chunkIndex] = { bool: { should: [], minimum_should_match: 1 } }; + } + // Add to the existing array element. Using mutatious push here since these arrays can get very large such as 10k+ and this is going to be a hot code spot. + accum[chunkIndex].bool.should.push(item); + return accum; + }, []); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts new file mode 100644 index 0000000000000..7542128d83769 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { getThreatList } from './get_threat_list'; +import { buildThreatMappingFilter } from './build_threat_mapping_filter'; + +import { getFilter } from '../get_filter'; +import { + searchAfterAndBulkCreate, + SearchAfterAndBulkCreateReturnType, +} from '../search_after_bulk_create'; +import { CreateThreatSignalOptions, ThreatListItem } from './types'; +import { combineResults } from './utils'; + +export const createThreatSignal = async ({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + interval, + updatedAt, + enabled, + refresh, + tags, + throttle, + threatFilters, + threatQuery, + buildRuleMessage, + threatIndex, + name, + currentThreatList, + currentResult, +}: CreateThreatSignalOptions): Promise<{ + threatList: SearchResponse; + results: SearchAfterAndBulkCreateReturnType; +}> => { + const threatFilter = buildThreatMappingFilter({ + threatMapping, + threatList: currentThreatList, + }); + + const esFilter = await getFilter({ + type, + filters: [...filters, threatFilter], + language, + query, + savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); + + const newResult = await searchAfterAndBulkCreate({ + gap, + previousStartedAt, + listClient, + exceptionsList: exceptionItems, + ruleParams: params, + services, + logger, + id: alertId, + inputIndexPattern: inputIndex, + signalsIndex: outputIndex, + filter: esFilter, + actions, + name, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + pageSize: searchAfterSize, + refresh, + tags, + throttle, + buildRuleMessage, + }); + + const results = combineResults(currentResult, newResult); + const searchAfter = currentThreatList.hits.hits[currentThreatList.hits.hits.length - 1].sort; + + const threatList = await getThreatList({ + callCluster: services.callCluster, + exceptionItems, + query: threatQuery, + threatFilters, + index: [threatIndex], + searchAfter, + sortField: undefined, + sortOrder: undefined, + }); + + return { threatList, results }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts new file mode 100644 index 0000000000000..9027475d71c4a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getThreatList } from './get_threat_list'; + +import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; +import { CreateThreatSignalsOptions } from './types'; +import { createThreatSignal } from './create_threat_signal'; + +export const createThreatSignals = async ({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + interval, + updatedAt, + enabled, + refresh, + tags, + throttle, + threatFilters, + threatQuery, + buildRuleMessage, + threatIndex, + name, +}: CreateThreatSignalsOptions): Promise => { + let results: SearchAfterAndBulkCreateReturnType = { + success: true, + bulkCreateTimes: [], + searchAfterTimes: [], + lastLookBackDate: null, + createdSignalsCount: 0, + errors: [], + }; + + let threatList = await getThreatList({ + callCluster: services.callCluster, + exceptionItems, + threatFilters, + query: threatQuery, + index: [threatIndex], + searchAfter: undefined, + sortField: undefined, + sortOrder: undefined, + }); + + while (threatList.hits.hits.length !== 0 && results.createdSignalsCount <= params.maxSignals) { + ({ threatList, results } = await createThreatSignal({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + tags, + refresh, + throttle, + threatFilters, + threatQuery, + buildRuleMessage, + threatIndex, + name, + currentThreatList: threatList, + currentResult: results, + })); + } + return results; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts new file mode 100644 index 0000000000000..f600463c213c2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getSortWithTieBreaker } from './get_threat_list'; + +describe('get_threat_signals', () => { + describe('getSortWithTieBreaker', () => { + test('it should return sort field of just timestamp if given no sort order', () => { + const sortOrder = getSortWithTieBreaker({ sortField: undefined, sortOrder: undefined }); + expect(sortOrder).toEqual([{ '@timestamp': 'asc' }]); + }); + + test('it should return sort field of timestamp with asc even if sortOrder is changed as it is hard wired in', () => { + const sortOrder = getSortWithTieBreaker({ sortField: undefined, sortOrder: 'desc' }); + expect(sortOrder).toEqual([{ '@timestamp': 'asc' }]); + }); + + test('it should return sort field of an extra field if given one', () => { + const sortOrder = getSortWithTieBreaker({ sortField: 'some-field', sortOrder: undefined }); + expect(sortOrder).toEqual([{ 'some-field': 'asc', '@timestamp': 'asc' }]); + }); + + test('it should return sort field of desc if given one', () => { + const sortOrder = getSortWithTieBreaker({ sortField: 'some-field', sortOrder: 'desc' }); + expect(sortOrder).toEqual([{ 'some-field': 'desc', '@timestamp': 'asc' }]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts new file mode 100644 index 0000000000000..8b381ca0d96dc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter'; +import { + GetSortWithTieBreakerOptions, + GetThreatListOptions, + SortWithTieBreaker, + ThreatListItem, +} from './types'; + +/** + * This should not exceed 10000 (10k) + */ +export const MAX_PER_PAGE = 9000; + +export const getThreatList = async ({ + callCluster, + query, + index, + perPage, + searchAfter, + sortField, + sortOrder, + exceptionItems, + threatFilters, +}: GetThreatListOptions): Promise> => { + const calculatedPerPage = perPage ?? MAX_PER_PAGE; + if (calculatedPerPage > 10000) { + throw new TypeError('perPage cannot exceed the size of 10000'); + } + const queryFilter = getQueryFilter(query, 'kuery', threatFilters, index, exceptionItems); + const response: SearchResponse = await callCluster('search', { + body: { + query: queryFilter, + search_after: searchAfter, + sort: getSortWithTieBreaker({ sortField, sortOrder }), + }, + ignoreUnavailable: true, + index, + size: calculatedPerPage, + }); + return response; +}; + +export const getSortWithTieBreaker = ({ + sortField, + sortOrder, +}: GetSortWithTieBreakerOptions): SortWithTieBreaker[] => { + const ascOrDesc = sortOrder ?? 'asc'; + if (sortField != null) { + return [{ [sortField]: ascOrDesc, '@timestamp': 'asc' }]; + } else { + return [{ '@timestamp': 'asc' }]; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts new file mode 100644 index 0000000000000..4c3cd9943adb4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Duration } from 'moment'; +import { SearchResponse } from 'elasticsearch'; +import { ListClient } from '../../../../../../lists/server'; +import { + Type, + LanguageOrUndefined, +} from '../../../../../common/detection_engine/schemas/common/schemas'; +import { + ThreatQuery, + ThreatMapping, + ThreatMappingEntries, +} from '../../../../../common/detection_engine/schemas/types/threat_mapping'; +import { PartialFilter, RuleTypeParams } from '../../types'; +import { AlertServices } from '../../../../../../alerts/server'; +import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas'; +import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; +import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/core/server'; +import { RuleAlertAction } from '../../../../../common/detection_engine/types'; +import { BuildRuleMessage } from '../rule_messages'; + +export interface CreateThreatSignalsOptions { + threatMapping: ThreatMapping; + query: string; + inputIndex: string[]; + type: Type; + filters: PartialFilter[]; + language: LanguageOrUndefined; + savedId: string | undefined; + services: AlertServices; + exceptionItems: ExceptionListItemSchema[]; + gap: Duration | null; + previousStartedAt: Date | null; + listClient: ListClient; + logger: Logger; + alertId: string; + outputIndex: string; + params: RuleTypeParams; + searchAfterSize: number; + actions: RuleAlertAction[]; + createdBy: string; + createdAt: string; + updatedBy: string; + updatedAt: string; + interval: string; + enabled: boolean; + tags: string[]; + refresh: false | 'wait_for'; + throttle: string; + threatFilters: PartialFilter[]; + threatQuery: ThreatQuery; + buildRuleMessage: BuildRuleMessage; + threatIndex: string; + name: string; +} + +export interface CreateThreatSignalOptions { + threatMapping: ThreatMapping; + query: string; + inputIndex: string[]; + type: Type; + filters: PartialFilter[]; + language: LanguageOrUndefined; + savedId: string | undefined; + services: AlertServices; + exceptionItems: ExceptionListItemSchema[]; + gap: Duration | null; + previousStartedAt: Date | null; + listClient: ListClient; + logger: Logger; + alertId: string; + outputIndex: string; + params: RuleTypeParams; + searchAfterSize: number; + actions: RuleAlertAction[]; + createdBy: string; + createdAt: string; + updatedBy: string; + updatedAt: string; + interval: string; + enabled: boolean; + tags: string[]; + refresh: false | 'wait_for'; + throttle: string; + threatFilters: PartialFilter[]; + threatQuery: ThreatQuery; + buildRuleMessage: BuildRuleMessage; + threatIndex: string; + name: string; + currentThreatList: SearchResponse; + currentResult: SearchAfterAndBulkCreateReturnType; +} + +export interface BuildThreatMappingFilterOptions { + threatMapping: ThreatMapping; + threatList: SearchResponse; + chunkSize?: number; +} + +export interface FilterThreatMappingOptions { + threatMapping: ThreatMapping; + threatListItem: ThreatListItem; +} + +export interface CreateInnerAndClausesOptions { + threatMappingEntries: ThreatMappingEntries; + threatListItem: ThreatListItem; +} + +export interface CreateAndOrClausesOptions { + threatMapping: ThreatMapping; + threatListItem: ThreatListItem; +} + +export interface BuildEntriesMappingFilterOptions { + threatMapping: ThreatMapping; + threatList: SearchResponse; + chunkSize: number; +} + +export interface SplitShouldClausesOptions { + should: BooleanFilter[]; + chunkSize: number; +} + +export interface BooleanFilter { + bool: { should: unknown[]; minimum_should_match: number }; +} + +export interface GetThreatListOptions { + callCluster: ILegacyScopedClusterClient['callAsCurrentUser']; + query: string; + index: string[]; + perPage?: number; + searchAfter: string[] | undefined; + sortField: string | undefined; + sortOrder: 'asc' | 'desc' | undefined; + threatFilters: PartialFilter[]; + exceptionItems: ExceptionListItemSchema[]; +} + +export interface GetSortWithTieBreakerOptions { + sortField: string | undefined; + sortOrder: 'asc' | 'desc' | undefined; +} + +/** + * This is an ECS document being returned, but the user could return or use non-ecs based + * documents potentially. + */ +export interface ThreatListItem { + [key: string]: unknown; +} + +export interface SortWithTieBreaker { + '@timestamp': 'asc'; + [key: string]: string; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts new file mode 100644 index 0000000000000..48bdf430b940e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; + +import { calculateAdditiveMax, combineResults } from './utils'; + +describe('utils', () => { + describe('calculateAdditiveMax', () => { + test('it should return 0 for two empty arrays', () => { + const max = calculateAdditiveMax([], []); + expect(max).toEqual(['0']); + }); + + test('it should return 10 for two arrays with the numbers 5', () => { + const max = calculateAdditiveMax(['5'], ['5']); + expect(max).toEqual(['10']); + }); + + test('it should return 5 for two arrays with second array having just 5', () => { + const max = calculateAdditiveMax([], ['5']); + expect(max).toEqual(['5']); + }); + + test('it should return 5 for two arrays with first array having just 5', () => { + const max = calculateAdditiveMax(['5'], []); + expect(max).toEqual(['5']); + }); + + test('it should return 10 for the max of the two arrays added together when the max of each array is 5, "5 + 5 = 10"', () => { + const max = calculateAdditiveMax(['3', '5', '1'], ['3', '5', '1']); + expect(max).toEqual(['10']); + }); + }); + + describe('combineResults', () => { + test('it should combine two results with success set to "true" if both are "true"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineResults(existingResult, newResult); + expect(combinedResults.success).toEqual(true); + }); + + test('it should combine two results with success set to "false" if one of them is "false"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineResults(existingResult, newResult); + expect(combinedResults.success).toEqual(false); + }); + + test('it should use the latest date if it is set in the new result', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineResults(existingResult, newResult); + expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); + }); + + test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineResults(existingResult, newResult); + expect(combinedResults).toEqual( + expect.objectContaining({ + searchAfterTimes: ['60'], + bulkCreateTimes: ['50'], + }) + ); + }); + + test('it should combine errors together without duplicates', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: ['error 1', 'error 2', 'error 3'], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: ['error 4', 'error 1', 'error 3', 'error 5'], + }; + const combinedResults = combineResults(existingResult, newResult); + expect(combinedResults).toEqual( + expect.objectContaining({ + errors: ['error 1', 'error 2', 'error 3', 'error 4', 'error 5'], + }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts new file mode 100644 index 0000000000000..38bbb70b6c4ec --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; + +/** + * Given two timers this will take the max of each and add them to each other and return that addition. + * Max(timer_array_1) + Max(timer_array_2) + * @param existingTimers String array of existing timers + * @param newTimers String array of new timers. + * @returns String array of the new maximum between the two timers + */ +export const calculateAdditiveMax = (existingTimers: string[], newTimers: string[]): string[] => { + const numericNewTimerMax = Math.max(0, ...newTimers.map((time) => +time)); + const numericExistingTimerMax = Math.max(0, ...existingTimers.map((time) => +time)); + return [String(numericNewTimerMax + numericExistingTimerMax)]; +}; + +/** + * Combines two results together and returns the results combined + * @param currentResult The current result to combine with a newResult + * @param newResult The new result to combine + */ +export const combineResults = ( + currentResult: SearchAfterAndBulkCreateReturnType, + newResult: SearchAfterAndBulkCreateReturnType +): SearchAfterAndBulkCreateReturnType => ({ + success: currentResult.success === false ? false : newResult.success, + bulkCreateTimes: calculateAdditiveMax(currentResult.bulkCreateTimes, newResult.bulkCreateTimes), + searchAfterTimes: calculateAdditiveMax( + currentResult.searchAfterTimes, + newResult.searchAfterTimes + ), + lastLookBackDate: newResult.lastLookBackDate, + createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, + errors: [...new Set([...currentResult.errors, ...newResult.errors])], +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index b101bc5754764..23aa786558a99 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { DslQuery, Filter } from 'src/plugins/data/common'; import { Status } from '../../../../common/detection_engine/schemas/common/schemas'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { AlertType, AlertTypeState, AlertExecutorOptions } from '../../../../../alerts/server'; @@ -166,3 +167,15 @@ export interface RuleAlertAttributes extends AlertAttributes { } export type BulkResponseErrorAggregation = Record; + +/** + * TODO: Remove this if/when the return filter has its own type exposed + */ +export interface QueryFilter { + bool: { + must: DslQuery[]; + filter: Filter[]; + should: unknown[]; + must_not: Filter[]; + }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index cbe756064b72b..b0554adcc46b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -38,6 +38,12 @@ import { TimestampOverrideOrUndefined, Type, } from '../../../common/detection_engine/schemas/common/schemas'; +import { + ThreatIndexOrUndefined, + ThreatQueryOrUndefined, + ThreatMappingOrUndefined, +} from '../../../common/detection_engine/schemas/types/threat_mapping'; + import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; import { Filter } from '../../../../../../src/plugins/data/server'; import { ListArrayOrUndefined } from '../../../common/detection_engine/schemas/types'; @@ -73,6 +79,10 @@ export interface RuleTypeParams { severityMapping: SeverityMappingOrUndefined; threat: ThreatOrUndefined; threshold: ThresholdOrUndefined; + threatFilters: PartialFilter[] | undefined; + threatIndex: ThreatIndexOrUndefined; + threatQuery: ThreatQueryOrUndefined; + threatMapping: ThreatMappingOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type;