[Security Solution] Adds logging and performance fan out API for threat/Indicator matching

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -48,6 +48,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  concurrent_searches,
+  items_per_search,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -130,6 +132,8 @@ export const addPrepackagedRulesSchema = t.intersection([
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      concurrent_searches, // defaults to "undefined" if not set during decode
+      items_per_search, // defaults to "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts

@@ -1702,5 +1702,23 @@ describe('create rules schema', () => {
       expect(getPaths(left(message.errors))).toEqual([]);
       expect(message.schema).toEqual(expected);
     });
+
+    test('You can set a threat query, index, mapping, filters, concurrent_searches, items_per_search with a when creating a rule', () => {
+      const payload: CreateRulesSchema = {
+        ...getCreateThreatMatchRulesSchemaMock(),
+        concurrent_searches: 10,
+        items_per_search: 10,
+      };
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected: CreateRulesSchemaDecoded = {
+        ...getCreateThreatMatchRulesSchemaDecodedMock(),
+        concurrent_searches: 10,
+        items_per_search: 10,
+      };
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
   });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts

@@ -49,6 +49,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  concurrent_searches,
+  items_per_search,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -126,6 +128,8 @@ export const createRulesSchema = t.intersection([
       threat_filters, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      concurrent_searches, // defaults "undefined" if not set during decode
+      items_per_search, // defaults "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts

@@ -125,4 +125,36 @@ describe('create_rules_type_dependents', () => {
     const errors = createRuleValidateTypeDependents(schema);
     expect(errors).toEqual([]);
   });
+
+  test('validates that both "items_per_search" and "concurrent_searches" works when together', () => {
+    const schema: CreateRulesSchema = {
+      ...getCreateThreatMatchRulesSchemaMock(),
+      concurrent_searches: 10,
+      items_per_search: 10,
+    };
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual([]);
+  });
+
+  test('does NOT validate when only "items_per_search" is present', () => {
+    const schema: CreateRulesSchema = {
+      ...getCreateThreatMatchRulesSchemaMock(),
+      items_per_search: 10,
+    };
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual([
+      'when "items_per_search" exists, "concurrent_searches" must also exist',
+    ]);
+  });
+
+  test('does NOT validate when only "concurrent_searches" is present', () => {
+    const schema: CreateRulesSchema = {
+      ...getCreateThreatMatchRulesSchemaMock(),
+      concurrent_searches: 10,
+    };
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual([
+      'when "concurrent_searches" exists, "items_per_search" must also exist',
+    ]);
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts

@@ -110,17 +110,23 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => {
 export const validateThreatMapping = (rule: CreateRulesSchema): string[] => {
   let errors: string[] = [];
   if (isThreatMatchRule(rule.type)) {
-    if (!rule.threat_mapping) {
+    if (rule.threat_mapping == null) {
       errors = ['when "type" is "threat_match", "threat_mapping" is required', ...errors];
     } else if (rule.threat_mapping.length === 0) {
       errors = ['threat_mapping" must have at least one element', ...errors];
     }
-    if (!rule.threat_query) {
+    if (rule.threat_query == null) {
       errors = ['when "type" is "threat_match", "threat_query" is required', ...errors];
     }
-    if (!rule.threat_index) {
+    if (rule.threat_index == null) {
       errors = ['when "type" is "threat_match", "threat_index" is required', ...errors];
     }
+    if (rule.concurrent_searches == null && rule.items_per_search != null) {
+      errors = ['when "items_per_search" exists, "concurrent_searches" must also exist', ...errors];
+    }
+    if (rule.concurrent_searches != null && rule.items_per_search == null) {
+      errors = ['when "concurrent_searches" exists, "items_per_search" must also exist', ...errors];
+    }
   }
   return errors;
 };

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -55,6 +55,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  items_per_search,
+  concurrent_searches,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -149,6 +151,8 @@ export const importRulesSchema = t.intersection([
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      concurrent_searches, // defaults to "undefined" if not set during decode
+      items_per_search, // defaults to "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -50,6 +50,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  concurrent_searches,
+  items_per_search,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -109,6 +111,8 @@ export const patchRulesSchema = t.exact(
     threat_filters,
     threat_mapping,
     threat_language,
+    concurrent_searches,
+    items_per_search,
   })
 );
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts

@@ -51,6 +51,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  concurrent_searches,
+  items_per_search,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -134,6 +136,8 @@ export const updateRulesSchema = t.intersection([
       threat_filters, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      concurrent_searches, // defaults to "undefined" if not set during decode
+      items_per_search, // defaults to "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts

@@ -762,9 +762,9 @@ describe('rules_schema', () => {
       expect(fields).toEqual(expected);
     });
 
-    test('should return 5 fields for a rule of type "threat_match"', () => {
+    test('should return 8 fields for a rule of type "threat_match"', () => {
       const fields = addThreatMatchFields({ type: 'threat_match' });
-      expect(fields.length).toEqual(6);
+      expect(fields.length).toEqual(8);
     });
   });
 

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts

@@ -63,6 +63,8 @@ import {
 } from '../common/schemas';
 import {
   threat_index,
+  concurrent_searches,
+  items_per_search,
   threat_query,
   threat_filters,
   threat_mapping,
@@ -144,6 +146,8 @@ export const dependentRulesSchema = t.partial({
   threat_filters,
   threat_index,
   threat_query,
+  concurrent_searches,
+  items_per_search,
   threat_mapping,
   threat_language,
 });
@@ -282,6 +286,12 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly):
       t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })),
       t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })),
       t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })),
+      t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })),
+      t.exact(
+        t.partial({
+          items_per_search: dependentRulesSchema.props.items_per_search,
+        })
+      ),
     ];
   } else {
     return [];

x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts

@@ -5,6 +5,8 @@
  */
 
 import {
+  concurrent_searches,
+  items_per_search,
   ThreatMapping,
   threatMappingEntries,
   ThreatMappingEntries,
@@ -33,7 +35,7 @@ describe('threat_mapping', () => {
       expect(message.schema).toEqual(payload);
     });
 
-    test('it should NOT validate an extra entry item', () => {
+    test('it should fail validation with an extra entry item', () => {
       const payload: ThreatMappingEntries & Array<{ extra: string }> = [
         {
           field: 'field.one',
@@ -50,7 +52,7 @@ describe('threat_mapping', () => {
       expect(message.schema).toEqual({});
     });
 
-    test('it should NOT validate a non string', () => {
+    test('it should fail validation with a non string', () => {
       const payload = ([
         {
           field: 5,
@@ -66,7 +68,7 @@ describe('threat_mapping', () => {
       expect(message.schema).toEqual({});
     });
 
-    test('it should NOT validate a wrong type', () => {
+    test('it should fail validation with a wrong type', () => {
       const payload = ([
         {
           field: 'field.one',
@@ -107,7 +109,7 @@ describe('threat_mapping', () => {
     });
   });
 
-  test('it should NOT validate an extra key', () => {
+  test('it should fail validate with an extra key', () => {
     const payload: ThreatMapping & Array<{ extra: string }> = [
       {
         entries: [
@@ -129,7 +131,7 @@ describe('threat_mapping', () => {
     expect(message.schema).toEqual({});
   });
 
-  test('it should NOT validate an extra inner entry', () => {
+  test('it should fail validate with an extra inner entry', () => {
     const payload: ThreatMapping & Array<{ entries: Array<{ extra: string }> }> = [
       {
         entries: [
@@ -151,7 +153,7 @@ describe('threat_mapping', () => {
     expect(message.schema).toEqual({});
   });
 
-  test('it should NOT validate an extra inner entry with the wrong data type', () => {
+  test('it should fail validate with an extra inner entry with the wrong data type', () => {
     const payload = ([
       {
         entries: [
@@ -173,4 +175,48 @@ describe('threat_mapping', () => {
     ]);
     expect(message.schema).toEqual({});
   });
+
+  test('it should fail validation when concurrent_searches is < 0', () => {
+    const payload = -1;
+    const decoded = concurrent_searches.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should fail validation when concurrent_searches is 0', () => {
+    const payload = 0;
+    const decoded = concurrent_searches.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should fail validation when items_per_search is 0', () => {
+    const payload = 0;
+    const decoded = items_per_search.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should fail validation when items_per_search is < 0', () => {
+    const payload = -1;
+    const decoded = items_per_search.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts

@@ -9,6 +9,7 @@
 import * as t from 'io-ts';
 import { language } from '../common/schemas';
 import { NonEmptyString } from './non_empty_string';
+import { PositiveIntegerGreaterThanZero } from './positive_integer_greater_than_zero';
 
 export const threat_query = t.string;
 export type ThreatQuery = t.TypeOf<typeof threat_query>;
@@ -55,3 +56,13 @@ export const threat_language = t.union([language, t.undefined]);
 export type ThreatLanguage = t.TypeOf<typeof threat_language>;
 export const threatLanguageOrUndefined = t.union([threat_language, t.undefined]);
 export type ThreatLanguageOrUndefined = t.TypeOf<typeof threatLanguageOrUndefined>;
+
+export const concurrent_searches = PositiveIntegerGreaterThanZero;
+export type ConcurrentSearches = t.TypeOf<typeof concurrent_searches>;
+export const concurrentSearchesOrUndefined = t.union([concurrent_searches, t.undefined]);
+export type ConcurrentSearchesOrUndefined = t.TypeOf<typeof concurrentSearchesOrUndefined>;
+
+export const items_per_search = PositiveIntegerGreaterThanZero;
+export type ItemsPerSearch = t.TypeOf<typeof concurrent_searches>;
+export const itemsPerSearchOrUndefined = t.union([items_per_search, t.undefined]);
+export type ItemsPerSearchOrUndefined = t.TypeOf<typeof itemsPerSearchOrUndefined>;

x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx

@@ -95,7 +95,7 @@ export const THREAT_MATCH_INDEX_HELPER_TEXT = i18n.translate(
 export const THREAT_MATCH_REQUIRED = i18n.translate(
   'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError',
   {
-    defaultMessage: 'At least one threat match is required.',
+    defaultMessage: 'At least one indicator match is required.',
   }
 );
 

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -407,6 +407,8 @@ export const getResult = (): RuleAlertType => ({
     note: '# Investigative notes',
     version: 1,
     exceptionsList: getListArrayMock(),
+    concurrentSearches: undefined,
+    itemsPerSearch: undefined,
   },
   createdAt: new Date('2019-12-13T16:40:33.400Z'),
   updatedAt: new Date('2019-12-13T16:40:33.400Z'),

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts

@@ -102,6 +102,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
               threat_mapping: threatMapping,
               threat_query: threatQuery,
               threat_language: threatLanguage,
+              concurrent_searches: concurrentSearches,
+              items_per_search: itemsPerSearch,
               threshold,
               throttle,
               timestamp_override: timestampOverride,
@@ -193,6 +195,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) =>
                 threatQuery,
                 threatIndex,
                 threatLanguage,
+                concurrentSearches,
+                itemsPerSearch,
                 threshold,
                 timestampOverride,
                 references,