[Security Solutions][Detection Engine] Adds threat matching API and rule type (#77395) (#77978)

## Summary

This is the backend, first iteration of threat matching API and rule type. You see elements using the backend API on the front end but cannot use the UI to add or edit a threshold rule with this PR.

Screen shots of it running in the UI elements that do work:
<img width="1862" alt="Screen Shot 2020-09-16 at 10 34 26 AM" src="https://user-images.githubusercontent.com/1151048/93366465-6e2b9c00-f808-11ea-923b-78e8d0fdfbaa.png">

<img width="1863" alt="Screen Shot 2020-09-16 at 10 34 48 AM" src="https://user-images.githubusercontent.com/1151048/93366476-71268c80-f808-11ea-8247-d2091ff1599a.png"> 

**Usage**
Since this is only backend API work and does not have the front end add/edit at the moment, you can use the existing UI's (for the most part) to validate the work here through CURL scripts below:

Go to the folder:
```ts
/kibana/x-pack/plugins/security_solution/server/lib/detection_engine/scripts
```

And post a small ECS threat mapping to the index called `mock-threat-list`:
```ts
./create_threat_mapping.sh
```

Then to post a small number of threats that represent simple port numbers you can run:
```ts
./create_threat_data.sh
```

However, feel free to also manually create them directly in your dev tools like so:

```ts
# Posts a threat list item called some-name with an IP but change these out for valid data in your system
PUT mock-threat-list-1/_doc/9999
{
  "@timestamp": "2020-09-09T20:30:45.725Z",
  "host": {
    "name": "some-name",
    "ip": "127.0.0.1"
  }
}
```

```ts
# Posts a destination port number to watch
PUT mock-threat-list-1/_doc/10000
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "destination": {
    "port": "443"
  }
}
```

```ts
# Posts a source port number to watch
PUT mock-threat-list-1/_doc/10001
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "source": {
    "port": "443"
  }
}
```

Then you can post a threat match rule:
```ts
./post_rule.sh ./rules/queries/query_with_threat_mapping.json
```
<details>
 <summary>Click here to see Response</summary>

```ts
{
  "actions": [],
  "author": [],
  "created_at": "2020-09-16T04:25:58.041Z",
  "created_by": "yo",
  "description": "Query with a threat mapping",
  "enabled": true,
  "exceptions_list": [],
  "false_positives": [],
  "from": "now-6m",
  "id": "f4226ab0-6f88-49c3-8f09-84cf5946ee7a",
  "immutable": false,
  "interval": "5m",
  "language": "kuery",
  "max_signals": 100,
  "name": "Query with a threat mapping",
  "output_index": ".siem-signals-hassanabad3-default",
  "query": "*:*",
  "references": [],
  "risk_score": 1,
  "risk_score_mapping": [],
  "rule_id": "threat-mapping",
  "severity": "high",
  "severity_mapping": [],
  "tags": [
    "tag_1",
    "tag_2"
  ],
  "threat": [],
  "threat_index": "mock-threat-list-1",
  "threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ],
  "threat_query": "*:*",
  "throttle": "no_actions",
  "to": "now",
  "type": "threat_match",
  "updated_at": "2020-09-16T04:25:58.051Z",
  "updated_by": "yo",
  "version": 1
}
```
</details>

**Structure**

You can see the rule structure in the file:
```ts
x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping.json
```
<details>
 <summary>Click here to see JSON</summary>

```ts
{
  "name": "Query with a threat mapping",
  "description": "Query with a threat mapping",
  "rule_id": "threat-mapping",
  "risk_score": 1,
  "severity": "high",
  "type": "threat_match",
  "query": "*:*",
  "tags": ["tag_1", "tag_2"],
  "threat_index": "mock-threat-list",
  "threat_query": "*:*",
  "threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ]
}
```

</details>

Structural elements that are new:

New type enum called "threat_match"
```ts
"type": "threat_match",
```

New `threat_index` string which can be set to a single threat index (This might change to an array in the near future before release):
```ts
"threat_index": "mock-threat-list"
```

New `threat_query` string which can be set any valid query to filter the threat list before executing the rule. This can be undefined, if you are only pushing in filters from the API.

```ts
"threat_query": "*:*",
```

New `threat_filters` array which can be set to any valid filter like `filters`. This can be `undefined` if you are only using the query from the API.
```ts
threat_filter": []
```

New `threat_mapping` array which can be set to a valid mapping between the threat list and the ECS list. This structure has an inner array called `entries` which represent a 2 level tree of 1st level OR elements followed by 2nd level AND elements.

For example, if you want to find all threat matches where ECS documents will match against some ${threatList} index where it would be like so:

<details>
 <summary>Click here to see array from the boolean</summary>

```ts
"threat_mapping": [
    {
      "entries": [
        {
          "field": "host.name",
          "type": "mapping",
          "value": "host.name"
        },
        {
          "field": "host.ip",
          "type": "mapping",
          "value": "host.ip"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "destination.ip",
          "type": "mapping",
          "value": "destination.ip"
        },
        {
          "field": "destination.port",
          "type": "mapping",
          "value": "destination.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.port",
          "type": "mapping",
          "value": "source.port"
        }
      ]
    },
    {
      "entries": [
        {
          "field": "source.ip",
          "type": "mapping",
          "value": "source.ip"
        }
      ]
    }
  ]
```

</details>

What that array represents in pseudo boolean logic is: 

<details>
 <summary>Click here to see pseduo logic</summary>

```ts
(host.name: ${threatList.host.name} AND host.ip: ${threatList.host.name}) OR
(destination.ip: ${threatList.destination.ip} AND destination.port: ${threatList.destination.port}) OR
(source.port ${threatList.source.port}) OR
(source.ip ${threatList.source.ip})
```

</details>

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

Author: FrankHassanabad

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -299,6 +299,7 @@ export const type = t.keyof({
   query: null,
   saved_query: null,
   threshold: null,
+  threat_match: null,
 });
 export type Type = t.TypeOf<typeof type>;
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts

@@ -48,3 +48,104 @@ export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSc
   exceptions_list: [],
   rule_id: 'rule-1',
 });
+
+export const getAddPrepackagedThreatMatchRulesSchemaMock = (): AddPrepackagedRulesSchema => ({
+  description: 'some description',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'threat_match',
+  risk_score: 55,
+  language: 'kuery',
+  rule_id: 'rule-1',
+  version: 1,
+  threat_query: '*:*',
+  threat_index: 'list-index',
+  threat_mapping: [
+    {
+      entries: [
+        {
+          field: 'host.name',
+          value: 'host.name',
+          type: 'mapping',
+        },
+      ],
+    },
+  ],
+  threat_filters: [
+    {
+      bool: {
+        must: [
+          {
+            query_string: {
+              query: 'host.name: linux',
+              analyze_wildcard: true,
+              time_zone: 'Zulu',
+            },
+          },
+        ],
+        filter: [],
+        should: [],
+        must_not: [],
+      },
+    },
+  ],
+});
+
+export const getAddPrepackagedThreatMatchRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({
+  author: [],
+  description: 'some description',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  severity_mapping: [],
+  type: 'threat_match',
+  risk_score: 55,
+  risk_score_mapping: [],
+  language: 'kuery',
+  references: [],
+  actions: [],
+  enabled: false,
+  false_positives: [],
+  from: 'now-6m',
+  interval: '5m',
+  max_signals: DEFAULT_MAX_SIGNALS,
+  tags: [],
+  to: 'now',
+  threat: [],
+  throttle: null,
+  version: 1,
+  exceptions_list: [],
+  rule_id: 'rule-1',
+  threat_query: '*:*',
+  threat_index: 'list-index',
+  threat_mapping: [
+    {
+      entries: [
+        {
+          field: 'host.name',
+          value: 'host.name',
+          type: 'mapping',
+        },
+      ],
+    },
+  ],
+  threat_filters: [
+    {
+      bool: {
+        must: [
+          {
+            query_string: {
+              query: 'host.name: linux',
+              analyze_wildcard: true,
+              time_zone: 'Zulu',
+            },
+          },
+        ],
+        filter: [],
+        should: [],
+        must_not: [],
+      },
+    },
+  ],
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -45,6 +45,12 @@ import {
   RiskScoreMapping,
   SeverityMapping,
 } from '../common/schemas';
+import {
+  threat_index,
+  threat_query,
+  threat_filters,
+  threat_mapping,
+} from '../types/threat_mapping';
 
 import {
   DefaultStringArray,
@@ -116,6 +122,10 @@ export const addPrepackagedRulesSchema = t.intersection([
       references: DefaultStringArray, // defaults to empty array of strings if not set during decode
       note, // defaults to "undefined" if not set during decode
       exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
+      threat_filters, // defaults to "undefined" if not set during decode
+      threat_mapping, // defaults to "undefined" if not set during decode
+      threat_query, // defaults to "undefined" if not set during decode
+      threat_index, // defaults to "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts

@@ -17,6 +17,8 @@ import { left } from 'fp-ts/lib/Either';
 import {
   getAddPrepackagedRulesSchemaMock,
   getAddPrepackagedRulesSchemaDecodedMock,
+  getAddPrepackagedThreatMatchRulesSchemaMock,
+  getAddPrepackagedThreatMatchRulesSchemaDecodedMock,
 } from './add_prepackaged_rules_schema.mock';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 import { getListArrayMock } from '../types/lists.mock';
@@ -1597,4 +1599,16 @@ describe('add prepackaged rules schema', () => {
       expect(message.schema).toEqual(expected);
     });
   });
+
+  describe('threat_mapping', () => {
+    test('You can set a threat query, index, mapping, filters on a pre-packaged rule', () => {
+      const payload = getAddPrepackagedThreatMatchRulesSchemaMock();
+      const decoded = addPrepackagedRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = getAddPrepackagedThreatMatchRulesSchemaDecodedMock();
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts

@@ -55,3 +55,103 @@ export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => (
   exceptions_list: [],
   rule_id: 'rule-1',
 });
+
+export const getCreateThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): CreateRulesSchema => ({
+  description: 'Detecting root and admin users',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'threat_match',
+  risk_score: 55,
+  language: 'kuery',
+  rule_id: ruleId,
+  threat_query: '*:*',
+  threat_index: 'list-index',
+  threat_mapping: [
+    {
+      entries: [
+        {
+          field: 'host.name',
+          value: 'host.name',
+          type: 'mapping',
+        },
+      ],
+    },
+  ],
+  threat_filters: [
+    {
+      bool: {
+        must: [
+          {
+            query_string: {
+              query: 'host.name: linux',
+              analyze_wildcard: true,
+              time_zone: 'Zulu',
+            },
+          },
+        ],
+        filter: [],
+        should: [],
+        must_not: [],
+      },
+    },
+  ],
+});
+
+export const getCreateThreatMatchRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({
+  author: [],
+  severity_mapping: [],
+  risk_score_mapping: [],
+  description: 'Detecting root and admin users',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'threat_match',
+  risk_score: 55,
+  language: 'kuery',
+  references: [],
+  actions: [],
+  enabled: true,
+  false_positives: [],
+  from: 'now-6m',
+  interval: '5m',
+  max_signals: DEFAULT_MAX_SIGNALS,
+  tags: [],
+  to: 'now',
+  threat: [],
+  throttle: null,
+  version: 1,
+  exceptions_list: [],
+  rule_id: 'rule-1',
+  threat_query: '*:*',
+  threat_index: 'list-index',
+  threat_mapping: [
+    {
+      entries: [
+        {
+          field: 'host.name',
+          value: 'host.name',
+          type: 'mapping',
+        },
+      ],
+    },
+  ],
+  threat_filters: [
+    {
+      bool: {
+        must: [
+          {
+            query_string: {
+              query: 'host.name: linux',
+              analyze_wildcard: true,
+              time_zone: 'Zulu',
+            },
+          },
+        ],
+        filter: [],
+        should: [],
+        must_not: [],
+      },
+    },
+  ],
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts

@@ -16,6 +16,8 @@ import { left } from 'fp-ts/lib/Either';
 import {
   getCreateRulesSchemaMock,
   getCreateRulesSchemaDecodedMock,
+  getCreateThreatMatchRulesSchemaMock,
+  getCreateThreatMatchRulesSchemaDecodedMock,
 } from './create_rules_schema.mock';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 import { getListArrayMock } from '../types/lists.mock';
@@ -1661,4 +1663,16 @@ describe('create rules schema', () => {
       expect(message.schema).toEqual(expected);
     });
   });
+
+  describe('threat_mapping', () => {
+    test('You can set a threat query, index, mapping, filters when creating a rule', () => {
+      const payload = getCreateThreatMatchRulesSchemaMock();
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = getCreateThreatMatchRulesSchemaDecodedMock();
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts

@@ -46,6 +46,12 @@ import {
   RiskScoreMapping,
   SeverityMapping,
 } from '../common/schemas';
+import {
+  threat_index,
+  threat_query,
+  threat_filters,
+  threat_mapping,
+} from '../types/threat_mapping';
 
 import {
   DefaultStringArray,
@@ -112,6 +118,10 @@ export const createRulesSchema = t.intersection([
       note, // defaults to "undefined" if not set during decode
       version: DefaultVersionNumber, // defaults to 1 if not set during decode
       exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
+      threat_mapping, // defaults to "undefined" if not set during decode
+      threat_query, // defaults to "undefined" if not set during decode
+      threat_filters, // defaults to "undefined" if not set during decode
+      threat_index, // defaults to "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts

@@ -4,7 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { getCreateRulesSchemaMock } from './create_rules_schema.mock';
+import {
+  getCreateRulesSchemaMock,
+  getCreateThreatMatchRulesSchemaMock,
+} from './create_rules_schema.mock';
 import { CreateRulesSchema } from './create_rules_schema';
 import { createRuleValidateTypeDependents } from './create_rules_type_dependents';
 
@@ -87,4 +90,39 @@ describe('create_rules_type_dependents', () => {
     const errors = createRuleValidateTypeDependents(schema);
     expect(errors).toEqual(['"threshold.value" has to be bigger than 0']);
   });
+
+  test('threat_index, threat_query, and threat_mapping are required when type is "threat_match" and validates with it', () => {
+    const schema: CreateRulesSchema = {
+      ...getCreateRulesSchemaMock(),
+      type: 'threat_match',
+    };
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual([
+      'when "type" is "threat_match", "threat_index" is required',
+      'when "type" is "threat_match", "threat_query" is required',
+      'when "type" is "threat_match", "threat_mapping" is required',
+    ]);
+  });
+
+  test('validates with threat_index, threat_query, and threat_mapping when type is "threat_match"', () => {
+    const schema = getCreateThreatMatchRulesSchemaMock();
+    const { threat_filters: threatFilters, ...noThreatFilters } = schema;
+    const errors = createRuleValidateTypeDependents(noThreatFilters);
+    expect(errors).toEqual([]);
+  });
+
+  test('does NOT validate when threat_mapping is an empty array', () => {
+    const schema: CreateRulesSchema = {
+      ...getCreateThreatMatchRulesSchemaMock(),
+      threat_mapping: [],
+    };
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual(['threat_mapping" must have at least one element']);
+  });
+
+  test('validates with threat_index, threat_query, threat_mapping, and an optional threat_filters, when type is "threat_match"', () => {
+    const schema = getCreateThreatMatchRulesSchemaMock();
+    const errors = createRuleValidateTypeDependents(schema);
+    expect(errors).toEqual([]);
+  });
 });