[Security Solutions][Detection Engine] Adds e2e FTR runtime support and 213 tests for exception lists

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.mock.ts

@@ -46,3 +46,13 @@ export const getCreateExceptionListMinimalSchemaMockWithoutId = (): CreateExcept
   name: NAME,
   type: ENDPOINT_TYPE,
 });
+
+/**
+ * Useful for end to end testing with detections
+ */
+export const getCreateExceptionListDetectionSchemaMock = (): CreateExceptionListSchema => ({
+  description: DESCRIPTION,
+  list_id: LIST_ID,
+  name: NAME,
+  type: 'detection',
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts

@@ -40,9 +40,11 @@ export const getCreateSavedQueryRulesSchemaMock = (ruleId = 'rule-1'): SavedQuer
 });
 
 export const getCreateThreatMatchRulesSchemaMock = (
-  ruleId = 'rule-1'
+  ruleId = 'rule-1',
+  enabled = false
 ): ThreatMatchCreateSchema => ({
   description: 'Detecting root and admin users',
+  enabled,
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',
   severity: 'high',

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -115,12 +115,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R
  * Useful for e2e backend tests where it doesn't have date time and other
  * server side properties attached to it.
  */
-export const getThreatMatchingSchemaPartialMock = (): Partial<RulesSchema> => {
+export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<RulesSchema> => {
   return {
     author: [],
     created_by: 'elastic',
     description: 'Detecting root and admin users',
-    enabled: true,
+    enabled,
     false_positives: [],
     from: 'now-6m',
     immutable: false,

x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts

@@ -5,6 +5,7 @@
  */
 
 import expect from '@kbn/expect';
+import { PrePackagedRulesAndTimelinesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/response';
 
 import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../plugins/security_solution/common/constants';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
@@ -13,6 +14,7 @@ import {
   deleteAllAlerts,
   deleteAllTimelines,
   deleteSignalsIndex,
+  installPrePackagedRules,
   waitFor,
 } from '../../utils';
 
@@ -45,71 +47,36 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
         await deleteAllTimelines(es);
       });
 
-      it('should contain rules_installed, rules_updated, timelines_installed, and timelines_updated', async () => {
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(Object.keys(body)).to.eql([
+      it('should create the prepackaged rules and return a count greater than zero, rules_updated to be zero, and contain the correct keys', async () => {
+        let responseBody: unknown;
+        await waitFor(async () => {
+          const { body, status } = await supertest
+            .put(DETECTION_ENGINE_PREPACKAGED_URL)
+            .set('kbn-xsrf', 'true')
+            .send();
+          if (status === 200) {
+            responseBody = body;
+          }
+          return status === 200;
+        }, DETECTION_ENGINE_PREPACKAGED_URL);
+
+        const prepackagedRules = responseBody as PrePackagedRulesAndTimelinesSchema;
+        expect(prepackagedRules.rules_installed).to.be.greaterThan(0);
+        expect(prepackagedRules.rules_updated).to.eql(0);
+        expect(Object.keys(prepackagedRules)).to.eql([
           'rules_installed',
           'rules_updated',
           'timelines_installed',
           'timelines_updated',
         ]);
       });
 
-      it('should create the prepackaged rules and return a count greater than zero', async () => {
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.rules_installed).to.be.greaterThan(0);
-      });
-
-      it('should create the prepackaged timelines and return a count greater than zero', async () => {
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.timelines_installed).to.be.greaterThan(0);
-      });
-
-      it('should create the prepackaged rules that the rules_updated is of size zero', async () => {
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.rules_updated).to.eql(0);
-      });
-
-      it('should create the prepackaged timelines and the timelines_updated is of size zero', async () => {
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.timelines_updated).to.eql(0);
-      });
-
-      it('should be possible to call the API twice and the second time the number of rules installed should be zero', async () => {
-        await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
+      it('should be possible to call the API twice and the second time the number of rules installed should be zero as well as timeline', async () => {
+        await installPrePackagedRules(supertest);
 
         // NOTE: I call the GET call until eventually it becomes consistent and that the number of rules to install are zero.
         // This is to reduce flakiness where it can for a short period of time try to install the same rule twice.
@@ -119,39 +86,23 @@ export default ({ getService }: FtrProviderContext): void => {
             .set('kbn-xsrf', 'true')
             .expect(200);
           return body.rules_not_installed === 0;
-        });
-
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.rules_installed).to.eql(0);
-      });
-
-      it('should be possible to call the API twice and the second time the number of timelines installed should be zero', async () => {
-        await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
+        }, `${DETECTION_ENGINE_PREPACKAGED_URL}/_status`);
 
+        let responseBody: unknown;
         await waitFor(async () => {
-          const { body } = await supertest
-            .get(`${DETECTION_ENGINE_PREPACKAGED_URL}/_status`)
+          const { body, status } = await supertest
+            .put(DETECTION_ENGINE_PREPACKAGED_URL)
             .set('kbn-xsrf', 'true')
-            .expect(200);
-          return body.timelines_not_installed === 0;
-        });
-
-        const { body } = await supertest
-          .put(DETECTION_ENGINE_PREPACKAGED_URL)
-          .set('kbn-xsrf', 'true')
-          .send()
-          .expect(200);
-
-        expect(body.timelines_installed).to.eql(0);
+            .send();
+          if (status === 200) {
+            responseBody = body;
+          }
+          return status === 200;
+        }, DETECTION_ENGINE_PREPACKAGED_URL);
+
+        const prepackagedRules = responseBody as PrePackagedRulesAndTimelinesSchema;
+        expect(prepackagedRules.rules_installed).to.eql(0);
+        expect(prepackagedRules.timelines_installed).to.eql(0);
       });
     });
   });

x-pack/test/detection_engine_api_integration/basic/tests/create_rules.ts

@@ -25,7 +25,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('create_rules', () => {
     describe('validation errors', () => {
@@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should create a single rule with a rule_id', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/create_rules_bulk.ts

@@ -23,7 +23,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('create_rules_bulk', () => {
     describe('validation errors', () => {
@@ -54,7 +53,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should create a single rule with a rule_id', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/delete_rules.ts

@@ -24,7 +24,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('delete_rules', () => {
     describe('deleting rules', () => {
@@ -34,7 +33,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should delete a single rule with a rule_id', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/delete_rules_bulk.ts

@@ -24,7 +24,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('delete_rules_bulk', () => {
     describe('deleting rules bulk using DELETE', () => {
@@ -34,7 +33,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should delete a single rule with a rule_id', async () => {
@@ -146,7 +145,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should delete a single rule with a rule_id', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts

@@ -22,7 +22,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('export_rules', () => {
     describe('exporting rules', () => {
@@ -32,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
       });
 
       it('should set the response content types to be expected', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/find_rules.ts

@@ -23,7 +23,6 @@ import {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('find_rules', () => {
     beforeEach(async () => {
@@ -32,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
-      await deleteAllAlerts(es);
+      await deleteAllAlerts(supertest);
     });
 
     it('should return an empty find body correctly if no rules are loaded', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/find_statuses.ts

@@ -30,7 +30,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
-      await deleteAllAlerts(es);
+      await deleteAllAlerts(supertest);
       await deleteAllRulesStatuses(es);
     });
 
@@ -45,7 +45,7 @@ export default ({ getService }: FtrProviderContext): void => {
     });
 
     it('should return a single rule status when a single rule is loaded from a find status with defaults added', async () => {
-      const resBody = await createRule(supertest, getSimpleRule());
+      const resBody = await createRule(supertest, getSimpleRule('rule-1', true));
 
       await waitForRuleSuccess(supertest, resBody.id);
 

x-pack/test/detection_engine_api_integration/basic/tests/get_prepackaged_rules_status.ts

@@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext): void => {
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
-        await deleteAllAlerts(es);
+        await deleteAllAlerts(supertest);
         await deleteAllTimelines(es);
       });