[SIEM][Detection Engine] Adds backend e2e tests for basic license testing

## Summary

Adds backend e2e basic license type tests for the detection engine API. Previously we only had tests for full security space that was platinum based licensing. These tests now cover the basic license test cases. This covers test cases for machine learning to ensure you cannot create machine learning based detection rules under the basic license. Instead those tests will return the expected 403 forbidden.

Testing just the subset of the tests from this PR locally:

```sh
node scripts/functional_tests --config x-pack/test/detection_engine_api_integration/basic/config.ts
```

You do want to go to the jenkins tests on CI and ensure you can see some of the strings such as the newer forbidden messages showing up as passing.

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

Author: FrankHassanabad

x-pack/scripts/functional_tests.js

@@ -21,6 +21,7 @@ const onlyNotInCoverageTests = [
   require.resolve('../test/alerting_api_integration/spaces_only/config.ts'),
   require.resolve('../test/alerting_api_integration/security_and_spaces/config.ts'),
   require.resolve('../test/detection_engine_api_integration/security_and_spaces/config.ts'),
+  require.resolve('../test/detection_engine_api_integration/basic/config.ts'),
   require.resolve('../test/plugin_api_integration/config.ts'),
   require.resolve('../test/kerberos_api_integration/config.ts'),
   require.resolve('../test/kerberos_api_integration/anonymous_access.config.ts'),

x-pack/test/detection_engine_api_integration/basic/config.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { createTestConfig } from '../common/config';
+
+// eslint-disable-next-line import/no-default-export
+export default createTestConfig('basic', {
+  disabledPlugins: [],
+  license: 'basic',
+  ssl: true,
+});

x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../plugins/siem/common/constants';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { createSignalsIndex, deleteAllAlerts, deleteSignalsIndex } from '../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext): void => {
+  const supertest = getService('supertest');
+  const es = getService('legacyEs');
+
+  describe('add_prepackaged_rules', () => {
+    describe('validation errors', () => {
+      it('should give an error that the index must exist first if it does not exist before adding prepackaged rules', async () => {
+        const { body } = await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(400);
+
+        expect(body).to.eql({
+          message:
+            'Pre-packaged rules cannot be installed until the signals index is created: .siem-signals-default',
+          status_code: 400,
+        });
+      });
+    });
+
+    describe('creating prepackaged rules', () => {
+      beforeEach(async () => {
+        await createSignalsIndex(supertest);
+      });
+
+      afterEach(async () => {
+        await deleteSignalsIndex(supertest);
+        await deleteAllAlerts(es);
+      });
+
+      it('should contain two output keys of rules_installed and rules_updated', async () => {
+        const { body } = await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(200);
+
+        expect(Object.keys(body)).to.eql(['rules_installed', 'rules_updated']);
+      });
+
+      it('should create the prepackaged rules and return a count greater than zero', async () => {
+        const { body } = await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(200);
+
+        expect(body.rules_installed).to.be.greaterThan(0);
+      });
+
+      it('should create the prepackaged rules that the rules_updated is of size zero', async () => {
+        const { body } = await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(200);
+
+        expect(body.rules_updated).to.eql(0);
+      });
+
+      it('should be possible to call the API twice and the second time the number of rules installed should be zero', async () => {
+        await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(200);
+
+        const { body } = await supertest
+          .put(DETECTION_ENGINE_PREPACKAGED_URL)
+          .set('kbn-xsrf', 'true')
+          .send()
+          .expect(200);
+
+        expect(body.rules_installed).to.eql(0);
+      });
+    });
+  });
+};

x-pack/test/detection_engine_api_integration/basic/tests/create_rules.ts

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import { DETECTION_ENGINE_RULES_URL } from '../../../../plugins/siem/common/constants';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import {
+  createSignalsIndex,
+  deleteAllAlerts,
+  deleteSignalsIndex,
+  getSimpleRule,
+  getSimpleRuleOutput,
+  getSimpleRuleOutputWithoutRuleId,
+  getSimpleRuleWithoutRuleId,
+  removeServerGeneratedProperties,
+  removeServerGeneratedPropertiesIncludingRuleId,
+  getSimpleMlRule,
+} from '../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const es = getService('legacyEs');
+
+  describe('create_rules', () => {
+    describe('validation errors', () => {
+      it('should give an error that the index must exist first if it does not exist before creating a rule', async () => {
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleRule())
+          .expect(400);
+
+        expect(body).to.eql({
+          message:
+            'To create a rule, the index must exist first. Index .siem-signals-default does not exist',
+          status_code: 400,
+        });
+      });
+    });
+
+    describe('creating rules', () => {
+      beforeEach(async () => {
+        await createSignalsIndex(supertest);
+      });
+
+      afterEach(async () => {
+        await deleteSignalsIndex(supertest);
+        await deleteAllAlerts(es);
+      });
+
+      it('should create a single rule with a rule_id', async () => {
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleRule())
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedProperties(body);
+        expect(bodyToCompare).to.eql(getSimpleRuleOutput());
+      });
+
+      it('should create a single rule without an input index', async () => {
+        const { index, ...payload } = getSimpleRule();
+        const { index: _index, ...expected } = getSimpleRuleOutput();
+
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(payload)
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedProperties(body);
+        expect(bodyToCompare).to.eql(expected);
+      });
+
+      it('should create a single rule without a rule_id', async () => {
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleRuleWithoutRuleId())
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedPropertiesIncludingRuleId(body);
+        expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId());
+      });
+
+      it('should give a 403 when trying to create a single Machine Learning rule since the license is basic', async () => {
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleMlRule())
+          .expect(403);
+
+        const bodyToCompare = removeServerGeneratedProperties(body);
+        expect(bodyToCompare).to.eql({
+          message: 'Your license does not support machine learning. Please upgrade your license.',
+          status_code: 403,
+        });
+      });
+
+      it('should cause a 409 conflict if we attempt to create the same rule_id twice', async () => {
+        await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleRule())
+          .expect(200);
+
+        const { body } = await supertest
+          .post(DETECTION_ENGINE_RULES_URL)
+          .set('kbn-xsrf', 'true')
+          .send(getSimpleRule())
+          .expect(409);
+
+        expect(body).to.eql({
+          message: 'rule_id: "rule-1" already exists',
+          status_code: 409,
+        });
+      });
+    });
+  });
+};

x-pack/test/detection_engine_api_integration/basic/tests/create_rules_bulk.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import { DETECTION_ENGINE_RULES_URL } from '../../../../plugins/siem/common/constants';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import {
+  createSignalsIndex,
+  deleteAllAlerts,
+  deleteSignalsIndex,
+  getSimpleRule,
+  getSimpleRuleOutput,
+  getSimpleRuleOutputWithoutRuleId,
+  getSimpleRuleWithoutRuleId,
+  removeServerGeneratedProperties,
+  removeServerGeneratedPropertiesIncludingRuleId,
+} from '../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext): void => {
+  const supertest = getService('supertest');
+  const es = getService('legacyEs');
+
+  describe('create_rules_bulk', () => {
+    describe('validation errors', () => {
+      it('should give a 200 even if the index does not exist as all bulks return a 200 but have an error of 409 bad request in the body', async () => {
+        const { body } = await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'true')
+          .send([getSimpleRule()])
+          .expect(200);
+
+        expect(body).to.eql([
+          {
+            error: {
+              message:
+                'To create a rule, the index must exist first. Index .siem-signals-default does not exist',
+              status_code: 400,
+            },
+            rule_id: 'rule-1',
+          },
+        ]);
+      });
+    });
+
+    describe('creating rules in bulk', () => {
+      beforeEach(async () => {
+        await createSignalsIndex(supertest);
+      });
+
+      afterEach(async () => {
+        await deleteSignalsIndex(supertest);
+        await deleteAllAlerts(es);
+      });
+
+      it('should create a single rule with a rule_id', async () => {
+        const { body } = await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'true')
+          .send([getSimpleRule()])
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedProperties(body[0]);
+        expect(bodyToCompare).to.eql(getSimpleRuleOutput());
+      });
+
+      it('should create a single rule without a rule_id', async () => {
+        const { body } = await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'true')
+          .send([getSimpleRuleWithoutRuleId()])
+          .expect(200);
+
+        const bodyToCompare = removeServerGeneratedPropertiesIncludingRuleId(body[0]);
+        expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId());
+      });
+
+      it('should return a 200 ok but have a 409 conflict if we attempt to create the same rule_id twice', async () => {
+        const { body } = await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'true')
+          .send([getSimpleRule(), getSimpleRule()])
+          .expect(200);
+
+        expect(body).to.eql([
+          {
+            error: {
+              message: 'rule_id: "rule-1" already exists',
+              status_code: 409,
+            },
+            rule_id: 'rule-1',
+          },
+        ]);
+      });
+
+      it('should return a 200 ok but have a 409 conflict if we attempt to create the same rule_id that already exists', async () => {
+        await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'true')
+          .send([getSimpleRule()])
+          .expect(200);
+
+        const { body } = await supertest
+          .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_create`)
+          .set('kbn-xsrf', 'foo')
+          .send([getSimpleRule()])
+          .expect(200);
+
+        expect(body).to.eql([
+          {
+            error: {
+              message: 'rule_id: "rule-1" already exists',
+              status_code: 409,
+            },
+            rule_id: 'rule-1',
+          },
+        ]);
+      });
+    });
+  });
+};