8.18 mirror of [Security Solution][Detection Engine] fixes siem-signal update when it was reindexed from v7 to v8 #206119

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts

@@ -102,9 +102,20 @@ export const createDetectionIndex = async (
   const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`);
 
   if (await templateNeedsUpdate({ alias: index, esClient })) {
+    const reIndexedIndexPatterns = await getReIndexedV8IndexPatterns({ index, esClient });
+    const template = getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record<
+      string,
+      unknown
+    >;
+
+    // addresses https://github.com/elastic/security-team/issues/11440
+    if (reIndexedIndexPatterns.length > 0 && Array.isArray(template.index_patterns)) {
+      template.index_patterns.push(...reIndexedIndexPatterns);
+    }
+
     await esClient.indices.putIndexTemplate({
       name: index,
-      body: getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record<string, unknown>,
+      body: template,
     });
   }
   // Check if the old legacy siem signals template exists and remove it
@@ -209,3 +220,25 @@ const addIndexAliases = async ({
   };
   await esClient.indices.updateAliases({ body: aliasActions });
 };
+
+/**
+ * checks if indices under alias were reIndexed from v7 to v8(prefixed with '.reindexed-v8-')
+ * returns wildcard index patterns to include these indices and possible rollovers in index template
+ */
+const getReIndexedV8IndexPatterns = async ({
+  esClient,
+  index,
+}: {
+  esClient: ElasticsearchClient;
+  index: string;
+}): Promise<string[]> => {
+  const V8_PREFIX = '.reindexed-v8-';
+  const indices = await esClient.indices.getAlias({ index: `${index}-*`, name: index });
+  return Object.keys(indices).reduce<string[]>((acc, concreteIndexName) => {
+    if (concreteIndexName.startsWith(V8_PREFIX)) {
+      acc.push(`${V8_PREFIX}${index.replace(/^\./, '')}-*`);
+    }
+
+    return acc;
+  }, []);
+};

x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json

@@ -0,0 +1,12 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": ".reindexed-v8-siem-signals-default-000001",
+    "source": {
+      "@timestamp": "2020-10-10T00:00:00.000Z",
+      "signal": {}
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json

@@ -0,0 +1,31 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+      ".siem-signals-default": {
+        "is_write_index": true
+      },
+      ".siem-signals-default-000001": {}
+    },
+    "index": ".reindexed-v8-siem-signals-default-000001",
+    "mappings": {
+      "_meta": {
+        "version": 1
+      },
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "signal": { "type": "object" }
+      }
+    },
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": ".siem-signals-default",
+          "rollover_alias": ".siem-signals-default"
+        }
+      }
+    }
+  }
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts

@@ -82,6 +82,43 @@ export default ({ getService }: FtrProviderContext) => {
           );
         });
       });
+
+      describe('with reIndexed from 7.xto 8.x .siem-signals index', () => {
+        beforeEach(async () => {
+          await esArchiver.load(
+            'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals'
+          );
+        });
+
+        afterEach(async () => {
+          await esArchiver.unload(
+            'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals'
+          );
+          await es.indices.delete({
+            index: '.reindexed-v8-siem-signals-default-000002',
+            ignore_unavailable: true,
+          });
+        });
+
+        it('should report that alerts index is outdated', async () => {
+          const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200);
+          expect(body).to.eql({
+            index_mapping_outdated: true,
+            name: `${DEFAULT_ALERTS_INDEX}-default`,
+          });
+        });
+
+        it('should update index mappings', async () => {
+          await supertest
+            .post(DETECTION_ENGINE_INDEX_URL)
+            .set('kbn-xsrf', 'true')
+            .send()
+            .expect({ acknowledged: true });
+
+          const { body: indexStatusBody } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send();
+          expect(indexStatusBody.index_mapping_outdated).to.be(false);
+        });
+      });
     });
   });
 };