From 8a3ca78e751888501326f62faad6267ea76f4945 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:54:47 +0000 Subject: [PATCH 1/4] [Security Solution][Detection Engine] fixes siem-signal update when it was reindexed from v7 to v8 --- .../routes/index/create_index_route.ts | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index 85fe5a2c29a1b..fc0414b47ae4e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -102,9 +102,20 @@ export const createDetectionIndex = async ( const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`); if (await templateNeedsUpdate({ alias: index, esClient })) { + const reIndexedIndices = await getMigratedToV8Indices({ index, esClient }); + const template = getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record< + string, + unknown + >; + + // addresses https://github.com/elastic/security-team/issues/11440 + if (reIndexedIndices.length > 0 && Array.isArray(template.index_patterns)) { + template.index_patterns.push(...reIndexedIndices); + } + await esClient.indices.putIndexTemplate({ name: index, - body: getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record, + body: template, }); } // Check if the old legacy siem signals template exists and remove it @@ -209,3 +220,25 @@ const addIndexAliases = async ({ }; await esClient.indices.updateAliases({ body: aliasActions }); }; + +/** + * checks if indices under alias were migrated from v7 to v8( prefixed with '.reindexed-v8-') + * returns wildcard index patterns to cover these indices and possible rollovers + */ +const getMigratedToV8Indices = async ({ + esClient, + index, +}: { + esClient: ElasticsearchClient; + index: string; +}): Promise => { + const V8_PREFIX = '.reindexed-v8-'; + const indices = await esClient.indices.getAlias({ index: `${index}-*`, name: index }); + return Object.keys(indices).reduce((acc, concreteIndexName) => { + if (concreteIndexName.startsWith(V8_PREFIX)) { + acc.push(`${V8_PREFIX}${index.replace(/^\./, '')}-*`); + } + + return acc; + }, []); +}; From 41a4fc5cbd359e5f7a43286d5ade5e61fb29b00d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:47:37 +0000 Subject: [PATCH 2/4] add FTR tests --- .../reindexed-v8-siem-signals/data.json | 12 ++++++ .../reindexed-v8-siem-signals/mappings.json | 31 ++++++++++++++++ .../ess_specific_index_logic/create_index.ts | 37 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json create mode 100644 x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json diff --git a/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json b/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json new file mode 100644 index 0000000000000..55b98b0d4b59b --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": ".reindexed-v8-siem-signals-default-000001", + "source": { + "@timestamp": "2020-10-10T00:00:00.000Z", + "signal": {} + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json b/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json new file mode 100644 index 0000000000000..43cd74e0838d0 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json @@ -0,0 +1,31 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": true + }, + ".siem-signals-default-000001": {} + }, + "index": ".reindexed-v8-siem-signals-default-000001", + "mappings": { + "_meta": { + "version": 1 + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": ".siem-signals-default", + "rollover_alias": ".siem-signals-default" + } + } + } + } +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts index 8ed33f1b763f1..cd2499173ea06 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts @@ -82,6 +82,43 @@ export default ({ getService }: FtrProviderContext) => { ); }); }); + + describe('with reIndexed from 7.xto 8.x .siem-signals index', () => { + beforeEach(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals' + ); + }); + + afterEach(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals' + ); + await es.indices.delete({ + index: '.reindexed-v8-siem-signals-default-000002', + ignore_unavailable: true, + }); + }); + + it('should report that alerts index is outdated', async () => { + const { body } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send().expect(200); + expect(body).to.eql({ + index_mapping_outdated: true, + name: `${DEFAULT_ALERTS_INDEX}-default`, + }); + }); + + it('should update index mappings', async () => { + await supertest + .post(DETECTION_ENGINE_INDEX_URL) + .set('kbn-xsrf', 'true') + .send() + .expect({ acknowledged: true }); + + const { body: indexStatusBody } = await supertest.get(DETECTION_ENGINE_INDEX_URL).send(); + expect(indexStatusBody.index_mapping_outdated).to.be(false); + }); + }); }); }); }; From bc8a4f65730453a2d580b60d861a17d75a684693 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:02:26 +0000 Subject: [PATCH 3/4] wording & refactoring --- .../routes/index/create_index_route.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index fc0414b47ae4e..5257486dcb53a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -102,15 +102,15 @@ export const createDetectionIndex = async ( const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`); if (await templateNeedsUpdate({ alias: index, esClient })) { - const reIndexedIndices = await getMigratedToV8Indices({ index, esClient }); + const reIndexedIndexPatterns = await getReIndexedV8IndexPatterns({ index, esClient }); const template = getSignalsTemplate(index, aadIndexAliasName, spaceId) as Record< string, unknown >; // addresses https://github.com/elastic/security-team/issues/11440 - if (reIndexedIndices.length > 0 && Array.isArray(template.index_patterns)) { - template.index_patterns.push(...reIndexedIndices); + if (reIndexedIndexPatterns.length > 0 && Array.isArray(template.index_patterns)) { + template.index_patterns.push(...reIndexedIndexPatterns); } await esClient.indices.putIndexTemplate({ @@ -222,10 +222,10 @@ const addIndexAliases = async ({ }; /** - * checks if indices under alias were migrated from v7 to v8( prefixed with '.reindexed-v8-') - * returns wildcard index patterns to cover these indices and possible rollovers + * checks if indices under alias were reIndexed from v7 to v8(prefixed with '.reindexed-v8-') + * returns wildcard index patterns to include these indices and possible rollovers in index template */ -const getMigratedToV8Indices = async ({ +const getReIndexedV8IndexPatterns = async ({ esClient, index, }: { From 4a3d287fa22a3ed993567a9ea94112a09df2f2cc Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:46:24 +0000 Subject: [PATCH 4/4] fixes CI checks --- .../data.json | 0 .../mappings.json | 0 .../ess_specific_index_logic/create_index.ts | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename x-pack/test/functional/es_archives/signals/{reindexed-v8-siem-signals => reindexed_v8_siem_signals}/data.json (100%) rename x-pack/test/functional/es_archives/signals/{reindexed-v8-siem-signals => reindexed_v8_siem_signals}/mappings.json (100%) diff --git a/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json similarity index 100% rename from x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/data.json rename to x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/data.json diff --git a/x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json b/x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json similarity index 100% rename from x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals/mappings.json rename to x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals/mappings.json diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts index cd2499173ea06..aa8abd2f33e50 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/create_index.ts @@ -86,13 +86,13 @@ export default ({ getService }: FtrProviderContext) => { describe('with reIndexed from 7.xto 8.x .siem-signals index', () => { beforeEach(async () => { await esArchiver.load( - 'x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals' + 'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals' ); }); afterEach(async () => { await esArchiver.unload( - 'x-pack/test/functional/es_archives/signals/reindexed-v8-siem-signals' + 'x-pack/test/functional/es_archives/signals/reindexed_v8_siem_signals' ); await es.indices.delete({ index: '.reindexed-v8-siem-signals-default-000002',