From f676682f23213ae75b015872a031dfd9af3855fa Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:28:35 +0000 Subject: [PATCH] [Security Solution][Detection Engine] add deprecation warning for non-migrated signals (#204247) ## Summary - addresses partly https://github.com/elastic/security-team/issues/10878 - shows deprecation warning if siem index was not migrated ### How to test #### How to create legacy siem index? run script that used for FTR tests ```bash node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space ``` These would create legacy siem indices. But be aware, it might break Kibana .alerts indices creation. But sufficient for testing Visit also detection rules page, to ensure alerts index created. Otherwise, https://www.elastic.co/guide/en/security/current/signals-migration-api.html#migration-1 API might not show these indices outdated #### How to test deprecated feature? 1. Observe warning feature deprecation on Kibana Upgrade page, if you set up legacy siem signals
Kibana Upgrade feature deprecation flyout Screenshot 2024-12-17 at 16 59 04
#### Test outdated indices created in 7.x 1. Create cloud env of 7.x version 2. Create rule, generate alerts for .siem-signals 3. Create cloud env of 8.18 from existing 7.x snapshot (from previous steps) 4. Connect local Kibana to 8.18 from mirror branch of this one(https://github.com/elastic/kibana/pull/204621) 5. Add to Kibana dev config following options to enable Upgrade assistant(UA) showing outdated indices ```yml xpack.upgrade_assistant.featureSet: mlSnapshots: true migrateDataStreams: true migrateSystemIndices: true reindexCorrectiveActions: true ``` 6. Go to Detection rules page, ensure rule is running and new .alerts index has been created (visiting rules table page should be enough) 7. Open UA, ensure Kibana deprecations show signals are not migrated 8. Open UA, check Elasticsearch deprecations 9. Find outdated siem-signals index 10. Migrate it 11. Check Kibana deprecations still signals are not migrated 12. Migrate signals using https://www.elastic.co/guide/en/security/current/signals-migration-api.html API 13. Ensure Kibana deprecations does not show that space as not migrated Demo video of migration .siem-signal from another-3 Kibana space https://github.com/user-attachments/assets/d2729482-d2c8-4a23-a780-ad19d4f52c73 (cherry picked from commit 9cccd303ef91686fc6097c24ba697bf3d8875e01) --- .../server/deprecations/index.ts | 26 +++++ .../server/deprecations/signals_migration.ts | 85 +++++++++++++++++ .../migrations/create_migration_index.ts | 5 + .../get_non_migrated_signals_info.test.ts | 84 ++++++++++++++++- .../get_non_migrated_signals_info.ts | 43 +++++++++ .../security_solution/server/plugin.ts | 3 + .../data.json | 12 +++ .../mappings.json | 29 ++++++ .../migrations/deprecations.ts | 94 +++++++++++++++++++ .../migrations/index.ts | 1 + 10 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/deprecations/index.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/deprecations/signals_migration.ts create mode 100644 x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/data.json create mode 100644 x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/mappings.json create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/deprecations.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/deprecations/index.ts b/x-pack/solutions/security/plugins/security_solution/server/deprecations/index.ts new file mode 100644 index 0000000000000..c84748c86dc2a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/deprecations/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { CoreSetup, Logger } from '@kbn/core/server'; +import type { ConfigType } from '../config'; + +import { getSignalsMigrationDeprecationsInfo } from './signals_migration'; + +export const registerDeprecations = ({ + core, + config, + logger, +}: { + core: CoreSetup; + config: ConfigType; + logger: Logger; +}) => { + core.deprecations.registerDeprecations({ + getDeprecations: async (ctx) => { + return [...(await getSignalsMigrationDeprecationsInfo(ctx, config, logger, core.docLinks))]; + }, + }); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/deprecations/signals_migration.ts b/x-pack/solutions/security/plugins/security_solution/server/deprecations/signals_migration.ts new file mode 100644 index 0000000000000..7af8a05cc2218 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/deprecations/signals_migration.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + DeprecationsDetails, + GetDeprecationsContext, + Logger, + DocLinksServiceSetup, +} from '@kbn/core/server'; + +import { i18n } from '@kbn/i18n'; +import { DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL } from '../../common/constants'; +import type { ConfigType } from '../config'; + +import { getNonMigratedSignalsInfo } from '../lib/detection_engine/migrations/get_non_migrated_signals_info'; + +const constructMigrationApiCall = (space: string, range: string) => + `GET :${ + space === 'default' ? '' : `/s/${space}` + }${DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL}?from=${range}`; + +export const getSignalsMigrationDeprecationsInfo = async ( + ctx: GetDeprecationsContext, + config: ConfigType, + logger: Logger, + docLinks: DocLinksServiceSetup +): Promise => { + const esClient = ctx.esClient.asInternalUser; + const { isMigrationRequired, spaces } = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: config.signalsIndex, + logger, + }); + // Deprecation API requires time range to be part of request (https://www.elastic.co/guide/en/security/current/signals-migration-api.html#migration-1) + // Return the earliest date, so it would capture the oldest possible signals + const fromRange = new Date(0).toISOString(); + + if (isMigrationRequired) { + return [ + { + deprecationType: 'feature', + title: i18n.translate('xpack.securitySolution.deprecations.signalsMigrationTitle', { + defaultMessage: 'Found not migrated detection alerts', + }), + level: 'warning', + message: i18n.translate('xpack.securitySolution.deprecations.signalsMigrationMessage', { + defaultMessage: `After upgrading Kibana, the latest Elastic Security features will be available for any newly generated detection alerts. However, in order to enable new features for existing detection alerts, migration may be necessary.`, + }), + documentationUrl: docLinks.links.securitySolution.signalsMigrationApi, + correctiveActions: { + manualSteps: [ + i18n.translate( + 'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepOne', + { + defaultMessage: `Visit "Learn more" link for instructions how to migrate detection alerts. Migrate indices for each space.`, + } + ), + i18n.translate( + 'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepTwo', + { + defaultMessage: 'Spaces with at least one non-migrated signals index: {spaces}.', + values: { + spaces: spaces.join(', '), + }, + } + ), + i18n.translate( + 'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepFour', + { + defaultMessage: 'Example of migration API calls:', + } + ), + ...spaces.map((space) => constructMigrationApiCall(space, fromRange)), + ], + }, + }, + ]; + } + + return []; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts index 30e4eb0b4e276..c2a625abd8112 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts @@ -42,6 +42,11 @@ export const createMigrationIndex = async ({ }, }, }, + mappings: { + _meta: { + version, + }, + }, }, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts index 36252ab792342..8128af890bed3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts @@ -7,7 +7,10 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; -import { getNonMigratedSignalsInfo } from './get_non_migrated_signals_info'; +import { + getNonMigratedSignalsInfo, + checkIfMigratedIndexOutdated, +} from './get_non_migrated_signals_info'; import { getIndexVersionsByIndex } from './get_index_versions_by_index'; import { getSignalVersionsByIndex } from './get_signal_versions_by_index'; import { getLatestIndexTemplateVersion } from './get_latest_index_template_version'; @@ -132,6 +135,39 @@ describe('getNonMigratedSignalsInfo', () => { spaces: ['default'], }); }); + it('return empty result for migrated in v8 index', async () => { + getIndexAliasPerSpaceMock.mockReturnValue({ + '.reindexed-v8-siem-signals-another-1-000001': { + alias: '.siem-signals-another-1', + indexName: '.reindexed-v8-siem-signals-another-1-000001', + space: 'another-1-000001', + }, + '.siem-signals-another-1-000002': { + alias: '.siem-signals-another-1', + indexName: '.siem-signals-another-1-000002', + space: 'another-1', + }, + }); + + getIndexVersionsByIndexMock.mockReturnValue({ + '.reindexed-v8-siem-signals-another-1-000001': 57, + '.siem-signals-another-1-000002': TEMPLATE_VERSION, + '.reindexed-v8-siem-signals-another-1-000001-r000077': TEMPLATE_VERSION, // outdated .reindexed-v8-siem-signals-another-1-000001 is already migrated + }); + getSignalVersionsByIndexMock.mockReturnValue({}); + + const result = await getNonMigratedSignalsInfo({ + esClient, + signalsIndex: 'siem-signals', + logger, + }); + + expect(result).toEqual({ + indices: [], + isMigrationRequired: false, + spaces: [], + }); + }); it('returns results for outdated signals in index', async () => { getIndexVersionsByIndexMock.mockReturnValue({ '.siem-signals-another-1-legacy': TEMPLATE_VERSION, @@ -175,3 +211,49 @@ describe('getNonMigratedSignalsInfo', () => { }); }); }); + +describe('checkIfMigratedIndexOutdated', () => { + const indexVersionsByIndex = { + '.siem-signals-default-000001': 57, + '.siem-signals-another-6-000001': 57, + '.siem-signals-default-000002': 77, + '.siem-signals-another-5-000001': 57, + '.reindexed-v8-siem-signals-another-1-000001': 57, + '.siem-signals-another-7-000001': 57, + '.reindexed-v8-siem-signals-another-2-000001': 57, + '.siem-signals-another-3-000001': 57, + '.reindexed-v8-siem-signals-another-4-000001': 57, + '.siem-signals-another-3-000002': 77, + '.siem-signals-another-9-000001': 57, + '.siem-signals-another-8-000001': 57, + '.siem-signals-another-2-000002': 77, + '.siem-signals-another-10-000001': 57, + '.siem-signals-another-1-000002': 77, + '.siem-signals-another-2-000001-r000077': 77, + '.reindexed-v8-siem-signals-another-1-000001-r000077': 77, + }; + + const migratedIndices = [ + '.reindexed-v8-siem-signals-another-1-000001', + '.reindexed-v8-siem-signals-another-2-000001', + '.reindexed-v8-siem-signals-another-1-000001-r000077', + ]; + + migratedIndices.forEach((index) => { + it(`should correctly find index "${index}" is migrated`, () => { + expect(checkIfMigratedIndexOutdated(index, indexVersionsByIndex, TEMPLATE_VERSION)).toBe( + false + ); + }); + }); + + it('should find non migrated index', () => { + expect( + checkIfMigratedIndexOutdated( + '.reindexed-v8-siem-signals-another-4-000001', + indexVersionsByIndex, + TEMPLATE_VERSION + ) + ).toBe(true); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts index d1f561fb3846c..9fadf2a1ab337 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts @@ -17,6 +17,41 @@ import { isOutdated as getIsOutdated, signalsAreOutdated } from './helpers'; import { getLatestIndexTemplateVersion } from './get_latest_index_template_version'; import { getIndexAliasPerSpace } from './get_index_alias_per_space'; +const REINDEXED_PREFIX = '.reindexed-v8-'; + +export const checkIfMigratedIndexOutdated = ( + indexName: string, + indexVersionsByIndex: IndexVersionsByIndex, + latestTemplateVersion: number +) => { + const isIndexOutdated = getIsOutdated({ + current: indexVersionsByIndex[indexName] ?? 0, + target: latestTemplateVersion, + }); + + if (!isIndexOutdated) { + return false; + } + + const nameWithoutPrefix = indexName.replace(REINDEXED_PREFIX, '.'); + + const hasOutdatedMigratedIndices = Object.entries(indexVersionsByIndex).every( + ([index, version]) => { + if (index === indexName) { + return true; + } + + if (index.startsWith(nameWithoutPrefix) || index.startsWith(indexName)) { + return getIsOutdated({ current: version ?? 0, target: latestTemplateVersion }); + } + + return true; + } + ); + + return hasOutdatedMigratedIndices; +}; + interface OutdatedSpaces { isMigrationRequired: boolean; spaces: string[]; @@ -85,6 +120,14 @@ export const getNonMigratedSignalsInfo = async ({ const version = indexVersionsByIndex[indexName] ?? 0; const signalVersions = signalVersionsByIndex[indexName] ?? []; + // filter out migrated from 7.x to 8 indices + if ( + indexName.startsWith(REINDEXED_PREFIX) && + !checkIfMigratedIndexOutdated(indexName, indexVersionsByIndex, latestTemplateVersion) + ) { + return acc; + } + const isOutdated = getIsOutdated({ current: version, target: latestTemplateVersion }) || signalsAreOutdated({ signalVersions, target: latestTemplateVersion }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts index c2f0fddb6e64f..980458e1d9c58 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts @@ -44,6 +44,7 @@ import { AppClientFactory } from './client'; import type { ConfigType } from './config'; import { createConfig } from './config'; import { initUiSettings } from './ui_settings'; +import { registerDeprecations } from './deprecations'; import { APP_ID, APP_UI_ID, @@ -212,6 +213,8 @@ export class Plugin implements ISecuritySolutionPlugin { this.ruleMonitoringService.setup(core, plugins); + registerDeprecations({ core, config: this.config, logger: this.logger }); + if (experimentalFeatures.riskScoringPersistence) { registerRiskScoringTask({ getStartServices: core.getStartServices, diff --git a/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/data.json b/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/data.json new file mode 100644 index 0000000000000..2547b46171e44 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": ".siem-signals-another-space-legacy", + "source": { + "@timestamp": "2020-10-10T00:00:00.000Z", + "signal": {} + }, + "type": "_doc" + } + } \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/mappings.json b/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/mappings.json new file mode 100644 index 0000000000000..0dbbf1d4e833e --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/mappings.json @@ -0,0 +1,29 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-another-space": { + "is_write_index": false + } + }, + "index": ".siem-signals-another-space-legacy", + "mappings": { + "_meta": { + "version": 1 + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "lifecycle": { + "indexing_complete": true + } + } + } + } + } \ No newline at end of file diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/deprecations.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/deprecations.ts new file mode 100644 index 0000000000000..6ec6d3d8aaeb1 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/deprecations.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import type { DeprecationsDetails } from '@kbn/core/server'; + +import { + createAlertsIndex, + deleteAllAlerts, +} from '../../../../../../../../common/utils/security_solution'; + +import { FtrProviderContext } from '../../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const log = getService('log'); + const es = getService('es'); + + const getDeprecations = async (): Promise => { + const { body } = await supertest.get('/api/deprecations/').set('kbn-xsrf', 'true').expect(200); + return body.deprecations; + }; + + const getLegacyIndicesDeprecation = async (): Promise => { + const deprecations = await getDeprecations(); + + return deprecations.find(({ title }) => title === 'Found not migrated detection alerts'); + }; + + describe('@ess Alerts migration deprecations API', () => { + describe('no siem legacy indices exist', () => { + it('should return empty siem signals deprecation', async () => { + const deprecation = await getLegacyIndicesDeprecation(); + + expect(deprecation).toBeUndefined(); + }); + }); + + describe('siem legacy indices exist', () => { + beforeEach(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/signals/legacy_signals_index'); + await createAlertsIndex(supertest, log); + }); + + afterEach(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/signals/legacy_signals_index'); + await deleteAllAlerts(supertest, log, es); + }); + + it('should return legacy siem signals deprecation', async () => { + const deprecation = await getLegacyIndicesDeprecation(); + + expect(deprecation?.level).toBe('warning'); + + // ensures space included in manual steps + expect(deprecation?.correctiveActions.manualSteps[1]).toContain( + 'Spaces with at least one non-migrated signals index: default.' + ); + expect(deprecation?.correctiveActions.manualSteps[2]).toContain( + 'Example of migration API calls:' + ); + expect(deprecation?.correctiveActions.manualSteps[3]).toContain( + 'GET :/api/detection_engine/signals/migration_status?from=1970-01-01T00:00:00.000Z' + ); + }); + + describe('multiple spaces', () => { + beforeEach(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space' + ); + }); + + afterEach(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space' + ); + }); + + it('should return legacy siem signals deprecation with multiple spaces', async () => { + const deprecation = await getLegacyIndicesDeprecation(); + + expect(deprecation?.correctiveActions.manualSteps[1]).toContain('another-space'); + expect(deprecation?.correctiveActions.manualSteps[1]).toContain('default'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/index.ts index 2c1aed2b1387b..2266e653b2493 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/migrations/index.ts @@ -12,5 +12,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./delete_alerts_migrations')); loadTestFile(require.resolve('./finalize_alerts_migrations')); loadTestFile(require.resolve('./get_alerts_migration_status')); + loadTestFile(require.resolve('./deprecations')); }); }