diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index dbd095815091d..d6da32cee7884 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -197,6 +197,10 @@ export const RuleMigration = z * The rule migration id */ id: NonEmptyString, + /** + * The rule migration name + */ + name: NonEmptyString, }) .merge(RuleMigrationData); @@ -324,6 +328,10 @@ export const RuleMigrationTaskStats = z.object({ * The migration id */ id: NonEmptyString, + /** + * The migration name + */ + name: NonEmptyString, /** * Indicates if the migration task status. */ diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index eca97ca87d88c..f3d8c96790ea3 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -122,10 +122,14 @@ components: - type: object required: - id + - name properties: id: description: The rule migration id $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + name: + description: The rule migration name + $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' - $ref: '#/components/schemas/RuleMigrationData' RuleMigrationData: @@ -205,6 +209,7 @@ components: description: The rule migration task stats object. required: - id + - name - status - rules - created_at @@ -213,6 +218,9 @@ components: id: description: The migration id $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' + name: + description: The migration name + $ref: '../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString' status: description: Indicates if the migration task status. $ref: '#/components/schemas/RuleMigrationTaskStatus' diff --git a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts index 9beb322127b3a..7b275975695c7 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/siem_migrations/types.ts @@ -18,5 +18,4 @@ export interface RuleMigrationFilters { partiallyTranslated?: boolean; untranslatable?: boolean; searchTerm?: string; - name?: string; } diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx index 55e45caee5830..e8b37d861715a 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/components/migration_status_panels/migration_ready_panel.test.tsx @@ -40,6 +40,7 @@ const mockMigrationStateWithError = { last_error: 'Failed to populate ELSER indices. Make sure the ELSER model is deployed and running at Machine Learning > Trained Models. Error: Exception when running inference id [.elser-2-elasticsearch] on field [elser_embedding]', id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', + name: 'Migration 1', rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', @@ -49,6 +50,7 @@ const mockMigrationStateWithError = { const mockMigrationStatsAborted = { status: SiemMigrationTaskStatus.ABORTED, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', + name: 'Migration 1', rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', @@ -58,6 +60,7 @@ const mockMigrationStatsAborted = { const mockMigrationStatsReady = { status: SiemMigrationTaskStatus.READY, id: 'c44d2c7d-0de1-4231-8b82-0dcfd67a9fe3', + name: 'Migration 1', rules: { total: 6, pending: 6, processing: 0, completed: 0, failed: 0 }, created_at: '2025-05-27T12:12:17.563Z', last_updated_at: '2025-05-27T12:12:17.563Z', diff --git a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/types.ts b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/types.ts index 77dae3c7206be..30bb0caba809e 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/siem_migrations/rules/types.ts @@ -9,11 +9,7 @@ import type { SiemMigrationTaskStatus } from '../../../common/siem_migrations/co import type { RuleMigrationTaskStats } from '../../../common/siem_migrations/model/rule_migration.gen'; export interface RuleMigrationStats extends RuleMigrationTaskStats { - status: SiemMigrationTaskStatus; - /** The sequential number of the migration */ - number: number; - /** The name of the migration */ - name?: string; + status: SiemMigrationTaskStatus; // use the native enum instead of the zod enum from the model } export enum AuthorFilter { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index 9fcd6bb08e6b5..032b9d377ccde 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -39,7 +39,7 @@ export type AddRuleMigrationRulesInput = Omit< RuleMigrationRule, '@timestamp' | 'id' | 'status' | 'created_by' >; -export type RuleMigrationDataStats = Omit; +export type RuleMigrationDataStats = Omit; export type RuleMigrationAllDataStats = RuleMigrationDataStats[]; export interface RuleMigrationGetRulesOptions { @@ -451,9 +451,6 @@ export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient } else if (filters.untranslatable === false) { filter.push(searchConditions.isNotUntranslatable()); } - if (filters.name) { - filter.push({ term: { name: filters.name } }); - } return { bool: { filter } }; } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index e22bc73a351b6..f962effaa7ea3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -97,6 +97,7 @@ export const getPrebuiltRulesFieldMap: ({ export const migrationsFieldMaps: FieldMap< SchemaFieldMapKeys> > = { + name: { type: 'keyword', required: true }, created_at: { type: 'date', required: true }, created_by: { type: 'keyword', required: true }, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_index_migrator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_index_migrator.ts index dea1831b01c25..9b12ffbb52f71 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_index_migrator.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_index_migrator.ts @@ -33,10 +33,10 @@ export class RuleMigrationIndexMigrator { async run() { const allSpaces = await this.getSpaceListForMigrations(); if (allSpaces.length === 0) { - this.logger.info('No spaces or index found for index migration'); + this.logger.debug('No spaces or index found for index migration'); return; } - this.logger.info( + this.logger.debug( `Starting index migration for rule migrations for spaces :${allSpaces.join(', ')}` ); for (const spaceId of allSpaces) { @@ -48,6 +48,6 @@ export class RuleMigrationIndexMigrator { ); await migrator.run(); } - this.logger.info('Finished index migration for rule migrations successfully'); + this.logger.debug('Finished index migration for rule migrations successfully'); } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts index 7b725267cf658..3d63e5637ce3f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/index_migrators/rule_migrations_per_space_index_migrator.ts @@ -16,6 +16,8 @@ import type { Adapters, StoredSiemMigration } from '../types'; import { MAX_ES_SEARCH_SIZE } from '../constants'; export class RuleMigrationSpaceIndexMigrator { + private namesMap?: Map; + constructor( private spaceId: string, private esClient: ElasticsearchClient, @@ -23,6 +25,111 @@ export class RuleMigrationSpaceIndexMigrator { private ruleMigrationIndexAdapters: Adapters ) {} + /** + * Runs the migrators for the rule migration index in the specified space. + * It migrates existing rules to create migration documents and populates migration names. + * + * If any errors occur they are logged but should not prevent the server from starting. + */ + async run() { + this.logger.debug(`Starting migrators for space ${this.spaceId}`); + try { + await this.migrateRuleMigrationIndex(); + } catch (error) { + this.logger.error( + `Error migrating rule migration index for space ${this.spaceId}: ${error.message}` + ); + } + try { + await this.populateMigrationNames(); + } catch (error) { + this.logger.error( + `Error populating migration names for space ${this.spaceId}: ${error.message}` + ); + } + this.logger.debug(`Finished migrators for space ${this.spaceId}`); + } + + /** + * Migrates the rule migration index by creating migration documents for existing rules + * that do not have corresponding migration documents in the migrations index. + */ + private async migrateRuleMigrationIndex() { + const installedIndexName = + await this.ruleMigrationIndexAdapters.migrations.getInstalledIndexName(this.spaceId); + if (!installedIndexName) { + await this.ruleMigrationIndexAdapters.migrations.createIndex(this.spaceId); + } + + const existingMigrationsFromRulesIndex = await this.getExistingMigrationFromRulesIndex(); + const existingMigrationsFromMigrationsIndex = + await this.getExistingMigrationIdsFromMigrationsIndex(); + + const migrationsToIndex = existingMigrationsFromRulesIndex.filter( + (migration) => !existingMigrationsFromMigrationsIndex.some((id) => id === migration.id) + ); + + if (migrationsToIndex.length > 0) { + const getMigrationName = await this.createGetMigrationsName(); + await this.createMigrationDocs( + migrationsToIndex.map((migration) => ({ + ...migration, + created_by: migration.created_by ?? '', + created_at: migration.created_at ?? new Date().toISOString(), + name: getMigrationName(migration.id), + })) + ); + this.logger.debug(`Created ${migrationsToIndex.length} migration documents missing.`); + } + } + + /** + * Populates migration documents that do not have a name field with generated names. + * The names are generated based on the migration creation order, like the existing migrations are named in the runtime. + */ + private async populateMigrationNames() { + const migrationIdsWithoutName = await this.getMigrationIdsWithoutName(); + + if (migrationIdsWithoutName.length > 0) { + const getMigrationName = await this.createGetMigrationsName(); + + const migrationsToUpdate = migrationIdsWithoutName.map((id) => { + return { id, name: getMigrationName(id) }; + }); + + await this.updateMigrationDocs(migrationsToUpdate); + this.logger.debug(`Updated ${migrationsToUpdate.length} migrations with generated name.`); + } + } + + /** + * Creates migration documents in the migrations index. + */ + private async createMigrationDocs(docs: StoredSiemMigration[]) { + const _index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); + const operations = docs.flatMap(({ id: _id, ...doc }) => [ + { create: { _id, _index } }, + { ...doc }, + ]); + return this.esClient.bulk({ refresh: 'wait_for', operations }); + } + + /** + * Updates migration documents in the migrations index. + */ + private async updateMigrationDocs(docs: Array>) { + const _index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); + const operations = docs.flatMap(({ id: _id, ...doc }) => [ + { update: { _id, _index } }, + { doc }, + ]); + return this.esClient.bulk({ refresh: 'wait_for', operations }); + } + + /** + * Retrieves existing migrations from the rules index. + * It aggregates by migration_id and returns the earliest created_at and created_by for each migration. + */ private async getExistingMigrationFromRulesIndex() { const index = this.ruleMigrationIndexAdapters.rules.getIndexName(this.spaceId); const aggregations: Record = { @@ -34,12 +141,7 @@ export class RuleMigrationSpaceIndexMigrator { }, }, }; - const result = await this.esClient - .search({ index, aggregations, _source: false }) - .catch((error) => { - this.logger.error(`Error getting all rule migrations stats: ${error.message}`); - throw error; - }); + const result = await this.esClient.search({ index, aggregations, _source: false }); const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; @@ -54,128 +156,81 @@ export class RuleMigrationSpaceIndexMigrator { })); } - private async getExistingMigrationFromMigrationsIndex() { + /** + * Retrieves existing migrations from the migrations index. + * It returns the IDs of all migration documents. + */ + private async getExistingMigrationIdsFromMigrationsIndex(): Promise { const index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); const result = await this.esClient.search({ index, size: MAX_ES_SEARCH_SIZE, - query: { - match_all: {}, - }, - _source: true, + query: { match_all: {} }, + _source: false, }); - return result.hits.hits.map(({ _id, _source }) => { + return result.hits.hits.map(({ _id }) => { assert(_id, 'document should have _id'); - return { - id: _id, - ..._source, - }; + return _id; }); } - private async indexMigrationDocs(docs: StoredSiemMigration[]) { - const indexName = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); - const createOperations = docs.flatMap((doc) => [ - { - create: { - _id: doc.id, - _index: indexName, - }, - }, - { - ...doc, - }, - ]); + /** + * Retrieves migration IDs from the migrations index that do not have a name field. + */ + private async getMigrationIdsWithoutName(): Promise { + const index = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); - return this.esClient.bulk({ - refresh: 'wait_for', - operations: createOperations, + const result = await this.esClient.search({ + index, + query: { bool: { must_not: { exists: { field: 'name' } } } }, + size: MAX_ES_SEARCH_SIZE, + _source: false, }); - } - - private async updateMigrationDocs(docs: StoredSiemMigration[]) { - const indexName = this.ruleMigrationIndexAdapters.migrations.getIndexName(this.spaceId); - const updateOperations = docs.flatMap((doc) => [ - { - update: { - _id: doc.id, - _index: indexName, - }, - }, - { - doc: { - name: doc.name, - }, - }, - ]); - return this.esClient.bulk({ - refresh: 'wait_for', - operations: updateOperations, + return result.hits.hits.map(({ _id }) => { + assert(_id, 'document should have _id'); + return _id; }); } - async run() { - await this.migrateRuleMigrationIndex(); - } - /** - * Creates the rule migration index if it doesn't exist and indexes any missing migration documents - * from the rules index. - * + * Retrieves the names of all migrations from the migrations index. + * The names are generated based on the migration the creation order. */ - private async migrateRuleMigrationIndex() { - const installedIndexName = - await this.ruleMigrationIndexAdapters.migrations.getInstalledIndexName(this.spaceId); - if (!installedIndexName) { - await this.ruleMigrationIndexAdapters.migrations.createIndex(this.spaceId); - } - - const existingMigrationsFromRulesIndex = await this.getExistingMigrationFromRulesIndex(); - const existingMigrationsFromMigrationsIndex = - await this.getExistingMigrationFromMigrationsIndex(); - - const migrationsToIndex = existingMigrationsFromRulesIndex.filter( - (migration) => !existingMigrationsFromMigrationsIndex.some((m) => m.id === migration.id) - ); + private async createGetMigrationsName(): Promise<(id: string) => string> { + // Cache the names map to avoid repeat the aggregation query + if (!this.namesMap) { + const index = this.ruleMigrationIndexAdapters.rules.getIndexName(this.spaceId); + + // Same migrationIds aggregation as the getAllStats method of the data client + const aggregations: { migrationIds: AggregationsAggregationContainer } = { + migrationIds: { + terms: { field: 'migration_id', order: { createdAt: 'asc' }, size: MAX_ES_SEARCH_SIZE }, + aggregations: { createdAt: { min: { field: '@timestamp' } } }, + }, + }; + const result = await this.esClient + .search({ index, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting all rule migrations stats: ${error.message}`); + throw error; + }); + + const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; + const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; + const migrationsNames = buckets.map((bucket, i) => ({ + id: `${bucket.key}`, + name: `SIEM rules migration #${i + 1}`, // the same naming pattern as in older versions + })); - if (migrationsToIndex.length > 0) { - this.logger.info( - `Found ${migrationsToIndex.length} rule migration documents from rules index with an absent migration doc. Creating corresponding migration documents.` - ); - await this.indexMigrationDocs( - migrationsToIndex.map((migration) => ({ - ...migration, - created_by: migration.created_by ?? '', - created_at: migration.created_at ?? new Date().toISOString(), - name: '', - })) - ); - this.logger.info( - `Created ${migrationsToIndex.length} rule migration documents from rules index with an absent migration doc.` - ); + this.namesMap = new Map(migrationsNames.map(({ id, name }) => [id, name])); } - const migrationsMaybeMissingName = existingMigrationsFromMigrationsIndex - .filter((migration) => !migration.name) - .map((migration) => ({ - ...migration, - created_by: migration.created_by ?? '', - created_at: migration.created_at ?? new Date().toISOString(), - name: `SIEM Migration ${ - existingMigrationsFromMigrationsIndex.findIndex((m) => m.id === migration.id) + 1 - }`, - })); + const getMigrationName = (migrationId: string): string => { + return this.namesMap?.get(migrationId) ?? `SIEM Migration ${migrationId}`; // Fallback name using the ID (should never happen, but just in case) + }; - if (migrationsMaybeMissingName.length > 0) { - this.logger.info( - `Found ${migrationsMaybeMissingName.length} migration documents with an absent name. Updating them.` - ); - await this.updateMigrationDocs(migrationsMaybeMissingName); - this.logger.info( - `Updated ${migrationsMaybeMissingName.length} migration documents with an absent name.` - ); - } + return getMigrationName; } } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 758d422b1e1c2..c6b2db4e2d937 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -140,7 +140,7 @@ export class RuleMigrationsTaskClient { } const dataStats = await this.data.rules.getStats(migrationId); const taskStats = this.getTaskStats(migration, dataStats.rules); - return { ...taskStats, ...dataStats }; + return { ...taskStats, ...dataStats, name: migration.name }; } /** Returns the stats of all migrations */