[Siem Migrations] Adds separate migration index to store migration metadata

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts

@@ -7,7 +7,6 @@
 
 import type { IKibanaResponse, Logger } from '@kbn/core/server';
 import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
-import { v4 as uuidV4 } from 'uuid';
 import { SIEM_RULE_MIGRATION_CREATE_PATH } from '../../../../../common/siem_migrations/constants';
 import {
   CreateRuleMigrationRequestBody,
@@ -44,17 +43,26 @@ export const registerSiemRuleMigrationsCreateRoute = (
       withLicense(
         async (context, req, res): Promise<IKibanaResponse<CreateRuleMigrationResponse>> => {
           const originalRules = req.body;
-          const migrationId = req.params.migration_id ?? uuidV4();
           const siemMigrationAuditLogger = new SiemMigrationAuditLogger(context.securitySolution);
+          const providedMigrationId = req.params?.migration_id;
           try {
             const [firstOriginalRule] = originalRules;
             if (!firstOriginalRule) {
               return res.noContent();
             }
             const ctx = await context.resolve(['securitySolution']);
             const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient();
+            await siemMigrationAuditLogger.logCreateMigration({ migrationId: providedMigrationId });
 
-            await siemMigrationAuditLogger.logCreateMigration({ migrationId });
+            let migrationId: string;
+
+            if (!providedMigrationId) {
+              /** if new migration */
+              migrationId = await ruleMigrationsClient.data.migrations.create();
+            } else {
+              /** if updating existing migration */
+              migrationId = providedMigrationId;
+            }
 
             const ruleMigrations = originalRules.map<CreateRuleMigrationInput>((originalRule) => ({
               migration_id: migrationId,
@@ -76,7 +84,10 @@ export const registerSiemRuleMigrationsCreateRoute = (
             return res.ok({ body: { migration_id: migrationId } });
           } catch (error) {
             logger.error(error);
-            await siemMigrationAuditLogger.logCreateMigration({ migrationId, error });
+            await siemMigrationAuditLogger.logCreateMigration({
+              migrationId: providedMigrationId,
+              error,
+            });
             return res.badRequest({ body: error.message });
           }
         }

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/api/util/audit.ts

@@ -106,7 +106,7 @@ export class SiemMigrationAuditLogger {
     }
   }
 
-  public async logCreateMigration(params: { migrationId: string; error?: Error }): Promise<void> {
+  public async logCreateMigration(params: { migrationId?: string; error?: Error }): Promise<void> {
     const { migrationId, error } = params;
     const message = `User created a new SIEM migration with [id=${migrationId}]`;
     return this.log({

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts

@@ -7,6 +7,7 @@
 
 import type { RuleMigrationsDataIntegrationsClient } from '../rule_migrations_data_integrations_client';
 import type { RuleMigrationsDataLookupsClient } from '../rule_migrations_data_lookups_client';
+import type { RuleMigrationsDataMigrationClient } from '../rule_migrations_data_migration_client';
 import type { RuleMigrationsDataPrebuiltRulesClient } from '../rule_migrations_data_prebuilt_rules_client';
 import type { RuleMigrationsDataResourcesClient } from '../rule_migrations_data_resources_client';
 import type { RuleMigrationsDataRulesClient } from '../rule_migrations_data_rules_client';
@@ -57,6 +58,10 @@ export const mockRuleMigrationsDataLookupsClient = {
   create: jest.fn().mockResolvedValue(undefined),
   indexData: jest.fn().mockResolvedValue(undefined),
 } as unknown as jest.Mocked<RuleMigrationsDataLookupsClient>;
+export const mockRuleMigrationsDataMigrationsClient = {
+  create: jest.fn().mockResolvedValue(undefined),
+  get: jest.fn().mockResolvedValue(undefined),
+} as unknown as jest.Mocked<RuleMigrationsDataMigrationClient>;
 
 // Rule migrations data client
 export const createRuleMigrationsDataClientMock = () => ({
@@ -65,6 +70,7 @@ export const createRuleMigrationsDataClientMock = () => ({
   integrations: mockRuleMigrationsDataIntegrationsClient,
   prebuiltRules: mockRuleMigrationsDataPrebuiltRulesClient,
   lookups: mockRuleMigrationsDataLookupsClient,
+  migrations: mockRuleMigrationsDataMigrationsClient,
 });
 
 export const MockRuleMigrationsDataClient = jest

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts

@@ -18,8 +18,7 @@ import type {
   Logger,
 } from '@kbn/core/server';
 import assert from 'assert';
-import type { Stored, SiemRuleMigrationsClientDependencies } from '../types';
-import type { IndexNameProvider } from './rule_migrations_data_client';
+import type { IndexNameProvider, SiemRuleMigrationsClientDependencies, Stored } from '../types';
 
 const DEFAULT_PIT_KEEP_ALIVE: Duration = '30s' as const;
 
@@ -53,22 +52,25 @@ export class RuleMigrationsDataBaseClient {
     }
   }
 
-  protected processResponseHits<T extends object>(
-    response: SearchResponse<T>,
-    override?: Partial<T>
-  ): Array<Stored<T>> {
-    return this.processHits(response.hits.hits, override);
+  protected processHit<T extends object>(hit: SearchHit<T>, override: Partial<T> = {}): Stored<T> {
+    const { _id, _source } = hit;
+    assert(_id, 'document should have _id');
+    assert(_source, 'document should have _source');
+    return { ..._source, ...override, id: _id };
   }
 
   protected processHits<T extends object>(
     hits: Array<SearchHit<T>> = [],
     override: Partial<T> = {}
   ): Array<Stored<T>> {
-    return hits.map(({ _id, _source }) => {
-      assert(_id, 'document should have _id');
-      assert(_source, 'document should have _source');
-      return { ..._source, ...override, id: _id };
-    });
+    return hits.map((hit) => this.processHit(hit, override));
+  }
+
+  protected processResponseHits<T extends object>(
+    response: SearchResponse<T>,
+    override?: Partial<T>
+  ): Array<Stored<T>> {
+    return this.processHits(response.hits.hits, override);
   }
 
   protected getTotalHits(response: SearchResponse) {

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts

@@ -11,13 +11,11 @@ import { RuleMigrationsDataPrebuiltRulesClient } from './rule_migrations_data_pr
 import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client';
 import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client';
 import { RuleMigrationsDataLookupsClient } from './rule_migrations_data_lookups_client';
-import type { SiemRuleMigrationsClientDependencies } from '../types';
-import type { AdapterId } from './rule_migrations_data_service';
-
-export type IndexNameProvider = () => Promise<string>;
-export type IndexNameProviders = Record<AdapterId, IndexNameProvider>;
+import type { IndexNameProviders, SiemRuleMigrationsClientDependencies } from '../types';
+import { RuleMigrationsDataMigrationClient } from './rule_migrations_data_migration_client';
 
 export class RuleMigrationsDataClient {
+  public readonly migrations: RuleMigrationsDataMigrationClient;
   public readonly rules: RuleMigrationsDataRulesClient;
   public readonly resources: RuleMigrationsDataResourcesClient;
   public readonly integrations: RuleMigrationsDataIntegrationsClient;
@@ -32,6 +30,13 @@ export class RuleMigrationsDataClient {
     spaceId: string,
     dependencies: SiemRuleMigrationsClientDependencies
   ) {
+    this.migrations = new RuleMigrationsDataMigrationClient(
+      indexNameProviders.migrations,
+      currentUser,
+      esScopedClient,
+      logger,
+      dependencies
+    );
     this.rules = new RuleMigrationsDataRulesClient(
       indexNameProviders.rules,
       currentUser,

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.test.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { IScopedClusterClient } from '@kbn/core/server';
+import type { SiemRuleMigrationsClientDependencies } from '../types';
+import { RuleMigrationsDataMigrationClient } from './rule_migrations_data_migration_client';
+import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
+import type { AuthenticatedUser } from '@kbn/security-plugin-types-common';
+import type IndexApi from '@elastic/elasticsearch/lib/api/api';
+import type GetApi from '@elastic/elasticsearch/lib/api/api/get';
+
+describe('RuleMigrationsDataMigrationClient', () => {
+  let ruleMigrationsDataMigrationClient: RuleMigrationsDataMigrationClient;
+  const esClient =
+    elasticsearchServiceMock.createCustomClusterClient() as unknown as IScopedClusterClient;
+
+  const logger = loggingSystemMock.createLogger();
+  const indexNameProvider = jest.fn().mockReturnValue('.kibana-siem-rule-migrations');
+  const currentUser = {
+    userName: 'testUser',
+    profile_uid: 'testProfileUid',
+  } as unknown as AuthenticatedUser;
+  const dependencies = {} as unknown as SiemRuleMigrationsClientDependencies;
+
+  beforeEach(() => {
+    ruleMigrationsDataMigrationClient = new RuleMigrationsDataMigrationClient(
+      indexNameProvider,
+      currentUser,
+      esClient,
+      logger,
+      dependencies
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('create', () => {
+    test('should create a new migration', async () => {
+      const index = '.kibana-siem-rule-migrations';
+
+      const result = await ruleMigrationsDataMigrationClient.create();
+
+      expect(result).not.toBeFalsy();
+      expect(esClient.asInternalUser.create).toHaveBeenCalledWith({
+        refresh: 'wait_for',
+        id: result,
+        index,
+        document: {
+          created_by: currentUser.profile_uid,
+          created_at: expect.any(String),
+        },
+      });
+    });
+
+    test('should throw an error if an error occurs', async () => {
+      (
+        esClient.asInternalUser.create as unknown as jest.MockedFn<typeof IndexApi>
+      ).mockRejectedValueOnce(new Error('Test error'));
+
+      await expect(ruleMigrationsDataMigrationClient.create()).rejects.toThrow('Test error');
+
+      expect(esClient.asInternalUser.create).toHaveBeenCalled();
+      expect(logger.error).toHaveBeenCalled();
+    });
+  });
+
+  describe('get', () => {
+    test('should get a migration', async () => {
+      const index = '.kibana-siem-rule-migrations';
+      const id = 'testId';
+      const response = {
+        _index: index,
+        found: true,
+        _source: {
+          created_by: currentUser.profile_uid,
+          created_at: new Date().toISOString(),
+        },
+        _id: id,
+      };
+
+      (
+        esClient.asInternalUser.get as unknown as jest.MockedFn<typeof GetApi>
+      ).mockResolvedValueOnce(response);
+
+      const result = await ruleMigrationsDataMigrationClient.get({ id });
+
+      expect(result).toEqual({
+        ...response._source,
+        id: response._id,
+      });
+    });
+    test('should throw an error if an error occurs', async () => {
+      const id = 'testId';
+      (
+        esClient.asInternalUser.get as unknown as jest.MockedFn<typeof GetApi>
+      ).mockRejectedValueOnce(new Error('Test error'));
+
+      await expect(ruleMigrationsDataMigrationClient.get({ id })).rejects.toThrow('Test error');
+
+      expect(esClient.asInternalUser.get).toHaveBeenCalled();
+      expect(logger.error).toHaveBeenCalledWith(`Error getting migration ${id}: Error: Test error`);
+    });
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_migration_client.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { v4 as uuidV4 } from 'uuid';
+import type { StoredSiemMigration } from '../types';
+import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
+
+export class RuleMigrationsDataMigrationClient extends RuleMigrationsDataBaseClient {
+  async create(): Promise<string> {
+    const migrationId = uuidV4();
+    const index = await this.getIndexName();
+    const profileUid = await this.getProfileUid();
+    const createdAt = new Date().toISOString();
+
+    await this.esClient
+      .create({
+        refresh: 'wait_for',
+        id: migrationId,
+        index,
+        document: {
+          created_by: profileUid,
+          created_at: createdAt,
+        },
+      })
+      .catch((error) => {
+        this.logger.error(`Error creating migration ${migrationId}: ${error}`);
+        throw error;
+      });
+
+    return migrationId;
+  }
+
+  async get({ id }: { id: string }): Promise<StoredSiemMigration> {
+    const index = await this.getIndexName();
+    return this.esClient
+      .get<StoredSiemMigration>({
+        index,
+        id,
+      })
+      .then((document) => {
+        return this.processHit(document);
+      })
+      .catch((error) => {
+        this.logger.error(`Error getting migration ${id}: ${error}`);
+        throw error;
+      });
+  }
+}

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts

@@ -21,14 +21,14 @@ import {
   SiemMigrationStatus,
   RuleTranslationResult,
 } from '../../../../../common/siem_migrations/constants';
+import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen';
 import {
-  type RuleMigration,
   type RuleMigrationTaskStats,
   type RuleMigrationTranslationStats,
 } from '../../../../../common/siem_migrations/model/rule_migration.gen';
-import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
 import { getSortingOptions, type RuleMigrationSort } from './sort';
 import { conditions as searchConditions } from './search';
+import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client';
 
 export type CreateRuleMigrationInput = Omit<
   RuleMigration,

x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts

@@ -12,8 +12,7 @@ import type { InstallParams } from '@kbn/index-adapter';
 import { IndexPatternAdapter, IndexAdapter } from '@kbn/index-adapter';
 import { loggerMock } from '@kbn/logging-mocks';
 import { Subject } from 'rxjs';
-import type { SiemRuleMigrationsClientDependencies } from '../types';
-import type { IndexNameProviders } from './rule_migrations_data_client';
+import type { IndexNameProviders, SiemRuleMigrationsClientDependencies } from '../types';
 import { INDEX_PATTERN, RuleMigrationsDataService } from './rule_migrations_data_service';
 
 jest.mock('@kbn/index-adapter');
@@ -45,7 +44,7 @@ describe('SiemRuleMigrationsDataService', () => {
   describe('constructor', () => {
     it('should create IndexPatternAdapters', () => {
       new RuleMigrationsDataService(logger, kibanaVersion);
-      expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(2);
+      expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(3);
       expect(MockedIndexAdapter).toHaveBeenCalledTimes(2);
     });
 
@@ -118,7 +117,8 @@ describe('SiemRuleMigrationsDataService', () => {
         logger: loggerMock.create(),
         pluginStop$: new Subject(),
       };
-      const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter] =
+
+      const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter, migrationIndexPatternAdapter] =
         MockedIndexPatternAdapter.mock.instances;
       (rulesIndexPatternAdapter.install as jest.Mock).mockResolvedValueOnce(undefined);
 
@@ -127,12 +127,16 @@ describe('SiemRuleMigrationsDataService', () => {
 
       await mockIndexNameProviders.rules();
       await mockIndexNameProviders.resources();
+      await mockIndexNameProviders.migrations();
 
       expect(rulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
       expect(rulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
 
       expect(resourcesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
       expect(resourcesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
+
+      expect(migrationIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1');
+      expect(migrationIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1');
     });
   });
 });