[SecuritySolution] Security Entity Store Fix Data view refresh does not support the indexPattern parameter

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -19,7 +19,10 @@ import { mockGlobalState } from '../../../../public/common/mock';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
 import { EntityType } from '../../../../common/search_strategy';
-import type { InitEntityEngineResponse } from '../../../../common/api/entity_analytics';
+import type {
+  EngineDescriptor,
+  InitEntityEngineResponse,
+} from '../../../../common/api/entity_analytics';
 import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 import { defaultOptions } from './constants';
 import type { SecurityPluginStart } from '@kbn/security-plugin/server';
@@ -50,13 +53,30 @@ const definition: EntityDefinition = convertToEntityManagerDefinition(
   { namespace: 'test', filter: '' }
 );
 
+const engine: EngineDescriptor = {
+  type: 'user',
+  frequency: '',
+  fieldHistoryLength: 0,
+  indexPattern: '',
+  lookbackPeriod: '',
+  timeout: '',
+  delay: '',
+  status: 'started',
+};
+
 const stubSecurityDataView = createStubDataView({
   spec: {
     id: 'security',
     title: 'security',
   },
 });
 
+const defaultIndexPatterns = [
+  stubSecurityDataView.getIndexPattern(),
+  '.asset-criticality.asset-criticality-default',
+  'risk-score.risk-score-latest-default',
+];
+
 const dataviewService = {
   ...dataViewPluginMocks.createStartContract(),
   get: () => Promise.resolve(stubSecurityDataView),
@@ -427,7 +447,7 @@ describe('EntityStoreDataClient', () => {
     });
 
     it('applies data view indices to the entity store', async () => {
-      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockListDescriptor.mockResolvedValueOnce({ engines: [engine] });
       mockGetEntityDefinition.mockResolvedValueOnce({
         definitions: [definition],
       });
@@ -439,6 +459,25 @@ describe('EntityStoreDataClient', () => {
       expect(response.successes.length).toBe(1);
     });
 
+    it('adds the engine indexPattern to the the entity store', async () => {
+      const indexPattern = 'testIndex';
+      mockListDescriptor.mockResolvedValueOnce({
+        engines: [{ ...engine, indexPattern }],
+      });
+      mockGetEntityDefinition.mockResolvedValueOnce({
+        definitions: [definition],
+      });
+
+      const response = await dataClient.applyDataViewIndices();
+
+      expect(mockUpdateEntityDefinition).toHaveBeenCalled();
+      expect(response.errors.length).toBe(0);
+      expect(response.successes.length).toBe(1);
+      expect(response.successes[0].changes).toEqual({
+        indexPatterns: [...defaultIndexPatterns, 'testIndex'],
+      });
+    });
+
     it('returns empty successes and errors if no engines found', async () => {
       mockListDescriptor.mockResolvedValueOnce({ engines: [] });
 
@@ -448,32 +487,35 @@ describe('EntityStoreDataClient', () => {
       expect(response.errors.length).toBe(0);
     });
 
-    it('throws an error if the user does not have required privileges', async () => {
+    it('return an error if the user does not have required privileges', async () => {
       mockCheckPrivileges.mockReturnValueOnce({
         hasAllRequested: false,
         privileges: {
           elasticsearch: { cluster: [], index: [] },
           kibana: [],
         },
       });
+      mockListDescriptor.mockResolvedValueOnce({
+        engines: [engine],
+      });
+      mockGetEntityDefinition.mockResolvedValueOnce({
+        definitions: [definition],
+      });
 
-      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      const result = await dataClient.applyDataViewIndices();
 
-      await expect(dataClient.applyDataViewIndices()).rejects.toThrow(
+      await expect(result.errors.length).toBe(1);
+      await expect(result.errors[0].message).toMatch(
         /The current user does not have the required indices privileges.*/
       );
     });
 
     it('skips update if index patterns are the same', async () => {
-      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockListDescriptor.mockResolvedValueOnce({ engines: [engine] });
       mockGetEntityDefinition.mockResolvedValueOnce({
         definitions: [
           {
-            indexPatterns: [
-              stubSecurityDataView.getIndexPattern(),
-              '.asset-criticality.asset-criticality-default',
-              'risk-score.risk-score-latest-default',
-            ],
+            indexPatterns: defaultIndexPatterns,
           },
         ],
       });
@@ -488,7 +530,7 @@ describe('EntityStoreDataClient', () => {
     it('handles errors during update', async () => {
       const testErrorMessages = 'Update failed';
       mockUpdateEntityDefinition.mockRejectedValueOnce(new Error(testErrorMessages));
-      mockListDescriptor.mockResolvedValueOnce({ engines: [{}] });
+      mockListDescriptor.mockResolvedValueOnce({ engines: [engine] });
       mockGetEntityDefinition.mockResolvedValueOnce({
         definitions: [definition],
       });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -88,6 +88,7 @@ import {
   getEntitiesIndexName,
   isPromiseFulfilled,
   isPromiseRejected,
+  mergeEntityStoreIndices,
 } from './utils';
 import { EntityEngineActions } from './auditing/actions';
 import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit';
@@ -803,31 +804,12 @@ export class EntityStoreDataClient {
       };
     }
 
-    const indexPatterns = await buildIndexPatterns(
+    const defaultIndexPatterns = await buildIndexPatterns(
       this.options.namespace,
       this.options.appClient,
       this.options.dataViewsService
     );
 
-    const privileges = await getEntityStoreSourceIndicesPrivileges(
-      this.options.request,
-      this.options.security,
-      indexPatterns
-    );
-
-    if (!privileges.has_all_required) {
-      const missingPrivilegesMsg = getAllMissingPrivileges(privileges).elasticsearch.index.map(
-        ({ indexName, privileges: missingPrivileges }) =>
-          `Missing [${missingPrivileges.join(', ')}] privileges for index '${indexName}'.`
-      );
-
-      throw new Error(
-        `The current user does not have the required indices privileges.\n${missingPrivilegesMsg.join(
-          '\n'
-        )}`
-      );
-    }
-
     const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = engines.map(
       async (engine) => {
         const originalStatus = engine.status;
@@ -843,6 +825,8 @@ export class EntityStoreDataClient {
           );
         }
 
+        const indexPatterns = mergeEntityStoreIndices(defaultIndexPatterns, engine.indexPattern);
+
         // Skip update if index patterns are the same
         if (isEqual(definition.indexPatterns, indexPatterns)) {
           logger.debug(
@@ -855,6 +839,25 @@ export class EntityStoreDataClient {
           );
         }
 
+        const privileges = await getEntityStoreSourceIndicesPrivileges(
+          this.options.request,
+          this.options.security,
+          indexPatterns
+        );
+
+        if (!privileges.has_all_required) {
+          const missingPrivilegesMsg = getAllMissingPrivileges(privileges).elasticsearch.index.map(
+            ({ indexName, privileges: missingPrivileges }) =>
+              `Missing [${missingPrivileges.join(', ')}] privileges for index '${indexName}'.`
+          );
+
+          throw new Error(
+            `The current user does not have the required indices privileges for updating the '${
+              engine.type
+            }' entity store.\n${missingPrivilegesMsg.join('\n')}`
+          );
+        }
+
         // Update savedObject status
         await this.engineClient.updateStatus(engine.type, ENGINE_STATUS.UPDATING);
 

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/installation/engine_description.ts

@@ -17,7 +17,7 @@ import {
   serviceEntityEngineDescription,
 } from '../entity_definitions/entity_descriptions';
 import type { EntityStoreConfig } from '../types';
-import { buildEntityDefinitionId } from '../utils';
+import { buildEntityDefinitionId, mergeEntityStoreIndices } from '../utils';
 import type { EntityDescription } from '../entity_definitions/types';
 import type { EntityEngineInstallationDescriptor } from './types';
 import { merge } from '../../../../../common/utils/objects/merge';
@@ -46,9 +46,7 @@ export const createEngineDescription = (params: EngineDescriptionParams) => {
   };
   const options = merge(defaultOptions, merge(fileConfig, requestParams));
 
-  const indexPatterns = options.indexPattern
-    ? defaultIndexPatterns.concat(options.indexPattern.split(','))
-    : defaultIndexPatterns;
+  const indexPatterns = mergeEntityStoreIndices(defaultIndexPatterns, options.indexPattern);
 
   const description = engineDescriptionRegistry[entityType];
 

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts

@@ -52,9 +52,9 @@ export const applyDataViewIndicesEntityEngineRoute = (
           if (successes.length === 0 && errors.length > 0) {
             return siemResponse.error({
               statusCode: 500,
-              body: `Error in ApplyEntityEngineDataViewIndices. Errors: [${errorMessages.join(
-                ', '
-              )}]`,
+              body: `Errors applying data view changes to the entity store. Errors: \n${errorMessages.join(
+                '\n\n'
+              )}`,
             });
           }
 

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.test.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mergeEntityStoreIndices } from './entity_utils';
+
+describe('mergeEntityStoreIndices', () => {
+  it('returns the original indices if indexPattern is empty', () => {
+    const indices = ['index1', 'index2'];
+    const result = mergeEntityStoreIndices(indices, '');
+    expect(result).toEqual(indices);
+  });
+
+  it('merges indices with indexPattern when indexPattern is provided', () => {
+    const indices = ['index1', 'index2'];
+    const indexPattern = 'index3,index4';
+    const result = mergeEntityStoreIndices(indices, indexPattern);
+    expect(result).toEqual(['index1', 'index2', 'index3', 'index4']);
+  });
+
+  it('deduplicate indices', () => {
+    const indices = ['index1', 'index2'];
+    const indexPattern = 'index2,index3';
+    const result = mergeEntityStoreIndices(indices, indexPattern);
+    expect(result).toEqual(['index1', 'index2', 'index3']);
+  });
+
+  it('returns an empty array if both indices and indexPattern are empty', () => {
+    const indices: string[] = [];
+    const indexPattern = '';
+    const result = mergeEntityStoreIndices(indices, indexPattern);
+    expect(result).toEqual([]);
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts

@@ -11,6 +11,7 @@ import {
   entitiesIndexPattern,
 } from '@kbn/entities-schema';
 import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
+import { uniq } from 'lodash/fp';
 import type { AppClient } from '../../../../types';
 import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
 import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
@@ -77,3 +78,6 @@ export const isPromiseFulfilled = <T>(
 export const isPromiseRejected = <T>(
   result: PromiseSettledResult<T>
 ): result is PromiseRejectedResult => result.status === 'rejected';
+
+export const mergeEntityStoreIndices = (indices: string[], indexPattern: string | undefined) =>
+  indexPattern ? uniq(indices.concat(indexPattern.split(','))) : indices;