[Privmon] Synchronize Privileged User index based on configured Entity Sources

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/privilege_monitoring/monitoring_entity_source/monitoring_entity_source.schema.yaml

@@ -73,6 +73,19 @@ paths:
         "200":
           description: Entity source deleted successfully
 
+  /api/entity_analytics/monitoring/entity_source/list:
+    get:
+      operationId: listEntitySources
+      summary: List all entity source configurations
+      responses:
+        "200":
+          description: List of entity sources retrieved
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/MonitoringEntitySourceDescriptor"
 components:
   schemas:
     MonitoringEntitySourceDescriptor:

x-pack/solutions/security/plugins/security_solution/common/entity_analytics/privilege_monitoring/constants.ts

@@ -14,3 +14,5 @@ export const getPrivilegedMonitorUsersIndex = (namespace: string) =>
 // Not required in phase 0.
 export const getPrivilegedMonitorGroupsIndex = (namespace: string) =>
   `${privilegedMonitorBaseIndexName}.groups-${namespace}`;
+// Default index for privileged monitoring users. Not required.
+export const defaultMonitoringUsersIndex = 'entity_analytics.privileged_monitoring';

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/auth/privileges.ts

@@ -6,7 +6,7 @@
  */
 
 import { PRIVILEGE_MONITORING_INTERNAL_INDICES_PATTERN } from '../constants';
-import { privilegeMonitoringTypeName } from '../saved_object/privilege_monitoring_type';
+import { privilegeMonitoringTypeName } from '../saved_objects';
 
 export const privilegeMonitoringRuntimePrivileges = (sourceIndices: string[]) => ({
   cluster: ['manage_ingest_pipelines', 'manage_index_templates'],

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/monitoring_entity_source_data_client.test.ts

@@ -11,7 +11,7 @@ import {
   elasticsearchServiceMock,
   loggingSystemMock,
 } from '@kbn/core/server/mocks';
-import { monitoringEntitySourceTypeName } from './saved_object/monitoring_entity_source_type';
+import { monitoringEntitySourceTypeName } from './saved_objects';
 import type { SavedObject, SavedObjectsFindResponse } from '@kbn/core/server';
 
 describe('MonitoringEntitySourceDataClient', () => {
@@ -32,6 +32,7 @@ describe('MonitoringEntitySourceDataClient', () => {
   const testDescriptor = {
     type: 'test-type',
     name: 'Test Source',
+    indexPattern: 'test-index-pattern',
     matchers: [
       {
         fields: ['user.role'],
@@ -49,13 +50,19 @@ describe('MonitoringEntitySourceDataClient', () => {
 
   describe('init', () => {
     it('should initialize Monitoring Entity Source Sync Config Successfully', async () => {
+      defaultOpts.soClient.update.mockImplementation(() => {
+        const err = new Error('Not found');
+        // Simulate Kibana-style 404 error
+        (err as Error & { output?: { statusCode: number } }).output = { statusCode: 404 };
+        throw err;
+      });
       defaultOpts.soClient.find.mockResolvedValue({
         total: 0,
         saved_objects: [],
       } as unknown as SavedObjectsFindResponse<unknown, unknown>);
 
       defaultOpts.soClient.create.mockResolvedValue({
-        id: `entity-analytics-monitoring-entity-source-${namespace}`,
+        id: 'temp-id', // TODO: update to use dynamic ID
         type: monitoringEntitySourceTypeName,
         attributes: testDescriptor,
         references: [],
@@ -66,7 +73,9 @@ describe('MonitoringEntitySourceDataClient', () => {
       expect(defaultOpts.soClient.create).toHaveBeenCalledWith(
         monitoringEntitySourceTypeName,
         testDescriptor,
-        { id: `entity-analytics-monitoring-entity-source-${namespace}` }
+        {
+          id: `entity-analytics-monitoring-entity-source-${namespace}-${testDescriptor.type}-${testDescriptor.indexPattern}`,
+        }
       );
 
       expect(result).toEqual(testDescriptor);
@@ -76,7 +85,6 @@ describe('MonitoringEntitySourceDataClient', () => {
   describe('get', () => {
     it('should get Monitoring Entity Source Sync Config Successfully', async () => {
       const getResponse = {
-        id: `entity-analytics-monitoring-entity-source-${namespace}`,
         type: monitoringEntitySourceTypeName,
         attributes: testDescriptor,
         references: [],
@@ -85,7 +93,7 @@ describe('MonitoringEntitySourceDataClient', () => {
       const result = await dataClient.get();
       expect(defaultOpts.soClient.get).toHaveBeenCalledWith(
         monitoringEntitySourceTypeName,
-        `entity-analytics-monitoring-entity-source-${namespace}`
+        `temp-id` // TODO: https://github.com/elastic/security-team/issues/12851
       );
       expect(result).toEqual(getResponse.attributes);
     });
@@ -98,12 +106,25 @@ describe('MonitoringEntitySourceDataClient', () => {
         saved_objects: [{ attributes: testDescriptor }],
       } as unknown as SavedObjectsFindResponse<unknown, unknown>;
 
+      const testSourceObject = {
+        filter: {},
+        indexPattern: 'test-index-pattern',
+        matchers: [
+          {
+            fields: ['user.role'],
+            values: ['admin'],
+          },
+        ],
+        name: 'Test Source',
+        type: 'test-type',
+      };
+
       defaultOpts.soClient.find.mockResolvedValue(
         existingDescriptor as unknown as SavedObjectsFindResponse<unknown, unknown>
       );
 
       defaultOpts.soClient.update.mockResolvedValue({
-        id: `entity-analytics-monitoring-entity-source-${namespace}`,
+        id: `temp-id`, // TODO: https://github.com/elastic/security-team/issues/12851
         type: monitoringEntitySourceTypeName,
         attributes: { ...testDescriptor, name: 'Updated Source' },
         references: [],
@@ -114,8 +135,8 @@ describe('MonitoringEntitySourceDataClient', () => {
 
       expect(defaultOpts.soClient.update).toHaveBeenCalledWith(
         monitoringEntitySourceTypeName,
-        `entity-analytics-monitoring-entity-source-${namespace}`,
-        testDescriptor,
+        `entity-analytics-monitoring-entity-source-${namespace}-${testDescriptor.type}-${testDescriptor.indexPattern}`,
+        testSourceObject,
         { refresh: 'wait_for' }
       );
 
@@ -128,7 +149,7 @@ describe('MonitoringEntitySourceDataClient', () => {
       await dataClient.delete();
       expect(mockSavedObjectClient.delete).toHaveBeenCalledWith(
         monitoringEntitySourceTypeName,
-        `entity-analytics-monitoring-entity-source-${namespace}`
+        `temp-id` // TODO: https://github.com/elastic/security-team/issues/12851
       );
     });
   });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/monitoring_entity_source_data_client.ts

@@ -10,7 +10,7 @@ import type {
   MonitoringEntitySourceDescriptor,
   MonitoringEntitySourceResponse,
 } from '../../../../common/api/entity_analytics/privilege_monitoring/monitoring_entity_source/monitoring_entity_source.gen';
-import { MonitoringEntitySourceDescriptorClient } from './saved_object/monitoring_entity_source';
+import { MonitoringEntitySourceDescriptorClient } from './saved_objects';
 
 interface MonitoringEntitySourceDataClientOpts {
   logger: Logger;
@@ -31,7 +31,9 @@ export class MonitoringEntitySourceDataClient {
   public async init(
     input: MonitoringEntitySourceDescriptor
   ): Promise<MonitoringEntitySourceResponse> {
-    const descriptor = await this.monitoringEntitySourceClient.create(input);
+    const descriptor = await this.monitoringEntitySourceClient.create({
+      ...input,
+    });
     this.log('debug', 'Initializing MonitoringEntitySourceDataClient Saved Object');
     return descriptor;
   }
@@ -46,7 +48,7 @@ export class MonitoringEntitySourceDataClient {
 
     const sanitizedUpdate = {
       ...update,
-      matchers: update.matchers?.map((matcher) => ({
+      matchers: update.matchers?.map((matcher: { fields: string[]; values: string[] }) => ({
         fields: matcher.fields ?? [],
         values: matcher.values ?? [],
       })),
@@ -60,6 +62,11 @@ export class MonitoringEntitySourceDataClient {
     return this.monitoringEntitySourceClient.delete();
   }
 
+  public async list(): Promise<MonitoringEntitySourceResponse[]> {
+    this.log('debug', 'Finding all Monitoring Entity Source Sync saved objects');
+    return this.monitoringEntitySourceClient.findAll();
+  }
+
   private log(level: Exclude<keyof Logger, 'get' | 'log' | 'isLevelEnabled'>, msg: string) {
     this.opts.logger[level](
       `[Monitoring Entity Source Sync][namespace: ${this.opts.namespace}] ${msg}`

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/privilege_monitoring_data_client.test.ts

@@ -24,17 +24,23 @@ jest.mock('./tasks/privilege_monitoring_task', () => {
   };
 });
 
-jest.mock('./saved_object/privilege_monitoring', () => {
+jest.mock('./saved_objects', () => {
   return {
+    MonitoringEntitySourceDescriptorClient: jest.fn().mockImplementation(() => ({
+      findByIndex: jest.fn().mockResolvedValue([]),
+      create: jest.fn(),
+    })),
     PrivilegeMonitoringEngineDescriptorClient: jest.fn().mockImplementation(() => ({
       init: jest.fn().mockResolvedValue({ status: 'success' }),
       update: jest.fn(),
     })),
   };
 });
+
 describe('Privilege Monitoring Data Client', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
   const clusterClientMock = elasticsearchServiceMock.createScopedClusterClient();
+  const esClientMock = clusterClientMock.asCurrentUser;
   const loggerMock = loggingSystemMock.createLogger();
   const auditMock = { log: jest.fn().mockReturnValue(undefined) };
   loggerMock.debug = jest.fn();
@@ -66,7 +72,7 @@ describe('Privilege Monitoring Data Client', () => {
 
       expect(mockCreateOrUpdateIndex).toHaveBeenCalled();
       expect(mockStartPrivilegeMonitoringTask).toHaveBeenCalled();
-      expect(loggerMock.debug).toHaveBeenCalledTimes(1);
+      expect(loggerMock.debug).toHaveBeenCalledTimes(3);
       expect(auditMock.log).toHaveBeenCalled();
       expect(result).toEqual(mockDescriptor);
     });
@@ -137,4 +143,124 @@ describe('Privilege Monitoring Data Client', () => {
       // TODO: implement once we have more auditing
     });
   });
+
+  describe('syncAllIndexUsers', () => {
+    const mockLog = jest.fn();
+
+    it('should sync all index users successfully', async () => {
+      const mockMonitoringSOSources = [
+        { name: 'source1', indexPattern: 'index1' },
+        { name: 'source2', indexPattern: 'index2' },
+      ];
+      const findByIndexMock = jest.fn().mockResolvedValue(mockMonitoringSOSources);
+      Object.defineProperty(dataClient, 'monitoringIndexSourceClient', {
+        value: {
+          init: jest.fn().mockResolvedValue({ status: 'success' }),
+          update: jest.fn(),
+          findByIndex: findByIndexMock,
+        },
+      });
+      dataClient.syncUsernamesFromIndex = jest.fn().mockResolvedValue(['user1', 'user2']);
+      await dataClient.plainIndexSync();
+      expect(findByIndexMock).toHaveBeenCalled();
+      expect(dataClient.syncUsernamesFromIndex).toHaveBeenCalledTimes(2);
+      expect(dataClient.syncUsernamesFromIndex).toHaveBeenCalledWith({
+        indexName: 'index1',
+        kuery: undefined,
+      });
+    });
+
+    it('logs and returns if no index sources', async () => {
+      Object.defineProperty(dataClient, 'log', { value: mockLog });
+      const findByIndexMock = jest.fn().mockResolvedValue([]);
+      Object.defineProperty(dataClient, 'monitoringIndexSourceClient', {
+        value: {
+          init: jest.fn().mockResolvedValue({ status: 'success' }),
+          update: jest.fn(),
+          findByIndex: findByIndexMock,
+        },
+      });
+
+      await dataClient.plainIndexSync();
+
+      expect(mockLog).toHaveBeenCalledWith(
+        'debug',
+        expect.stringContaining('No monitoring index sources found. Skipping sync.')
+      );
+    });
+
+    it('skips sources without indexPattern', async () => {
+      Object.defineProperty(dataClient, 'monitoringIndexSourceClient', {
+        value: {
+          findByIndex: jest.fn().mockResolvedValue([
+            { name: 'no-index', indexPattern: undefined },
+            { name: 'with-index', indexPattern: 'foo' },
+          ]),
+          init: jest.fn().mockResolvedValue({ status: 'success' }),
+          update: jest.fn(),
+        },
+      });
+
+      dataClient.syncUsernamesFromIndex = jest.fn().mockResolvedValue(['user1']);
+      Object.defineProperty(dataClient, 'findStaleUsersForIndex', {
+        value: jest.fn().mockResolvedValue([]),
+      });
+      await dataClient.plainIndexSync();
+      // Should only be called for the source with indexPattern
+      expect(dataClient.syncUsernamesFromIndex).toHaveBeenCalledTimes(1);
+      expect(dataClient.syncUsernamesFromIndex).toHaveBeenCalledWith({
+        indexName: 'foo',
+        kuery: undefined,
+      });
+    });
+
+    it('should retrieve all usernames from index and perform bulk ops', async () => {
+      const mockHits = [
+        {
+          _source: { user: { name: 'frodo' } },
+          _id: '1',
+          sort: [1],
+        },
+        {
+          _source: { user: { name: 'samwise' } },
+          _id: '2',
+          sort: [2],
+        },
+      ];
+
+      const mockMonitoredUserHits = {
+        hits: {
+          hits: [
+            {
+              _source: { user: { name: 'frodo' } },
+              _id: '1',
+            },
+            {
+              _source: { user: { name: 'samwise' } },
+              _id: '2',
+            },
+          ],
+        },
+      };
+
+      dataClient.searchUsernamesInIndex = jest
+        .fn()
+        .mockResolvedValueOnce({ hits: { hits: mockHits } }) // first batch
+        .mockResolvedValueOnce({ hits: { hits: [] } }); // second batch = end
+
+      dataClient.getMonitoredUsers = jest.fn().mockResolvedValue(mockMonitoredUserHits);
+      dataClient.buildBulkOperationsForUsers = jest.fn().mockReturnValue([{ index: { _id: '1' } }]);
+      dataClient.getIndex = jest.fn().mockReturnValue('test-index');
+
+      const usernames = await dataClient.syncUsernamesFromIndex({
+        indexName: 'test-index',
+      });
+
+      expect(usernames).toEqual(['frodo', 'samwise']);
+      expect(dataClient.searchUsernamesInIndex).toHaveBeenCalledTimes(2);
+      expect(esClientMock.bulk).toHaveBeenCalled();
+      expect(dataClient.getMonitoredUsers).toHaveBeenCalledWith(['frodo', 'samwise']);
+      expect(dataClient.buildBulkOperationsForUsers).toHaveBeenCalled();
+    });
+  });
 });