Add API Key Management and Enable Index Sync via Scheduled Task

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -913,6 +913,9 @@
   "privilege-monitoring-status": [
     "status"
   ],
+  "privmon-api-key": [
+    "apiKey"
+  ],
   "product-doc-install-status": [
     "index_name",
     "inference_id",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -3036,6 +3036,11 @@
       }
     }
   },
+  "privmon-api-key": {
+    "dynamic": false,
+    "properties": {
+    }
+  },
   "product-doc-install-status": {
     "dynamic": false,
     "properties": {

src/core/packages/saved-objects/server-internal/src/object_types/index.ts

@@ -11,4 +11,4 @@ export { registerCoreObjectTypes } from './registration';
 
 // set minimum number of registered saved objects to ensure no object types are removed after 8.8
 // declared in internal implementation explicitly to prevent unintended changes.
-export const SAVED_OBJECT_TYPES_COUNT = 136 as const;
+export const SAVED_OBJECT_TYPES_COUNT = 137 as const;

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -154,6 +154,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "osquery-saved-query": "a8ef11610473e3d1b51a8fdacb2799d8a610818e",
         "policy-settings-protection-updates-note": "c05c4c33a5e5bd1fa153991f300d040ac5d6f38d",
         "privilege-monitoring-status": "4daec76df427409bcd64250f5c23f5ab86c8bac3",
+        "privmon-api-key": "c06b1614786ce7271087378b47d465c956ab1537",
         "product-doc-install-status": "f94e3e5ad2cc933df918f2cd159044c626e01011",
         "query": "1966ccce8e9853018111fb8a1dee500228731d9e",
         "risk-engine-configuration": "533a0a3f2dbef1c95129146ec4d5714de305be1a",

src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts

@@ -58,6 +58,7 @@ const previouslyRegisteredTypes = [
   'enterprise_search_telemetry',
   'entity-analytics-monitoring-entity-source',
   'entity-definition',
+  'privmon-api-key',
   'entity-discovery-api-key',
   'epm-packages',
   'epm-packages-assets',

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts

@@ -191,6 +191,7 @@ const createSecuritySolutionRequestContextMock = (
     getAuditLogger: jest.fn(() => mockAuditLogger),
     getDataViewsService: jest.fn(),
     getEntityStoreApiKeyManager: jest.fn(),
+    getPrivilegedUserMonitoringApiKeyManager: jest.fn(),
     getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient),
     getPrivilegeMonitoringDataClient: jest.fn(() => clients.privilegeMonitorDataClient),
     getPadPackageInstallationClient: jest.fn(() => clients.padPackageInstallationClient),

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/auth/api_key.ts

@@ -13,9 +13,9 @@ import type { EncryptedSavedObjectsPluginStart } from '@kbn/encrypted-saved-obje
 import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request';
 
 import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
-import type { SavedObjectsType } from '@kbn/core/server';
-import { getPrivmonEncryptedSavedObjectId } from './saved_object';
-import { privilegeMonitoringRuntimePrivileges } from './privileges';
+
+import { PrivilegeMonitoringApiKeyType, getPrivmonEncryptedSavedObjectId } from './saved_object';
+import { monitoringEntitySourceType } from '../saved_objects';
 
 export interface ApiKeyManager {
   generate: () => Promise<void>;
@@ -32,9 +32,8 @@ export interface ApiKeyManagerDependencies {
 
 export const getApiKeyManager = (deps: ApiKeyManagerDependencies) => {
   return {
-    generate: generate(deps),
-    getApiKey: getApiKey(deps),
-    getClientFromApiKey: getClientFromApiKey(deps),
+    generate: () => generate(deps),
+    getClient: () => getClient(deps),
     getRequestFromApiKey,
   };
 };
@@ -51,7 +50,7 @@ const generate = async (deps: ApiKeyManagerDependencies) => {
     const apiKey = await generateAPIKey(request, deps);
 
     const soClient = core.savedObjects.getScopedClient(request, {
-      includedHiddenTypes: [PrivilegeMonitoringApiKeyType.name],
+      includedHiddenTypes: [PrivilegeMonitoringApiKeyType.name, monitoringEntitySourceType.name],
     });
 
     await soClient.create(PrivilegeMonitoringApiKeyType.name, apiKey, {
@@ -92,31 +91,34 @@ const getRequestFromApiKey = async (apiKey: PrivilegeMonitoringAPIKey) => {
     api_key: apiKey.apiKey,
   });
 };
-const getClientFromApiKey =
-  (deps: ApiKeyManagerDependencies) => async (apiKey: PrivilegeMonitoringAPIKey) => {
-    const fakeRequest = getFakeKibanaRequest({
-      id: apiKey.id,
-      api_key: apiKey.apiKey,
-    });
-    const clusterClient = deps.core.elasticsearch.client.asScoped(fakeRequest);
-    const soClient = deps.core.savedObjects.getScopedClient(fakeRequest, {
-      includedHiddenTypes: [PrivilegeMonitoringApiKeyType.name],
-    });
-    return {
-      clusterClient,
-      soClient,
-    };
+const getClient = async (deps: ApiKeyManagerDependencies) => {
+  const apiKey = await getApiKey(deps);
+  if (!apiKey) return undefined;
+  const fakeRequest = getFakeKibanaRequest({
+    id: apiKey.id,
+    api_key: apiKey.apiKey,
+  });
+  const clusterClient = deps.core.elasticsearch.client.asScoped(fakeRequest);
+  return {
+    clusterClient,
   };
+};
 
-export const generateAPIKey = async (
+const generateAPIKey = async (
   req: KibanaRequest,
   deps: ApiKeyManagerDependencies
 ): Promise<PrivilegeMonitoringAPIKey | undefined> => {
+  deps.logger.info('Generating Privmon API key');
   const apiKey = await deps.security.authc.apiKeys.grantAsInternalUser(req, {
     name: 'Privilege Monitoring API key',
-    role_descriptors: {
-      privmon_admin: privilegeMonitoringRuntimePrivileges([]),
-    },
+    /**
+     * Intentionally passing empty array - generates a snapshot (empty object).
+     * Due to not knowing what index pattern changes customer may make to index list.
+     *
+     * - If the customer later adds new indices they *do* have access to, the key will still function.
+     * - If they add indices they *don't* have access to, they will need to reinitialize once their access is elevated.
+     */
+    role_descriptors: {},
     metadata: {
       description: 'API key used to manage the resources in the privilege monitoring engine',
     },
@@ -131,24 +133,6 @@ export const generateAPIKey = async (
   }
 };
 
-export const SO_PRIVILEGE_MONITORING_API_KEY_TYPE = 'privmon-api-key';
-
-export const PrivilegeMonitoringApiKeyType: SavedObjectsType = {
-  name: SO_PRIVILEGE_MONITORING_API_KEY_TYPE,
-  hidden: true,
-  namespaceType: 'multiple-isolated',
-  mappings: {
-    dynamic: false,
-    properties: {
-      apiKey: { type: 'binary' },
-    },
-  },
-  management: {
-    importableAndExportable: false,
-    displayName: 'Privilege Monitoring API key',
-  },
-};
-
 export interface PrivilegeMonitoringAPIKey {
   id: string;
   name: string;

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/auth/saved_object.ts

@@ -4,10 +4,34 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import type { SavedObjectsType } from '@kbn/core/server';
+import type { EncryptedSavedObjectTypeRegistration } from '@kbn/encrypted-saved-objects-plugin/server';
 import { v5 as uuidv5 } from 'uuid';
 
-const PRIVMON_API_KEY_SO_ID = '19540C97-E35C-485B-8566-FB86EC8455E4';
+const PRIVMON_API_KEY_SO_ID = 'd2ee7992-cb4d-473a-8f1a-44ba187d4ac9';
 
 export const getPrivmonEncryptedSavedObjectId = (space: string) => {
   return uuidv5(space, PRIVMON_API_KEY_SO_ID);
 };
+
+export const SO_PRIVILEGE_MONITORING_API_KEY_TYPE = 'privmon-api-key';
+
+export const PrivilegeMonitoringApiKeyType: SavedObjectsType = {
+  name: SO_PRIVILEGE_MONITORING_API_KEY_TYPE,
+  hidden: true,
+  namespaceType: 'multiple-isolated',
+  mappings: {
+    dynamic: false,
+    properties: {},
+  },
+  management: {
+    importableAndExportable: false,
+    displayName: 'Privilege Monitoring API key',
+  },
+};
+
+export const PrivilegeMonitoringApiKeyEncryptionParams: EncryptedSavedObjectTypeRegistration = {
+  type: SO_PRIVILEGE_MONITORING_API_KEY_TYPE,
+  attributesToEncrypt: new Set(['apiKey']),
+  attributesToIncludeInAAD: new Set(['id', 'name']),
+};

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/privilege_monitoring_data_client.ts

@@ -164,8 +164,6 @@ export class PrivilegeMonitoringDataClient {
       this.opts.telemetry?.reportEvent(PRIVMON_ENGINE_INITIALIZATION_EVENT.eventType, {
         duration,
       });
-      // sync all index users from monitoring sources
-      await this.plainIndexSync();
     } catch (e) {
       this.log('error', `Error initializing privilege monitoring engine: ${e}`);
       this.audit(
@@ -539,6 +537,8 @@ export class PrivilegeMonitoringDataClient {
         existingUserId: existingUserMap.get(username),
       }));
 
+      if (usersToWrite.length === 0) return batchUsernames;
+
       const ops = this.buildBulkOperationsForUsers(usersToWrite, this.getIndex());
       this.log('debug', `Executing bulk operations for ${usersToWrite.length} users`);
       try {

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/saved_objects/monitoring_entity_source.ts

@@ -80,9 +80,11 @@ export class MonitoringEntitySourceDescriptorClient {
   }
 
   async find() {
-    return this.dependencies.soClient.find<MonitoringEntitySourceDescriptor>({
+    const scopedSoClient = this.dependencies.soClient.asScopedToNamespace(
+      this.dependencies.namespace
+    );
+    return scopedSoClient.find<MonitoringEntitySourceDescriptor>({
       type: monitoringEntitySourceTypeName,
-      namespaces: [this.dependencies.namespace],
     });
   }
 

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/privilege_monitoring/tasks/privilege_monitoring_task.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { Logger, AnalyticsServiceSetup } from '@kbn/core/server';
+import type { Logger, AnalyticsServiceSetup, AuditLogger } from '@kbn/core/server';
 import type {
   ConcreteTaskInstance,
   TaskManagerSetupContract,
@@ -23,11 +23,15 @@ import {
   stateSchemaByVersion,
   type LatestTaskStateSchema as PrivilegeMonitoringTaskState,
 } from './state';
+import { getApiKeyManager } from '../auth/api_key';
+import { PrivilegeMonitoringDataClient } from '../privilege_monitoring_data_client';
+import { buildScopedInternalSavedObjectsClientUnsafe } from '../../risk_score/tasks/helpers';
 
 interface RegisterParams {
   getStartServices: EntityAnalyticsRoutesDeps['getStartServices'];
   logger: Logger;
   telemetry: AnalyticsServiceSetup;
+  auditLogger?: AuditLogger;
   taskManager: TaskManagerSetupContract | undefined;
   experimentalFeatures: ExperimentalFeatures;
   kibanaVersion: string;
@@ -39,6 +43,9 @@ interface RunParams {
   telemetry: AnalyticsServiceSetup;
   experimentalFeatures: ExperimentalFeatures;
   taskInstance: ConcreteTaskInstance;
+  getPrivilegedUserMonitoringDataClient: (
+    namespace: string
+  ) => Promise<undefined | PrivilegeMonitoringDataClient>;
 }
 
 interface StartParams {
@@ -54,6 +61,7 @@ const getTaskId = (namespace: string): string => `${TYPE}:${namespace}:${VERSION
 export const registerPrivilegeMonitoringTask = ({
   getStartServices,
   logger,
+  auditLogger,
   telemetry,
   taskManager,
   kibanaVersion,
@@ -65,6 +73,39 @@ export const registerPrivilegeMonitoringTask = ({
     );
     return;
   }
+  const getPrivilegedUserMonitoringDataClient = async (namespace: string) => {
+    const [core, { taskManager: taskManagerStart, security, encryptedSavedObjects }] =
+      await getStartServices();
+
+    const apiKeyManager = getApiKeyManager({
+      core,
+      logger,
+      security,
+      encryptedSavedObjects,
+      namespace,
+    });
+
+    const client = await apiKeyManager.getClient();
+
+    if (!client) {
+      logger.error('[Privilege Monitoring] Unable to create Elasticsearch client from API key.');
+      return undefined;
+    }
+
+    const soClient = buildScopedInternalSavedObjectsClientUnsafe({ coreStart: core, namespace });
+
+    return new PrivilegeMonitoringDataClient({
+      logger,
+      clusterClient: client.clusterClient,
+      namespace,
+      soClient,
+      taskManager: taskManagerStart,
+      auditLogger,
+      kibanaVersion,
+      telemetry,
+      apiKeyManager,
+    });
+  };
 
   taskManager.registerTaskDefinitions({
     [getTaskName()]: {
@@ -75,6 +116,7 @@ export const registerPrivilegeMonitoringTask = ({
         logger,
         telemetry,
         experimentalFeatures,
+        getPrivilegedUserMonitoringDataClient,
       }),
     },
   });
@@ -85,6 +127,9 @@ const createPrivilegeMonitoringTaskRunnerFactory =
     logger: Logger;
     telemetry: AnalyticsServiceSetup;
     experimentalFeatures: ExperimentalFeatures;
+    getPrivilegedUserMonitoringDataClient: (
+      namespace: string
+    ) => Promise<undefined | PrivilegeMonitoringDataClient>;
   }): TaskRunCreatorFunction =>
   ({ taskInstance }) => {
     let cancelled = false;
@@ -97,6 +142,7 @@ const createPrivilegeMonitoringTaskRunnerFactory =
           telemetry: deps.telemetry,
           taskInstance,
           experimentalFeatures: deps.experimentalFeatures,
+          getPrivilegedUserMonitoringDataClient: deps.getPrivilegedUserMonitoringDataClient,
         }),
       cancel: async () => {
         cancelled = true;
@@ -110,6 +156,7 @@ const runPrivilegeMonitoringTask = async ({
   telemetry,
   taskInstance,
   experimentalFeatures,
+  getPrivilegedUserMonitoringDataClient,
 }: RunParams): Promise<{
   state: PrivilegeMonitoringTaskState;
 }> => {
@@ -127,9 +174,16 @@ const runPrivilegeMonitoringTask = async ({
 
   try {
     logger.info('[Privilege Monitoring] Running privilege monitoring task');
+    const dataClient = await getPrivilegedUserMonitoringDataClient(state.namespace);
+    if (!dataClient) {
+      logger.error('[Privilege Monitoring] error creating data client.');
+      throw Error('No data client was found');
+    }
+    await dataClient.plainIndexSync();
   } catch (e) {
-    logger.error('[Privilege Monitoring] Error running privilege monitoring task', e);
+    logger.error(`[Privilege Monitoring] Error running privilege monitoring task: ${e.message}`);
   }
+  logger.info('[Privilege Monitoring] Finished running privilege monitoring task');
   return { state: updatedState };
 };
 

x-pack/solutions/security/plugins/security_solution/server/plugin.ts

@@ -43,7 +43,7 @@ import { initRoutes } from './routes';
 import { registerLimitedConcurrencyRoutes } from './routes/limited_concurrency';
 import { ManifestConstants, ManifestTask } from './endpoint/lib/artifacts';
 import { CheckMetadataTransformsTask } from './endpoint/lib/metadata';
-import { initSavedObjects } from './saved_objects';
+import { initEncryptedSavedObjects, initSavedObjects } from './saved_objects';
 import { AppClientFactory } from './client';
 import type { ConfigType } from './config';
 import { createConfig } from './config';
@@ -222,6 +222,10 @@ export class Plugin implements ISecuritySolutionPlugin {
     const experimentalFeatures = config.experimentalFeatures;
 
     initSavedObjects(core.savedObjects);
+    initEncryptedSavedObjects({
+      encryptedSavedObjects: plugins.encryptedSavedObjects,
+      logger: this.logger,
+    });
 
     initUiSettings(core.uiSettings, experimentalFeatures, config.enableUiSettingsValidations);
     productFeaturesService.init(plugins.features);
@@ -283,6 +287,7 @@ export class Plugin implements ISecuritySolutionPlugin {
       getStartServices: core.getStartServices,
       taskManager: plugins.taskManager,
       logger: this.logger,
+      auditLogger: plugins.security?.audit.withoutRequest,
       telemetry: core.analytics,
       kibanaVersion: pluginContext.env.packageInfo.version,
       experimentalFeatures,