[SecuritySolution] Update Entity Store Dashboard to prompt for Service Entity Type

oas_docs/output/kibana.serverless.yaml

@@ -9721,6 +9721,10 @@ paths:
             schema:
               type: object
               properties:
+                entityTypes:
+                  items:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
+                  type: array
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.

oas_docs/output/kibana.yaml

@@ -11892,6 +11892,10 @@ paths:
             schema:
               type: object
               properties:
+                entityTypes:
+                  items:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
+                  type: array
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts

@@ -16,7 +16,7 @@
 
 import { z } from '@kbn/zod';
 
-import { IndexPattern, EngineDescriptor } from './common.gen';
+import { IndexPattern, EntityType, EngineDescriptor } from './common.gen';
 
 export type InitEntityStoreRequestBody = z.infer<typeof InitEntityStoreRequestBody>;
 export const InitEntityStoreRequestBody = z.object({
@@ -26,6 +26,7 @@ export const InitEntityStoreRequestBody = z.object({
   fieldHistoryLength: z.number().int().optional().default(10),
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
+  entityTypes: z.array(EntityType).optional(),
 });
 export type InitEntityStoreRequestBodyInput = z.input<typeof InitEntityStoreRequestBody>;
 

x-pack/solutions/security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml

@@ -27,6 +27,10 @@ paths:
                   $ref: './common.schema.yaml#/components/schemas/IndexPattern'
                 filter:
                   type: string
+                entityTypes:
+                  type: array
+                  items:
+                    $ref: './common.schema.yaml#/components/schemas/EntityType'
       responses:
         '200':
           description: Successful response

x-pack/solutions/security/plugins/security_solution/common/entity_analytics/asset_criticality/parse_asset_criticality_csv_row.test.ts

@@ -54,7 +54,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"invalid\\", expected to be one of: user, host"`
+      `"Invalid entity type \\"invalid\\", expected to be one of: user, host, service"`
     );
   });
 
@@ -68,7 +68,7 @@ describe('parseAssetCriticalityCsvRow', () => {
 
     // @ts-ignore result can now only be InvalidRecord
     expect(result.error).toMatchInlineSnapshot(
-      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host"`
+      `"Invalid entity type \\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...\\", expected to be one of: user, host, service"`
     );
   });
 

x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts

@@ -234,7 +234,7 @@ export const allowedExperimentalValues = Object.freeze({
   /**
    * Enables the Service Entity Store. The Entity Store feature will install the service engine by default.
    */
-  serviceEntityStoreEnabled: false,
+  serviceEntityStoreEnabled: true,
 
   /**
    * Enables the siem migrations feature

x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -307,6 +307,10 @@ paths:
             schema:
               type: object
               properties:
+                entityTypes:
+                  items:
+                    $ref: '#/components/schemas/EntityType'
+                  type: array
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.

x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -307,6 +307,10 @@ paths:
             schema:
               type: object
               properties:
+                entityTypes:
+                  items:
+                    $ref: '#/components/schemas/EntityType'
+                  type: array
                 fieldHistoryLength:
                   default: 10
                   description: The number of historical values to keep for each field.

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/api/entity_store.ts

@@ -7,7 +7,7 @@
 import { useMemo } from 'react';
 import type { GetEntityStoreStatusResponse } from '../../../common/api/entity_analytics/entity_store/status.gen';
 import type {
-  InitEntityStoreRequestBody,
+  InitEntityStoreRequestBodyInput,
   InitEntityStoreResponse,
 } from '../../../common/api/entity_analytics/entity_store/enable.gen';
 import type {
@@ -24,9 +24,7 @@ export const useEntityStoreRoutes = () => {
   const http = useKibana().services.http;
 
   return useMemo(() => {
-    const enableEntityStore = async (
-      options: InitEntityStoreRequestBody = { fieldHistoryLength: 10 }
-    ) => {
+    const enableEntityStore = async (options: InitEntityStoreRequestBodyInput = {}) => {
       return http.fetch<InitEntityStoreResponse>('/api/entity_store/enable', {
         method: 'POST',
         version: API_VERSIONS.public.v1,

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/components/dashboard_enablement_panel.tsx

@@ -16,6 +16,7 @@ import {
 } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { UseQueryResult } from '@tanstack/react-query';
+import { i18n } from '@kbn/i18n';
 import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
 import type {
   RiskEngineStatusResponse,
@@ -37,6 +38,7 @@ import {
 import type { Enablements } from './enablement_modal';
 import { EntityStoreEnablementModal } from './enablement_modal';
 import dashboardEnableImg from '../../../images/entity_store_dashboard.png';
+import { useStoreEntityTypes } from '../../../hooks/use_enabled_entity_types';
 
 interface EnableEntityStorePanelProps {
   state: {
@@ -48,6 +50,8 @@ interface EnableEntityStorePanelProps {
 export const EnablementPanel: React.FC<EnableEntityStorePanelProps> = ({ state }) => {
   const riskEngineStatus = state.riskEngine.data?.risk_engine_status;
   const entityStoreStatus = state.entityStore.data?.status;
+  const engines = state.entityStore.data?.engines;
+  const enabledEntityTypes = useStoreEntityTypes();
 
   const [modal, setModalState] = useState({ visible: false });
   const [riskEngineInitializing, setRiskEngineInitializing] = useState(false);
@@ -62,7 +66,7 @@ export const EnablementPanel: React.FC<EnableEntityStorePanelProps> = ({ state }
           onSuccess: () => {
             setRiskEngineInitializing(false);
             if (enable.entityStore) {
-              storeEnablement.mutate();
+              storeEnablement.mutate({});
             }
           },
         };
@@ -73,29 +77,36 @@ export const EnablementPanel: React.FC<EnableEntityStorePanelProps> = ({ state }
       }
 
       if (enable.entityStore) {
-        storeEnablement.mutate();
+        storeEnablement.mutate({});
         setModalState({ visible: false });
       }
     },
     [storeEnablement, initRiskEngine]
   );
 
+  const installedTypes = engines?.map((engine) => engine.type);
+  const uninstalledTypes = enabledEntityTypes.filter(
+    (type) => !(installedTypes || []).includes(type)
+  );
+
+  const enableUninstalledEntityStore = () => {
+    storeEnablement.mutate({ entityTypes: uninstalledTypes });
+  };
+
   if (storeEnablement.error) {
     return (
-      <>
-        <EuiCallOut
-          title={
-            <FormattedMessage
-              id="xpack.securitySolution.entityAnalytics.entityStore.enablement.mutation.errorTitle"
-              defaultMessage={'There was a problem initializing the entity store'}
-            />
-          }
-          color="danger"
-          iconType="error"
-        >
-          <p>{storeEnablement.error.body.message}</p>
-        </EuiCallOut>
-      </>
+      <EuiCallOut
+        title={
+          <FormattedMessage
+            id="xpack.securitySolution.entityAnalytics.entityStore.enablement.mutation.errorTitle"
+            defaultMessage={'There was a problem initializing the entity store'}
+          />
+        }
+        color="danger"
+        iconType="error"
+      >
+        <p>{storeEnablement.error.body.message}</p>
+      </EuiCallOut>
     );
   }
 
@@ -129,6 +140,51 @@ export const EnablementPanel: React.FC<EnableEntityStorePanelProps> = ({ state }
     );
   }
 
+  if (entityStoreStatus === 'running' && uninstalledTypes.length > 0) {
+    const title = i18n.translate(
+      'xpack.securitySolution.entityAnalytics.entityStore.enablement.moreEntityTypesTitle',
+      {
+        defaultMessage: 'More entity types available',
+      }
+    );
+
+    return (
+      <EuiEmptyPrompt
+        css={{ minWidth: '100%' }}
+        hasBorder
+        layout="horizontal"
+        actions={
+          <EuiToolTip content={title}>
+            <EuiButton
+              color="primary"
+              fill
+              onClick={enableUninstalledEntityStore}
+              data-test-subj={`entityStoreEnablementButton`}
+            >
+              <FormattedMessage
+                id="xpack.securitySolution.entityAnalytics.entityStore.enablement.enableButton"
+                defaultMessage="Enable"
+              />
+            </EuiButton>
+          </EuiToolTip>
+        }
+        icon={<EuiImage size="l" hasShadow src={dashboardEnableImg} alt={title} />}
+        data-test-subj="entityStoreEnablementPanel"
+        title={<h2>{title}</h2>}
+        body={
+          <p>
+            <FormattedMessage
+              id="xpack.securitySolution.entityAnalytics.entityStore.enablement.moreEntityTypes"
+              defaultMessage={
+                'Enable missing types in the entity store to capture even more entities observed in events'
+              }
+            />
+          </p>
+        }
+      />
+    );
+  }
+
   if (
     riskEngineStatus !== RiskEngineStatusEnum.NOT_INSTALLED &&
     (entityStoreStatus === 'running' || entityStoreStatus === 'stopped')

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entity_store.ts

@@ -10,7 +10,10 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 
 import type { IHttpFetchError } from '@kbn/core-http-browser';
 import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
-import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
+import type {
+  InitEntityStoreRequestBodyInput,
+  InitEntityStoreResponse,
+} from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { useKibana } from '../../../../common/lib/kibana/kibana_react';
 import type { EntityType } from '../../../../../common/api/entity_analytics';
 import {
@@ -47,18 +50,28 @@ export const useEntityStoreStatus = (opts: Options = {}) => {
 };
 
 export const ENABLE_STORE_STATUS_KEY = ['POST', 'ENABLE_ENTITY_STORE'];
-export const useEnableEntityStoreMutation = (options?: UseMutationOptions<{}>) => {
+export const useEnableEntityStoreMutation = (
+  options?: UseMutationOptions<
+    InitEntityStoreResponse,
+    ResponseError,
+    InitEntityStoreRequestBodyInput
+  >
+) => {
   const { telemetry } = useKibana().services;
   const queryClient = useQueryClient();
   const { enableEntityStore } = useEntityStoreRoutes();
 
-  return useMutation<InitEntityStoreResponse, ResponseError>(
-    () => {
+  return useMutation<
+    InitEntityStoreResponse,
+    ResponseError,
+    Partial<InitEntityStoreRequestBodyInput>
+  >(
+    (params) => {
       telemetry?.reportEvent(EntityEventTypes.EntityStoreEnablementToggleClicked, {
         timestamp: new Date().toISOString(),
         action: 'start',
       });
-      return enableEntityStore();
+      return enableEntityStore(params);
     },
     {
       mutationKey: ENABLE_STORE_STATUS_KEY,

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/pages/entity_store_management_page.tsx

@@ -96,7 +96,7 @@ export const EntityStoreManagementPage = () => {
     if (isEntityStoreEnabled(entityStoreStatus.data?.status)) {
       stopEntityEngineMutation.mutate();
     } else {
-      enableStoreMutation.mutate();
+      enableStoreMutation.mutate({});
     }
   }, [entityStoreStatus.data?.status, stopEntityEngineMutation, enableStoreMutation]);
 

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -19,6 +19,8 @@ import { mockGlobalState } from '../../../../public/common/mock';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import { convertToEntityManagerDefinition } from './entity_definitions/entity_manager_conversion';
 import { EntityType } from '../../../../common/search_strategy';
+import type { InitEntityEngineResponse } from '../../../../common/api/entity_analytics';
+import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 
 const definition: EntityDefinition = convertToEntityManagerDefinition(
   {
@@ -56,6 +58,7 @@ describe('EntityStoreDataClient', () => {
     appClient: {} as AppClient,
     config: {} as EntityStoreConfig,
     experimentalFeatures: mockGlobalState.app.enableExperimental,
+    taskManager: {} as TaskManagerStartContract,
   });
 
   const defaultSearchParams = {
@@ -338,4 +341,33 @@ describe('EntityStoreDataClient', () => {
       ]);
     });
   });
+
+  describe('enable entities', () => {
+    let spyInit: jest.SpyInstance;
+
+    beforeEach(() => {
+      jest.resetAllMocks();
+      spyInit = jest
+        .spyOn(dataClient, 'init')
+        .mockImplementation(() => Promise.resolve({} as InitEntityEngineResponse));
+    });
+
+    it('only enable engine for the given entityType', async () => {
+      await dataClient.enable({
+        entityTypes: [EntityType.host],
+        fieldHistoryLength: 1,
+      });
+
+      expect(spyInit).toHaveBeenCalledWith(EntityType.host, expect.anything(), expect.anything());
+    });
+
+    it('does not enable engine when the given entity type is disabled', async () => {
+      await dataClient.enable({
+        entityTypes: [EntityType.universal],
+        fieldHistoryLength: 1,
+      });
+
+      expect(spyInit).not.toHaveBeenCalled();
+    });
+  });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -203,7 +203,12 @@ export class EntityStoreDataClient {
   }
 
   public async enable(
-    { indexPattern = '', filter = '', fieldHistoryLength = 10 }: InitEntityStoreRequestBody,
+    {
+      indexPattern = '',
+      filter = '',
+      fieldHistoryLength = 10,
+      entityTypes,
+    }: InitEntityStoreRequestBody,
     { pipelineDebugMode = false }: { pipelineDebugMode?: boolean } = {}
   ): Promise<InitEntityStoreResponse> {
     if (!this.options.taskManager) {
@@ -215,7 +220,12 @@ export class EntityStoreDataClient {
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
 
     const { experimentalFeatures } = this.options;
-    const enginesTypes = getEnabledStoreEntityTypes(experimentalFeatures);
+    const enabledEntityTypes = getEnabledStoreEntityTypes(experimentalFeatures);
+
+    // When entityTypes param is defined it only enables the engines that are provided
+    const enginesTypes = entityTypes
+      ? (entityTypes as EntityType[]).filter((type) => enabledEntityTypes.includes(type))
+      : enabledEntityTypes;
 
     const promises = enginesTypes.map((entity) =>
       run(() =>

x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_risk_scores.test.ts

@@ -201,7 +201,7 @@ describe('calculateRiskScores()', () => {
       expect(response).toHaveProperty('scores');
       expect(response.scores.host).toHaveLength(2);
       expect(response.scores.user).toHaveLength(2);
-      expect(response.scores.service).toHaveLength(0);
+      expect(response.scores.service).toHaveLength(2);
     });
 
     it('calculates risk score for service when the experimental flag is enabled', async () => {