[Entity Store] Implement EUID

.buildkite/scout_ci_config.yml

@@ -21,6 +21,7 @@ plugins:
     - workflows_extensions
     - transform
     - fleet
+    - entity_store
   disabled:
 
 packages:

x-pack/solutions/security/plugins/entity_store/moon.yml

@@ -36,6 +36,7 @@ dependsOn:
   - '@kbn/es-types'
   - '@kbn/security-plugin-types-server'
   - '@kbn/encrypted-saved-objects-plugin'
+  - '@kbn/scout-security'
   - '@kbn/core-saved-objects-server'
 tags:
   - plugin
@@ -50,6 +51,7 @@ fileGroups:
     - public/**/*.ts
     - public/**/*.tsx
     - server/**/*.ts
+    - test/**/*.ts
     - '!target/**/*'
 tasks:
   jest:

x-pack/solutions/security/plugins/entity_store/server/domain/assets/component_templates.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
+import type { MappingProperty, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import type { EntityDefinition } from '../definitions/entity_schema';
 import { ENTITY_BASE_PREFIX } from '../constants';
 
@@ -16,6 +16,8 @@ const BASE_ENTITY_INDEX_MAPPING = {
   'event.ingested': { type: 'date' },
   labels: { type: 'object' },
   tags: { type: 'keyword', ignore_above: 1024 },
+  'entity.id': { type: 'keyword' },
+  'entity.EngineMetadata.Type': { type: 'keyword' },
 
   // 'asset.criticality': { type: 'keyword' },
   // 'entity.name': { type: 'keyword' },
@@ -35,7 +37,7 @@ export const getEntityDefinitionComponentTemplate = (definition: EntityDefinitio
 const getIndexMappings = (definition: EntityDefinition): MappingTypeMapping => ({
   properties: {
     ...BASE_ENTITY_INDEX_MAPPING,
-    ...Object.fromEntries(definition.identityFields.map((c) => [c.field, c.mapping])),
+    ...getIdentityFieldMapping(definition),
     ...Object.fromEntries(
       definition.fields
         .filter(({ mapping }) => mapping)
@@ -57,7 +59,7 @@ export const getUpdatesEntityDefinitionComponentTemplate = (definition: EntityDe
 const getUpdatesIndexMappings = (definition: EntityDefinition): MappingTypeMapping => ({
   properties: {
     ...BASE_ENTITY_INDEX_MAPPING,
-    ...Object.fromEntries(definition.identityFields.map((c) => [c.field, c.mapping])),
+    ...getIdentityFieldMapping(definition),
     ...Object.fromEntries(
       definition.fields
         .filter(({ mapping }) => mapping)
@@ -66,3 +68,11 @@ const getUpdatesIndexMappings = (definition: EntityDefinition): MappingTypeMappi
     ),
   },
 });
+function getIdentityFieldMapping({
+  identityField,
+}: EntityDefinition): Record<string, MappingProperty> {
+  if (!identityField.calculated) {
+    return { [identityField.field]: identityField.mapping };
+  }
+  return { [identityField.defaultIdField]: identityField.defaultIdFieldMapping };
+}

x-pack/solutions/security/plugins/entity_store/server/domain/definitions/entity_schema.ts

@@ -28,23 +28,45 @@ const fieldSchema = z.object({
   retention: retentionOperationSchema,
 });
 
+const calculatedIdentityFieldLogic = z.object({
+  // The field that will be used as the default id field.
+  defaultIdField: z.string(),
+  defaultIdFieldMapping: mappingSchema,
+
+  // Filter to be applied before evaluating the evaluation logic
+  requiresOneOfFields: z.array(z.string()),
+
+  // Evaluation logic to be used to generate the identity field.
+  // Here, the default id (e.g. `host.entity.id`) should not be mentioned,
+  // because this is part of the main query building logic.
+  // The ids that were generated using the esqlEvaluation will also be prepended
+  // with the type (e.g. `host:`). The fields found on the default id won't be prepended.
+  esqlEvaluation: z.string(),
+
+  calculated: z.literal(true),
+});
+
 const identityFieldSchema = z.object({
   field: z.string(),
   mapping: mappingSchema,
+  calculated: z.literal(false),
 });
 
 export const entitySchema = z.object({
   id: z.string(),
   name: z.string(),
-  type: z.string(),
+  type: EntityType,
   filter: z.string().optional(),
+  entityTypeFallback: z.string().optional(),
   fields: z.array(fieldSchema),
-  identityFields: z.array(identityFieldSchema),
+  identityField: z.union([identityFieldSchema, calculatedIdentityFieldLogic]),
   indexPatterns: z.array(z.string()),
 });
 
 export type EntityField = z.infer<typeof fieldSchema>; // entities fields
-export type EntityIdentityField = z.infer<typeof identityFieldSchema>; // entities identity field
+export type EntityCalculatedIdentityLogic = z.infer<typeof calculatedIdentityFieldLogic>; // logic to generate identity field
+export type EntityIdentityField = z.infer<typeof identityFieldSchema>; // field to use as identity field
+export type EntityIdentity = EntityIdentityField | EntityCalculatedIdentityLogic; // instructions to use or generate identity field
 export type EntityDefinition = z.infer<typeof entitySchema>; // entity with id generated in runtime
 export type EntityDefinitionWithoutId = Omit<EntityDefinition, 'id'>;
 export type ManagedEntityDefinition = EntityDefinition & { type: EntityType }; // entity with a known 'type'

x-pack/solutions/security/plugins/entity_store/server/domain/definitions/generic.ts

@@ -14,7 +14,12 @@ export const GENERIC_IDENTITY_FIELD = 'entity.id';
 export const genericEntityDefinition: EntityDefinitionWithoutId = {
   type: 'generic',
   name: `Security 'generic' Entity Store Definition`,
-  identityFields: [{ field: GENERIC_IDENTITY_FIELD, mapping: { type: 'keyword' } }],
+  // Generic doesn't have type prefix on the id
+  identityField: {
+    calculated: false,
+    field: GENERIC_IDENTITY_FIELD,
+    mapping: { type: 'keyword' },
+  },
   indexPatterns: [],
   fields: [
     ...getEntityFieldsDescriptions(),

x-pack/solutions/security/plugins/entity_store/server/domain/definitions/host.ts

@@ -8,17 +8,49 @@
 import { collectValues as collect } from './field_retention_operations';
 import type { EntityDefinitionWithoutId } from './entity_schema';
 import { getCommonFieldDescriptions, getEntityFieldsDescriptions } from './common_fields';
-
-export const HOST_IDENTITY_FIELD = 'host.name';
+import { esqlIsNotNullOrEmpty } from '../logs_extraction/esql_strings';
 
 // Mostly copied from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/host.ts
 
 export const hostEntityDefinition: EntityDefinitionWithoutId = {
   type: 'host',
   name: `Security 'host' Entity Store Definition`,
-  identityFields: [{ field: HOST_IDENTITY_FIELD, mapping: { type: 'keyword' } }],
+  identityField: {
+    calculated: true,
+    defaultIdField: 'host.entity.id',
+    defaultIdFieldMapping: { type: 'keyword' },
+    requiresOneOfFields: ['host.id', 'host.name', 'host.hostname'],
+
+    /*
+      Implements the following rank
+      1. host.entity.id           --> implemented as the default id field
+      2. host.id
+      3. host.name . host.domain
+      4. host.hostname . host.domain
+      5. host.name
+      6. host.hostname
+    */
+    esqlEvaluation: `COALESCE(
+                CASE(${esqlIsNotNullOrEmpty('host.id')}, host.id, NULL),
+                CASE(${esqlIsNotNullOrEmpty('host.domain')},
+                  CASE(
+                    ${esqlIsNotNullOrEmpty('host.name')}, CONCAT(host.name, ".", host.domain),
+                    ${esqlIsNotNullOrEmpty(
+                      'host.hostname'
+                    )}, CONCAT(host.hostname, ".", host.domain),
+                    NULL
+                  ),
+                  NULL
+                ),
+                CASE(${esqlIsNotNullOrEmpty('host.name')}, host.name, NULL),
+                CASE(${esqlIsNotNullOrEmpty('host.hostname')}, host.hostname, NULL),
+                NULL
+              )`,
+  },
+  entityTypeFallback: 'Host',
   indexPatterns: [],
   fields: [
+    collect({ source: 'host.name' }),
     collect({ source: 'host.domain' }),
     collect({ source: 'host.hostname' }),
     collect({ source: 'host.id' }),

x-pack/solutions/security/plugins/entity_store/server/domain/definitions/service.ts

@@ -5,18 +5,31 @@
  * 2.0.
  */
 
+import { esqlIsNotNullOrEmpty } from '../logs_extraction/esql_strings';
 import { getCommonFieldDescriptions, getEntityFieldsDescriptions } from './common_fields';
 import type { EntityDefinitionWithoutId } from './entity_schema';
 import { collectValues as collect, newestValue } from './field_retention_operations';
 
-export const SERVICE_IDENTITY_FIELD = 'service.name';
-
 export const serviceEntityDefinition: EntityDefinitionWithoutId = {
   type: 'service',
   name: `Security 'service' Entity Store Definition`,
-  identityFields: [{ field: SERVICE_IDENTITY_FIELD, mapping: { type: 'keyword' } }],
+  identityField: {
+    calculated: true,
+    defaultIdField: 'service.entity.id',
+    defaultIdFieldMapping: { type: 'keyword' },
+    requiresOneOfFields: ['service.name'],
+
+    /*
+      Implements the following rank
+      1. service.entity.id    --> implemented as the default id field
+      2. service.name
+    */
+    esqlEvaluation: `CASE(${esqlIsNotNullOrEmpty('service.name')}, service.name, NULL)`,
+  },
   indexPatterns: [],
+  entityTypeFallback: 'Service',
   fields: [
+    collect({ source: 'service.name' }),
     collect({ source: 'service.address' }),
     collect({ source: 'service.environment' }),
     collect({ source: 'service.ephemeral_id' }),

x-pack/solutions/security/plugins/entity_store/server/domain/definitions/user.ts

@@ -5,28 +5,60 @@
  * 2.0.
  */
 
+import { esqlIsNotNullOrEmpty } from '../logs_extraction/esql_strings';
 import { getCommonFieldDescriptions, getEntityFieldsDescriptions } from './common_fields';
 import type { EntityDefinitionWithoutId } from './entity_schema';
 import { collectValues as collect } from './field_retention_operations';
 
-export const USER_IDENTITY_FIELD = 'user.name';
-
 export const userEntityDefinition: EntityDefinitionWithoutId = {
   type: 'user',
   name: `Security 'user' Entity Store Definition`,
-  identityFields: [
-    {
-      field: USER_IDENTITY_FIELD,
-      mapping: {
-        type: 'keyword',
-        fields: { text: { type: 'match_only_text' } },
-      },
-    },
-  ],
+  identityField: {
+    calculated: true,
+    defaultIdField: 'user.entity.id',
+    defaultIdFieldMapping: { type: 'keyword' },
+    requiresOneOfFields: ['user.id', 'user.name', 'user.email'],
+
+    /*
+      Implements the following rank
+      1. user.entity.id           --> implemented as the default id field
+      2. user.name @ host.entity.id
+      3. user.name @ host.id
+      4. user.name @ host.name
+      5. user.id
+      6. user.email
+      7. user.name @ user.domain
+      8. user.name
+    */
+    esqlEvaluation: `COALESCE(
+                CASE(${esqlIsNotNullOrEmpty('user.name')},
+                  CASE(
+                    ${esqlIsNotNullOrEmpty(
+                      'host.entity.id'
+                    )}, CONCAT(user.name, "@", host.entity.id),
+                    ${esqlIsNotNullOrEmpty('host.id')}, CONCAT(user.name, "@", host.id),
+                    ${esqlIsNotNullOrEmpty('host.name')}, CONCAT(user.name, "@", host.name),
+                    NULL
+                  ),
+                  NULL
+                ),
+                CASE(${esqlIsNotNullOrEmpty('user.id')}, user.id, NULL),
+                CASE(${esqlIsNotNullOrEmpty('user.email')}, user.email, NULL),
+                CASE(
+                  ${esqlIsNotNullOrEmpty('user.name')} AND ${esqlIsNotNullOrEmpty('user.domain')}, 
+                  CONCAT(user.name, ".", user.domain),
+                  NULL
+                ),  
+                CASE(${esqlIsNotNullOrEmpty('user.name')}, user.name, NULL),
+                NULL
+              )`,
+  },
+  entityTypeFallback: 'Identity',
   indexPatterns: [],
   fields: [
     collect({ source: 'user.domain' }),
     collect({ source: 'user.email' }),
+    collect({ source: 'user.name' }),
     collect({
       source: 'user.full_name',
       mapping: {
@@ -44,6 +76,11 @@ export const userEntityDefinition: EntityDefinitionWithoutId = {
     ...getCommonFieldDescriptions('user'),
     ...getEntityFieldsDescriptions('user'),
 
+    // Used to populate the identity field
+    collect({ source: 'host.entity.id' }),
+    collect({ source: 'host.id' }),
+    collect({ source: 'host.name' }),
+
     collect({
       source: `user.entity.relationships.Accesses_frequently`,
       destination: 'entity.relationships.Accesses_frequently',