[9.4] Update entities schema to support integrations data (#262242)

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts

@@ -161,7 +161,6 @@ export const GRAPH_TARGET_ENTITY_FIELDS = [
   'service.target.entity.id',
   'entity.target.id',
 ] as const;
-
 /**
  * Raw source fields used to compute actor EUIDs in entity store v2.
  * These mirror the identity fields from Entity Store definitions.

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/fetch_entity_relationships_graph.test.ts

@@ -69,6 +69,7 @@ describe('fetchEntityRelationships', () => {
       // Verify query uses v2 index and LOOKUP JOIN
       expect(query).toContain(`FROM ${indexName}`);
       expect(query).toContain(`LOOKUP JOIN ${indexName} ON entity.id`);
+      expect(query).toContain('`entity.relationships.owns.ids`');
     });
 
     it('should return empty result when entities index is not in lookup mode', async () => {
@@ -151,11 +152,22 @@ describe('fetchEntityRelationships', () => {
           'entity.id': ['entity-1', 'entity-2', 'entity-3'],
         },
       });
-      // Verify it queries for entities that have these IDs in their relationships (all fields)
+      const ids = ['entity-1', 'entity-2', 'entity-3'];
+
+      // Relationship bags: match `entity.relationships.<leaf>.ids`; resolution uses resolved_to path
       ENTITY_RELATIONSHIP_FIELDS.forEach((field) => {
+        if (field === 'resolution.resolved_to') {
+          expect(filterArg.bool.should).toContainEqual({
+            terms: {
+              'entity.relationships.resolution.resolved_to': ids,
+            },
+          });
+          return;
+        }
+
         expect(filterArg.bool.should).toContainEqual({
           terms: {
-            [`entity.relationships.${field}`]: ['entity-1', 'entity-2', 'entity-3'],
+            [`entity.relationships.${field}.ids`]: ids,
           },
         });
       });
@@ -311,6 +323,7 @@ describe('fetchEntityRelationships', () => {
       const query = esqlCallArgs[0].query;
 
       // Verify doc data fields are generated
+      expect(query).toContain('_rel_targets_owns');
       expect(query).toContain('actorsDocData');
       expect(query).toContain('targetsDocData');
       expect(query).toContain('availableInEntityStore');

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/fetch_entity_relationships_graph.ts

@@ -27,6 +27,22 @@ interface BuildRelationshipsEsqlQueryParams {
   relationshipFields: readonly string[];
 }
 
+const RESOLUTION_RELATIONSHIP_FIELD = 'resolution.resolved_to' as const;
+
+/**
+ * ECS relationship leaves store canonical target EUIDs under `entity.relationships.<leaf>.ids`
+ * and raw dimensions under `entity.relationships.<leaf>.raw_identifiers.*` (dynamic bag).
+ * Resolution still uses `entity.relationships.resolution.resolved_to`.
+ */
+const buildRelationshipTargetsEval = (field: string): string => {
+  const col = `\`_rel_targets_${field}\``;
+  if (field === RESOLUTION_RELATIONSHIP_FIELD) {
+    return `${col} = COALESCE(\`entity.relationships.resolution.resolved_to\`, [""])`;
+  }
+
+  return `${col} = COALESCE(\`entity.relationships.${field}.ids\`, [""])`;
+};
+
 /**
  * Builds ES|QL query for fetching entity relationships from the generic entities index.
  * Uses FORK to expand each relationship field and aggregates results.
@@ -38,20 +54,16 @@ const buildRelationshipsEsqlQuery = ({
   indexName,
   relationshipFields,
 }: BuildRelationshipsEsqlQueryParams): string => {
-  // Build COALESCE statements for each relationship field
-  const coalesceStatements = relationshipFields
-    .map(
-      (field) =>
-        `| EVAL entity.relationships.${field} = COALESCE(entity.relationships.${field}, [""])`
-    )
-    .join('\n');
+  const targetsEval = relationshipFields
+    .map((field) => buildRelationshipTargetsEval(field))
+    .join(',\n  ');
 
-  // Build FORK branches for each relationship field
+  // Build FORK branches: expand flattened targets per relationship leaf
   const forkBranches = relationshipFields
-    .map(
-      (field) =>
-        `  (MV_EXPAND entity.relationships.${field} | EVAL relationship = "${field}" | EVAL _target_id = entity.relationships.${field} | DROP entity.relationships.*)`
-    )
+    .map((field) => {
+      const col = `\`_rel_targets_${field}\``;
+      return `  (MV_EXPAND ${col} | EVAL relationship = "${field}" | EVAL _target_id = TO_STRING(${col}) | DROP entity.relationships.*, ${col})`;
+    })
     .join('\n');
 
   // Store source entity fields before LOOKUP JOIN as they get overwritten by target entity fields
@@ -79,7 +91,8 @@ const buildRelationshipsEsqlQuery = ({
 | RENAME entity.EngineMetadata.Type = _source_engine_metadata_type`;
 
   return `FROM ${indexName}
-${coalesceStatements}
+| EVAL
+  ${targetsEval}
 | FORK
 ${forkBranches}
 | WHERE _target_id != ""
@@ -193,12 +206,21 @@ const buildRelationshipDslFilter = (entityIds: EntityId[]) => {
   // Extract just the IDs for the terms query
   const ids = entityIds.map((entity) => entity.id);
 
-  // Build terms queries for each relationship field
-  const relationshipQueries = ENTITY_RELATIONSHIP_FIELDS.map((field) => ({
-    terms: {
-      [`entity.relationships.${field}`]: ids,
-    },
-  }));
+  const relationshipQueries = ENTITY_RELATIONSHIP_FIELDS.map((field) => {
+    if (field === RESOLUTION_RELATIONSHIP_FIELD) {
+      return {
+        terms: {
+          'entity.relationships.resolution.resolved_to': ids,
+        },
+      };
+    }
+
+    return {
+      terms: {
+        [`entity.relationships.${field}.ids`]: ids,
+      },
+    };
+  });
 
   return {
     bool: {

x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts

@@ -9,6 +9,31 @@ import type { Condition } from '@kbn/streamlang';
 import type { EntityType, EntityField, FieldEvaluation } from './entity_schema';
 import { collectValues, newestValue, oldestValue } from './field_retention_operations';
 
+/**
+ * Dotted ECS paths collected into `entity.relationships.*.raw_identifiers.<path>`.
+ * Keep `EntityRelationship.raw_identifiers` in `entity.schema.yaml` in sync (same paths plus
+ * `entity.id` on the schema for target hints; ingest maps canonical EUIDs via `.entity.id` → `ids`).
+ */
+export const ENTITY_RELATIONSHIP_IDENTIFIER_FIELDS = [
+  'host.id',
+  'user.id',
+  'user.email',
+  'host.name',
+  'user.name',
+  'service.name',
+] as const;
+
+const ENTITY_RELATIONSHIP_COLLECT_LEAVES = [
+  'administers',
+  'communicates_with',
+  'depends_on',
+  'owns_inferred',
+  'accesses_infrequently',
+  'accesses_frequently',
+  'owns',
+  'supervises',
+] as const;
+
 export const ENTITY_ID_FIELD = 'entity.id';
 export const ENTITY_SOURCE_FIELD = 'entity.source';
 // Copied from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts
@@ -82,6 +107,30 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => {
       mapping: { type: 'boolean' },
       allowAPIUpdate: true,
     }),
+    newestValue({
+      source: `${prefix}.attributes.storage_class`,
+      destination: 'entity.attributes.storage_class',
+      mapping: { type: 'keyword' },
+      allowAPIUpdate: true,
+    }),
+    collectValues({
+      source: `${prefix}.attributes.permissions`,
+      destination: 'entity.attributes.permissions',
+      mapping: { type: 'keyword' },
+      allowAPIUpdate: true,
+    }),
+    collectValues({
+      source: `${prefix}.attributes.known_redirects`,
+      destination: 'entity.attributes.known_redirects',
+      mapping: { type: 'keyword' },
+      allowAPIUpdate: true,
+    }),
+    newestValue({
+      source: `${prefix}.attributes.oauth_consent_restriction`,
+      destination: 'entity.attributes.oauth_consent_restriction',
+      mapping: { type: 'keyword' },
+      allowAPIUpdate: true,
+    }),
 
     // LIFECYCLE ------------------------------------------------------------
     oldestValue({
@@ -121,48 +170,24 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => {
     }),
 
     // RELATIONSHIPS ------------------------------------------------------------
-    collectValues({
-      source: `${prefix}.relationships.communicates_with`,
-      destination: 'entity.relationships.communicates_with',
-      mapping: { type: 'keyword' },
-      fieldHistoryLength: 50,
-      allowAPIUpdate: true,
-    }),
-    collectValues({
-      source: `${prefix}.relationships.depends_on`,
-      destination: 'entity.relationships.depends_on',
-      mapping: { type: 'keyword' },
-      allowAPIUpdate: true,
-    }),
-    collectValues({
-      source: `${prefix}.relationships.owns_inferred`,
-      destination: 'entity.relationships.owns_inferred',
-      mapping: { type: 'keyword' },
-    }),
-    collectValues({
-      source: `${prefix}.relationships.accesses_infrequently`,
-      destination: 'entity.relationships.accesses_infrequently',
-      mapping: { type: 'keyword' },
-      allowAPIUpdate: true,
-    }),
-    collectValues({
-      source: `${prefix}.relationships.accesses_frequently`,
-      destination: 'entity.relationships.accesses_frequently',
-      mapping: { type: 'keyword' },
-      allowAPIUpdate: true,
-    }),
-    collectValues({
-      source: `${prefix}.relationships.owns`,
-      destination: 'entity.relationships.owns',
-      mapping: { type: 'keyword' },
-      allowAPIUpdate: true,
-    }),
-    collectValues({
-      source: `${prefix}.relationships.supervises`,
-      destination: 'entity.relationships.supervises',
-      mapping: { type: 'keyword' },
-      allowAPIUpdate: true,
-    }),
+    // Source logs use flat `host.entity.relationships.<relationship>.<identifier>`; the entity index
+    // stores raw bags under `raw_identifiers` and canonical EUIDs under `ids`.
+    ...ENTITY_RELATIONSHIP_COLLECT_LEAVES.flatMap((relationship) => [
+      ...ENTITY_RELATIONSHIP_IDENTIFIER_FIELDS.map((idField) =>
+        collectValues({
+          source: `${prefix}.relationships.${relationship}.${idField}`,
+          destination: `entity.relationships.${relationship}.raw_identifiers.${idField}`,
+          mapping: { type: 'keyword' },
+          allowAPIUpdate: true,
+        })
+      ),
+      collectValues({
+        source: `${prefix}.relationships.${relationship}.entity.id`,
+        destination: `entity.relationships.${relationship}.ids`,
+        mapping: { type: 'keyword' },
+        allowAPIUpdate: true,
+      }),
+    ]),
     newestValue({
       source: `${prefix}.relationships.resolution.resolved_to`,
       destination: 'entity.relationships.resolution.resolved_to',

x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts

@@ -16,6 +16,34 @@
 
 import { z } from '@kbn/zod/v4';
 
+/**
+ * One relationship direction: `raw_identifiers` holds ECS-style dotted keys → keyword arrays (aligned with ENTITY_RELATIONSHIP_IDENTIFIER_FIELDS plus entity.id), and canonical target EUIDs under `ids`.
+ */
+export type EntityRelationship = z.infer<typeof EntityRelationship>;
+export const EntityRelationship = z
+  .object({
+    /**
+     * Raw identifier dimensions for graph / resolution hints. Keys match the entity store relationship identifier field set (see ENTITY_RELATIONSHIP_IDENTIFIER_FIELDS in code).
+     */
+    raw_identifiers: z
+      .object({
+        'entity.id': z.array(z.string()).optional(),
+        'host.id': z.array(z.string()).optional(),
+        'host.name': z.array(z.string()).optional(),
+        'user.email': z.array(z.string()).optional(),
+        'user.id': z.array(z.string()).optional(),
+        'user.name': z.array(z.string()).optional(),
+        'service.name': z.array(z.string()).optional(),
+      })
+      .strict()
+      .optional(),
+    /**
+     * Target entity EUIDs for this relationship; used for graph LOOKUP JOIN and DSL filters.
+     */
+    ids: z.array(z.string()).optional(),
+  })
+  .strict();
+
 export type EngineMetadata = z.infer<typeof EngineMetadata>;
 export const EngineMetadata = z
   .object({
@@ -48,6 +76,22 @@ export const EntityField = z
         asset: z.boolean().optional(),
         managed: z.boolean().optional(),
         mfa_enabled: z.boolean().optional(),
+        /**
+         * Storage tier or class assigned to a storage resource (e.g. hot, warm, cold, standard, archive).
+         */
+        storage_class: z.string().optional(),
+        /**
+         * Action-level permissions granted to this entity (not roles or groups).
+         */
+        permissions: z.array(z.string()).optional(),
+        /**
+         * Known redirect URIs or URLs (e.g. OAuth application callbacks).
+         */
+        known_redirects: z.array(z.string()).optional(),
+        /**
+         * OAuth consent restriction (e.g. admin_only, verified_only, unrestricted).
+         */
+        oauth_consent_restriction: z.string().optional(),
       })
       .strict()
       .optional(),
@@ -74,13 +118,14 @@ export const EntityField = z
       .optional(),
     relationships: z
       .object({
-        communicates_with: z.array(z.string()).optional(),
-        depends_on: z.array(z.string()).optional(),
-        owns: z.array(z.string()).optional(),
-        accesses_frequently: z.array(z.string()).optional(),
-        accesses_infrequently: z.array(z.string()).optional(),
-        owns_inferred: z.array(z.string()).optional(),
-        supervises: z.array(z.string()).optional(),
+        administers: EntityRelationship.optional(),
+        communicates_with: EntityRelationship.optional(),
+        depends_on: EntityRelationship.optional(),
+        owns_inferred: EntityRelationship.optional(),
+        accesses_infrequently: EntityRelationship.optional(),
+        accesses_frequently: EntityRelationship.optional(),
+        owns: EntityRelationship.optional(),
+        supervises: EntityRelationship.optional(),
         resolution: z
           .object({
             /**