[Contextual Security] Replace ENRICH with LOOKUP JOIN queries when fetching entities

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts

@@ -111,6 +111,12 @@ export const GENERIC_ENTITY_INDEX_ENRICH_POLICY =
 
 export const CLOUD_SECURITY_PLUGIN_VERSION = '1.9.0';
 
+/**
+ * Entity store latest index pattern for LOOKUP JOIN queries.
+ * The <space> placeholder should be replaced with the actual space ID.
+ */
+export const ENTITIES_LATEST_INDEX = '.entities.v2.latest.security_generic_<space>';
+
 /**
  * ECS entity actor fields used for graph visualization.
  * NOTE: The order has meaning - it represents the fallback mechanism for detecting the actor field.

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/utils/helpers.test.ts

@@ -14,6 +14,7 @@ import {
   buildMisconfigurationEntityFlyoutPreviewQuery,
   buildVulnerabilityEntityFlyoutPreviewQuery,
   getEnrichPolicyId,
+  getEntitiesLatestIndexName,
 } from './helpers';
 
 const fallbackMessage = 'thisIsAFallBackMessage';
@@ -495,4 +496,36 @@ describe('test helper methods', () => {
       expect(policyId2).toEqual('entity_store_field_retention_generic_space2_v1.0.0');
     });
   });
+
+  describe('getEntitiesLatestIndexName', () => {
+    it('should return the index name with the provided spaceId', () => {
+      const indexName = getEntitiesLatestIndexName('default');
+      const expected = '.entities.v2.latest.security_generic_default';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should return a index name with a custom space', () => {
+      const space = 'test-space';
+      const indexName = getEntitiesLatestIndexName(space);
+      const expected = '.entities.v2.latest.security_generic_test-space';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should handle special characters in space IDs', () => {
+      const space = 'special-chars_123';
+      const indexName = getEntitiesLatestIndexName(space);
+      const expected = '.entities.v2.latest.security_generic_special-chars_123';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should produce a different index name for each space', () => {
+      const space1 = 'space1';
+      const space2 = 'space2';
+      const indexName1 = getEntitiesLatestIndexName(space1);
+      const indexName2 = getEntitiesLatestIndexName(space2);
+      expect(indexName1).not.toEqual(indexName2);
+      expect(indexName1).toEqual('.entities.v2.latest.security_generic_space1');
+      expect(indexName2).toEqual('.entities.v2.latest.security_generic_space2');
+    });
+  });
 });

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/utils/helpers.ts

@@ -8,7 +8,7 @@ import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types
 
 import { i18n } from '@kbn/i18n';
 import type { CspBenchmarkRulesStates } from '../schema/rules/latest';
-import { GENERIC_ENTITY_INDEX_ENRICH_POLICY } from '../constants';
+import { GENERIC_ENTITY_INDEX_ENRICH_POLICY, ENTITIES_LATEST_INDEX } from '../constants';
 
 interface BuildEntityAlertsQueryParams {
   field: string;
@@ -201,3 +201,10 @@ export const buildEntityAlertsQuery = ({
 export const getEnrichPolicyId = (space: string = 'default'): string => {
   return GENERIC_ENTITY_INDEX_ENRICH_POLICY.replace('<space>', space);
 };
+
+/**
+ * Gets the entities latest index name for a specific space
+ */
+export const getEntitiesLatestIndexName = (spaceId: string = 'default'): string => {
+  return ENTITIES_LATEST_INDEX.replace('<space>', spaceId);
+};

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/fetch_graph.test.ts

@@ -39,6 +39,7 @@ describe('fetchGraph', () => {
 
     logger = {
       trace: jest.fn(),
+      debug: jest.fn(),
       info: jest.fn(),
       error: jest.fn(),
     } as unknown as Logger;

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/fetch_graph.ts

@@ -15,18 +15,27 @@ import {
   DOCUMENT_TYPE_ENTITY,
   INDEX_PATTERN_REGEX,
 } from '@kbn/cloud-security-posture-common/schema/graph/v1';
-import { getEnrichPolicyId } from '@kbn/cloud-security-posture-common/utils/helpers';
+import {
+  getEnrichPolicyId,
+  getEntitiesLatestIndexName,
+} from '@kbn/cloud-security-posture-common/utils/helpers';
 import {
   GRAPH_ACTOR_ENTITY_FIELDS,
   GRAPH_TARGET_ENTITY_FIELDS,
 } from '@kbn/cloud-security-posture-common/constants';
-import { generateFieldHintCases, concatPropIfExists } from './utils';
+import {
+  generateFieldHintCases,
+  formatJsonProperty,
+  buildLookupJoinEsql,
+  buildEnrichPolicyEsql,
+} from './utils';
 import type { EsQuery, GraphEdge, OriginEventId } from './types';
 
 interface BuildEsqlQueryParams {
   indexPatterns: string[];
   originEventIds: OriginEventId[];
   originAlertIds: OriginEventId[];
+  isLookupIndexAvailable: boolean;
   isEnrichPolicyExists: boolean;
   spaceId: string;
   alertsMappingsIncluded: boolean;
@@ -65,8 +74,12 @@ export const fetchGraph = async ({
     }
   });
 
-  const isEnrichPolicyExists = await checkEnrichPolicyExists(esClient, logger, spaceId);
-
+  // Check if the entities lookup index exists and is in lookup mode (preferred)
+  // If not, fall back to checking if the enrich policy exists (deprecated)
+  const isLookupIndexAvailable = await checkIfEntitiesIndexLookupMode(esClient, logger, spaceId);
+  const isEnrichPolicyExists = isLookupIndexAvailable
+    ? false
+    : await checkEnrichPolicyExists(esClient, logger, spaceId);
   const SECURITY_ALERTS_PARTIAL_IDENTIFIER = '.alerts-security.alerts-';
   const alertsMappingsIncluded = indexPatterns.some((indexPattern) =>
     indexPattern.includes(SECURITY_ALERTS_PARTIAL_IDENTIFIER)
@@ -76,6 +89,7 @@ export const fetchGraph = async ({
     indexPatterns,
     originEventIds,
     originAlertIds,
+    isLookupIndexAvailable,
     isEnrichPolicyExists,
     spaceId,
     alertsMappingsIncluded,
@@ -159,6 +173,11 @@ const checkEnrichPolicyExists = async (
       name: getEnrichPolicyId(spaceId),
     });
 
+    logger.debug(
+      `Enrich policy check for [${getEnrichPolicyId(spaceId)}]: found ${
+        policies?.length
+      } policies, policies: ${JSON.stringify(policies?.map((p) => p.config.match?.name))}`
+    );
     return policies.some((policy) => policy.config.match?.name === getEnrichPolicyId(spaceId));
   } catch (error) {
     logger.error(`Error fetching enrich policy ${error.message}`);
@@ -167,10 +186,97 @@ const checkEnrichPolicyExists = async (
   }
 };
 
+/**
+ * Checks if the entities latest index exists and is configured in lookup mode.
+ * This is the preferred method for entity enrichment (replaces deprecated ENRICH policy).
+ */
+const checkIfEntitiesIndexLookupMode = async (
+  esClient: IScopedClusterClient,
+  logger: Logger,
+  spaceId: string
+): Promise<boolean> => {
+  const indexName = getEntitiesLatestIndexName(spaceId);
+  try {
+    const response = await esClient.asInternalUser.indices.getSettings({
+      index: indexName,
+    });
+    const indexSettings = response[indexName];
+    if (!indexSettings) {
+      logger.debug(`Entities index ${indexName} not found`);
+      return false;
+    }
+
+    // Check if index is in lookup mode
+    const mode = indexSettings.settings?.index?.mode;
+    const isLookupMode = mode === 'lookup';
+
+    if (!isLookupMode) {
+      logger.debug(`Entities index ${indexName} exists but is not in lookup mode (mode: ${mode})`);
+    }
+
+    return isLookupMode;
+  } catch (error) {
+    if (error.statusCode === 404) {
+      logger.debug(`Entities index ${indexName} does not exist`);
+      return false;
+    }
+    logger.error(`Error checking entities index ${indexName}: ${error.message}`);
+    return false;
+  }
+};
+
+/**
+ * Generates ESQL statements for building entity fields with enrichment data.
+ * This is used when entity store enrichment is available (via LOOKUP JOIN or ENRICH).
+ */
+const buildEnrichedEntityFieldsEsql = (): string => {
+  return `// Construct actor and target entities data
+// Build entity field conditionally - only include fields that have values
+| EVAL actorEntityField = CASE(
+    actorEntityName IS NOT NULL OR actorEntityType IS NOT NULL OR actorEntitySubType IS NOT NULL,
+    CONCAT(",\\"entity\\":", "{",
+      ${formatJsonProperty('name', 'actorEntityName', false)},
+      ${formatJsonProperty('type', 'actorEntityType')},
+      ${formatJsonProperty('sub_type', 'actorEntitySubType')},
+      CASE(
+        actorHostIp IS NOT NULL,
+        CONCAT(",\\"host\\":", "{", "\\"ip\\":\\"", TO_STRING(actorHostIp), "\\"", "}"),
+        ""
+      ),
+      ",\\"availableInEntityStore\\":true",
+      ",\\"ecsParentField\\":\\"", actorEntityFieldHint, "\\"",
+    "}"),
+    CONCAT(",\\"entity\\":", "{",
+      "\\"availableInEntityStore\\":false",
+      ",\\"ecsParentField\\":\\"", actorEntityFieldHint, "\\"",
+    "}")
+  )
+| EVAL targetEntityField = CASE(
+    targetEntityName IS NOT NULL OR targetEntityType IS NOT NULL OR targetEntitySubType IS NOT NULL,
+    CONCAT(",\\"entity\\":", "{",
+      ${formatJsonProperty('name', 'targetEntityName', false)},
+      ${formatJsonProperty('type', 'targetEntityType')},
+      ${formatJsonProperty('sub_type', 'targetEntitySubType')},
+      CASE(
+        targetHostIp IS NOT NULL,
+        CONCAT(",\\"host\\":", "{", "\\"ip\\":\\"", TO_STRING(targetHostIp), "\\"", "}"),
+        ""
+      ),
+      ",\\"availableInEntityStore\\":true",
+      ",\\"ecsParentField\\":\\"", targetEntityFieldHint, "\\"",
+    "}"),
+    CONCAT(",\\"entity\\":", "{",
+      "\\"availableInEntityStore\\":false",
+      ",\\"ecsParentField\\":\\"", targetEntityFieldHint, "\\"",
+    "}")
+  )`;
+};
+
 const buildEsqlQuery = ({
   indexPatterns,
   originEventIds,
   originAlertIds,
+  isLookupIndexAvailable,
   isEnrichPolicyExists,
   spaceId,
   alertsMappingsIncluded,
@@ -226,52 +332,17 @@ ${targetFieldHintCases},
     ""
 )
 ${
-  isEnrichPolicyExists
+  isLookupIndexAvailable
     ? `
+${buildLookupJoinEsql(getEntitiesLatestIndexName(spaceId))}
 
-| ENRICH ${enrichPolicyName} ON actorEntityId WITH actorEntityName = entity.name, actorEntityType = entity.type, actorEntitySubType = entity.sub_type, actorHostIp = host.ip
-| ENRICH ${enrichPolicyName} ON targetEntityId WITH targetEntityName = entity.name, targetEntityType = entity.type, targetEntitySubType = entity.sub_type, targetHostIp = host.ip
+${buildEnrichedEntityFieldsEsql()}
+`
+    : isEnrichPolicyExists
+    ? `
+${buildEnrichPolicyEsql(enrichPolicyName)}
 
-// Construct actor and target entities data
-// Build entity field conditionally - only include fields that have values
-| EVAL actorEntityField = CASE(
-    actorEntityName IS NOT NULL OR actorEntityType IS NOT NULL OR actorEntitySubType IS NOT NULL,
-    CONCAT(",\\"entity\\":", "{",
-      ${concatPropIfExists('name', 'actorEntityName', false)},
-      ${concatPropIfExists('type', 'actorEntityType')},
-      ${concatPropIfExists('sub_type', 'actorEntitySubType')},
-      CASE(
-        actorHostIp IS NOT NULL,
-        CONCAT(",\\"host\\":", "{", "\\"ip\\":\\"", TO_STRING(actorHostIp), "\\"", "}"),
-        ""
-      ),
-      ",\\"availableInEntityStore\\":true",
-      ",\\"ecsParentField\\":\\"", actorEntityFieldHint, "\\"",
-    "}"),
-    CONCAT(",\\"entity\\":", "{",
-      "\\"availableInEntityStore\\":false",
-      ",\\"ecsParentField\\":\\"", actorEntityFieldHint, "\\"",
-    "}")
-  )
-| EVAL targetEntityField = CASE(
-    targetEntityName IS NOT NULL OR targetEntityType IS NOT NULL OR targetEntitySubType IS NOT NULL,
-    CONCAT(",\\"entity\\":", "{",
-      ${concatPropIfExists('name', 'targetEntityName', false)},
-      ${concatPropIfExists('type', 'targetEntityType')},
-      ${concatPropIfExists('sub_type', 'targetEntitySubType')},
-      CASE(
-        targetHostIp IS NOT NULL,
-        CONCAT(",\\"host\\":", "{", "\\"ip\\":\\"", TO_STRING(targetHostIp), "\\"", "}"),
-        ""
-      ),
-      ",\\"availableInEntityStore\\":true",
-      ",\\"ecsParentField\\":\\"", targetEntityFieldHint, "\\"",
-    "}"),
-    CONCAT(",\\"entity\\":", "{",
-      "\\"availableInEntityStore\\":false",
-      ",\\"ecsParentField\\":\\"", targetEntityFieldHint, "\\"",
-    "}")
-  )
+${buildEnrichedEntityFieldsEsql()}
 `
     : `
 | EVAL actorEntityField = CONCAT(",\\"entity\\":", "{",