[Contextual Security] Replace ENRICH with LOOKUP JOIN queries when fetching entities

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/constants.ts

@@ -111,6 +111,12 @@ export const GENERIC_ENTITY_INDEX_ENRICH_POLICY =
 
 export const CLOUD_SECURITY_PLUGIN_VERSION = '1.9.0';
 
+/**
+ * Entity store latest index pattern for LOOKUP JOIN queries.
+ * The <space> placeholder should be replaced with the actual space ID.
+ */
+export const ENTITIES_LATEST_INDEX = '.entities.v2.latest.security_generic_<space>';
+
 /**
  * ECS entity actor fields used for graph visualization.
  * NOTE: The order has meaning - it represents the fallback mechanism for detecting the actor field.

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/utils/helpers.test.ts

@@ -14,6 +14,7 @@ import {
   buildMisconfigurationEntityFlyoutPreviewQuery,
   buildVulnerabilityEntityFlyoutPreviewQuery,
   getEnrichPolicyId,
+  getEntitiesLatestIndexName,
 } from './helpers';
 
 const fallbackMessage = 'thisIsAFallBackMessage';
@@ -495,4 +496,36 @@ describe('test helper methods', () => {
       expect(policyId2).toEqual('entity_store_field_retention_generic_space2_v1.0.0');
     });
   });
+
+  describe('getEntitiesLatestIndexName', () => {
+    it('should return the index name with the provided spaceId', () => {
+      const indexName = getEntitiesLatestIndexName('default');
+      const expected = '.entities.v2.latest.security_generic_default';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should return a index name with a custom space', () => {
+      const space = 'test-space';
+      const indexName = getEntitiesLatestIndexName(space);
+      const expected = '.entities.v2.latest.security_generic_test-space';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should handle special characters in space IDs', () => {
+      const space = 'special-chars_123';
+      const indexName = getEntitiesLatestIndexName(space);
+      const expected = '.entities.v2.latest.security_generic_special-chars_123';
+      expect(indexName).toEqual(expected);
+    });
+
+    it('should produce a different index name for each space', () => {
+      const space1 = 'space1';
+      const space2 = 'space2';
+      const indexName1 = getEntitiesLatestIndexName(space1);
+      const indexName2 = getEntitiesLatestIndexName(space2);
+      expect(indexName1).not.toEqual(indexName2);
+      expect(indexName1).toEqual('.entities.v2.latest.security_generic_space1');
+      expect(indexName2).toEqual('.entities.v2.latest.security_generic_space2');
+    });
+  });
 });

x-pack/solutions/security/packages/kbn-cloud-security-posture/common/utils/helpers.ts

@@ -8,7 +8,7 @@ import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types
 
 import { i18n } from '@kbn/i18n';
 import type { CspBenchmarkRulesStates } from '../schema/rules/latest';
-import { GENERIC_ENTITY_INDEX_ENRICH_POLICY } from '../constants';
+import { GENERIC_ENTITY_INDEX_ENRICH_POLICY, ENTITIES_LATEST_INDEX } from '../constants';
 
 interface BuildEntityAlertsQueryParams {
   field: string;
@@ -201,3 +201,10 @@ export const buildEntityAlertsQuery = ({
 export const getEnrichPolicyId = (space: string = 'default'): string => {
   return GENERIC_ENTITY_INDEX_ENRICH_POLICY.replace('<space>', space);
 };
+
+/**
+ * Gets the entities latest index name for a specific space
+ */
+export const getEntitiesLatestIndexName = (spaceId: string = 'default'): string => {
+  return ENTITIES_LATEST_INDEX.replace('<space>', spaceId);
+};

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/entity_type_constants.ts

@@ -90,6 +90,7 @@ const buildEntityTypeMappings = (): EntityTypeMappings => {
         'Snapshot',
         'Volume',
         'Volume Claim',
+        'Storage',
         'Storage Bucket',
         'Backup Service',
         'Managed Certificate',

x-pack/solutions/security/plugins/cloud_security_posture/server/routes/graph/fetch_graph.test.ts

@@ -9,7 +9,10 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
 import { fetchGraph } from './fetch_graph';
 import type { Logger } from '@kbn/core/server';
 import type { OriginEventId, EsQuery } from './types';
-import { getEnrichPolicyId } from '@kbn/cloud-security-posture-common/utils/helpers';
+import {
+  getEnrichPolicyId,
+  getEntitiesLatestIndexName,
+} from '@kbn/cloud-security-posture-common/utils/helpers';
 
 describe('fetchGraph', () => {
   const esClient = elasticsearchServiceMock.createScopedClusterClient();
@@ -39,6 +42,7 @@ describe('fetchGraph', () => {
 
     logger = {
       trace: jest.fn(),
+      debug: jest.fn(),
       info: jest.fn(),
       error: jest.fn(),
     } as unknown as Logger;
@@ -138,7 +142,161 @@ describe('fetchGraph', () => {
     expect(result).toEqual([{ id: 'dummy' }]);
   });
 
+  describe('LOOKUP JOIN integration', () => {
+    it('should include LOOKUP JOIN clause when entities index is in lookup mode', async () => {
+      const indexName = getEntitiesLatestIndexName('default');
+
+      // Mock the indices.getSettings to return lookup mode
+      (esClient.asInternalUser.indices as jest.Mocked<any>).getSettings = jest
+        .fn()
+        .mockResolvedValueOnce({
+          [indexName]: {
+            settings: {
+              index: {
+                mode: 'lookup',
+              },
+            },
+          },
+        });
+
+      const validIndexPatterns = ['valid_index'];
+      const params = {
+        esClient,
+        logger,
+        start: 0,
+        end: 1000,
+        originEventIds: [] as OriginEventId[],
+        showUnknownTarget: false,
+        indexPatterns: validIndexPatterns,
+        spaceId: 'default',
+        esQuery: undefined as EsQuery | undefined,
+      };
+
+      const result = await fetchGraph(params);
+
+      expect(esClient.asCurrentUser.helpers.esql).toBeCalledTimes(1);
+      const esqlCallArgs = esClient.asCurrentUser.helpers.esql.mock.calls[0];
+      const query = esqlCallArgs[0].query;
+
+      // Verify LOOKUP JOIN is used (preferred over ENRICH)
+      expect(query).toContain(`LOOKUP JOIN ${indexName} ON entity.id`);
+
+      // Verify LOOKUP JOIN populates expected fields
+      expect(query).toContain('actorEntityName');
+      expect(query).toContain('actorEntityType');
+      expect(query).toContain('actorEntitySubType');
+      expect(query).toContain('targetEntityName');
+      expect(query).toContain('targetEntityType');
+      expect(query).toContain('targetEntitySubType');
+
+      // Verify ENRICH is NOT used when LOOKUP JOIN is available
+      expect(query).not.toContain('ENRICH');
+
+      expect(result).toEqual([{ id: 'dummy' }]);
+    });
+
+    it('should not include LOOKUP JOIN clause when entities index is not in lookup mode', async () => {
+      const indexName = getEntitiesLatestIndexName('default');
+
+      // Mock the indices.getSettings to return standard mode (not lookup)
+      (esClient.asInternalUser.indices as jest.Mocked<any>).getSettings = jest
+        .fn()
+        .mockResolvedValueOnce({
+          [indexName]: {
+            settings: {
+              index: {
+                mode: 'standard',
+              },
+            },
+          },
+        });
+
+      // Also mock enrich policy to not exist
+      (esClient.asInternalUser.enrich as jest.Mocked<any>).getPolicy = jest
+        .fn()
+        .mockResolvedValueOnce({
+          policies: [],
+        });
+
+      const validIndexPatterns = ['valid_index'];
+      const params = {
+        esClient,
+        logger,
+        start: 0,
+        end: 1000,
+        originEventIds: [] as OriginEventId[],
+        showUnknownTarget: false,
+        indexPatterns: validIndexPatterns,
+        spaceId: 'default',
+        esQuery: undefined as EsQuery | undefined,
+      };
+
+      const result = await fetchGraph(params);
+
+      expect(esClient.asCurrentUser.helpers.esql).toBeCalledTimes(1);
+      const esqlCallArgs = esClient.asCurrentUser.helpers.esql.mock.calls[0];
+      const query = esqlCallArgs[0].query;
+
+      // Verify LOOKUP JOIN is NOT used
+      expect(query).not.toContain('LOOKUP JOIN');
+
+      // Verify ENRICH is also NOT used (no policy exists)
+      expect(query).not.toContain('ENRICH');
+
+      // Verify fallback EVALs are present for null values
+      expect(query).toMatch(/EVAL\s+actorEntityName\s*=\s*TO_STRING\(null\)/);
+      expect(query).toMatch(/EVAL\s+targetEntityName\s*=\s*TO_STRING\(null\)/);
+
+      expect(result).toEqual([{ id: 'dummy' }]);
+    });
+
+    it('should not include LOOKUP JOIN when entities index does not exist', async () => {
+      // Mock the indices.getSettings to throw 404 (index not found)
+      (esClient.asInternalUser.indices as jest.Mocked<any>).getSettings = jest
+        .fn()
+        .mockRejectedValueOnce({ statusCode: 404 });
+
+      // Also mock enrich policy to not exist
+      (esClient.asInternalUser.enrich as jest.Mocked<any>).getPolicy = jest
+        .fn()
+        .mockResolvedValueOnce({
+          policies: [],
+        });
+
+      const validIndexPatterns = ['valid_index'];
+      const params = {
+        esClient,
+        logger,
+        start: 0,
+        end: 1000,
+        originEventIds: [] as OriginEventId[],
+        showUnknownTarget: false,
+        indexPatterns: validIndexPatterns,
+        spaceId: 'default',
+        esQuery: undefined as EsQuery | undefined,
+      };
+
+      const result = await fetchGraph(params);
+
+      expect(esClient.asCurrentUser.helpers.esql).toBeCalledTimes(1);
+      const esqlCallArgs = esClient.asCurrentUser.helpers.esql.mock.calls[0];
+      const query = esqlCallArgs[0].query;
+
+      // Verify LOOKUP JOIN is NOT used
+      expect(query).not.toContain('LOOKUP JOIN');
+
+      expect(result).toEqual([{ id: 'dummy' }]);
+    });
+  });
+
   describe('ENRICH policy integration', () => {
+    beforeEach(() => {
+      // Default: no lookup index available (falls back to ENRICH check)
+      (esClient.asInternalUser.indices as jest.Mocked<any>).getSettings = jest
+        .fn()
+        .mockRejectedValue({ statusCode: 404 });
+    });
+
     it('should include ENRICH clause when policy exists', async () => {
       // Mock the enrich.getPolicy method to return a policy that exists
       (esClient.asInternalUser.enrich as jest.Mocked<any>).getPolicy = jest