Remove runtime mappings from CDR data grid queries

x-pack/solutions/security/plugins/cloud_security_posture/common/utils/findings_sort.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * By default, ES will sort keyword fields in case-sensitive format, the
+ * following fields are required to have a case-insensitive sorting.
+ */
+export const FIELDS_REQUIRING_CASE_INSENSITIVE_SORT = [
+  'rule.section',
+  'resource.name',
+  'resource.sub_type',
+];
+
+/**
+ * Generates Painless sorting if the given field is matched or returns default sorting.
+ * This painless script will sort the field in case-insensitive manner.
+ * Missing values are placed last regardless of sort direction.
+ */
+export const getSortField = ({
+  field,
+  direction,
+}: {
+  field: string;
+  direction: 'asc' | 'desc';
+}) => {
+  if (FIELDS_REQUIRING_CASE_INSENSITIVE_SORT.includes(field)) {
+    // Use a high Unicode sentinel for ascending so missing values sort last,
+    // and an empty string for descending so missing values also sort last.
+    // Note: Painless double-quoted strings only support \\ and \" escapes,
+    // so we embed the actual U+FFFF character rather than a \uffff escape sequence.
+    const missingFallback = direction === 'asc' ? '\uffff' : '';
+    return {
+      _script: {
+        type: 'string' as const,
+        order: direction,
+        script: {
+          source: `doc.containsKey("${field}") && !doc["${field}"].empty ? doc["${field}"].value.toLowerCase() : "${missingFallback}"`,
+          lang: 'painless',
+        },
+      },
+    };
+  }
+  return { [field]: { order: direction, unmapped_type: 'keyword' } };
+};
+
+export const getMultiFieldsSort = (sort: string[][]) => {
+  return sort.map(([id, direction]) => {
+    return {
+      ...getSortField({ field: id, direction: direction as 'asc' | 'desc' }),
+    };
+  });
+};

x-pack/solutions/security/plugins/cloud_security_posture/public/common/constants.ts

@@ -117,30 +117,3 @@ export const VULNERABILITY_GROUPING_MULTIPLE_VALUE_FIELDS: string[] = [
   VULNERABILITY_FIELDS.PACKAGE_VERSION,
   VULNERABILITY_FIELDS.PACKAGE_FIXED_VERSION,
 ];
-
-/*
-The fields below are default columns of the Cloud Security Data Table that need to have keyword mapping.
-The runtime mappings are used to prevent filtering out the data when any of these columns are sorted in the Data Table.
-TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove
-the fields from the runtime mappings if they are removed from the Data Table.
-*/
-export const CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [];
-export const CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS: string[] = [
-  'rule.benchmark.rule_number',
-  'rule.section',
-  'resource.sub_type',
-];
-
-/*
-The fields below are used to group the data in the Cloud Security Data Table.
-The keys are the fields that are used to group the data, and the values are the fields that need to have keyword mapping
-to prevent filtering out the data when grouping by the key field.
-WARNING: only add keys which are not mapped as keywords - casting to keywords could have negative effect on performance.
-TODO: Remove the fields below once they are mapped as Keyword in the Third Party integrations, or remove
-the fields from the runtime mappings if they are removed from the Data Table.
-*/
-export const CDR_VULNERABILITY_GROUPING_RUNTIME_MAPPING_FIELDS: Record<string, string[]> = {};
-export const CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS: Record<string, string[]> = {
-  [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_ID]: ['orchestrator.cluster.id'],
-  [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_ID]: ['cloud.account.id'],
-};

x-pack/solutions/security/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getFindingsQuery } from './use_latest_findings';
+
+describe('getFindingsQuery', () => {
+  const baseOptions = {
+    query: { bool: { filter: [], must: [], must_not: [], should: [] } },
+    sort: [['@timestamp', 'desc']],
+    enabled: true,
+    pageSize: 25,
+  };
+  const emptyRulesStates = {};
+
+  describe('sort behavior', () => {
+    it('uses regular field sort with unmapped_type for standard fields', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['@timestamp', 'desc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      expect(result.sort).toEqual([{ '@timestamp': { order: 'desc', unmapped_type: 'keyword' } }]);
+    });
+
+    it('uses painless script sort for case-insensitive fields', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['rule.section', 'asc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      expect(result.sort).toEqual([
+        {
+          _script: {
+            type: 'string',
+            order: 'asc',
+            script: expect.objectContaining({
+              lang: 'painless',
+            }),
+          },
+        },
+      ]);
+    });
+
+    it('uses high sentinel for missing values in ascending script sort', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['rule.section', 'asc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      const script = result.sort![0] as { _script: { script: { source: string } } };
+      // Missing values should use high sentinel (U+FFFF) so they sort last in ascending order
+      expect(script._script.script.source).toContain('\uffff');
+      expect(script._script.script.source).not.toContain(': ""');
+    });
+
+    it('uses empty string for missing values in descending script sort', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['rule.section', 'desc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      const script = result.sort![0] as { _script: { script: { source: string } } };
+      // Missing values should use empty string so they sort last in descending order
+      expect(script._script.script.source).toContain(': ""');
+      expect(script._script.script.source).not.toContain('\uffff');
+    });
+
+    it('handles missing fields safely with containsKey check', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['resource.sub_type', 'asc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      const script = result.sort![0] as { _script: { script: { source: string } } };
+      expect(script._script.script.source).toContain('doc.containsKey');
+      expect(script._script.script.source).toContain('!doc["resource.sub_type"].empty');
+    });
+
+    it('applies case-insensitive sorting for resource.name', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['resource.name', 'desc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      const script = result.sort![0] as { _script: { script: { source: string } } };
+      expect(script._script.script.source).toContain('.toLowerCase()');
+    });
+
+    it('does not use runtime_mappings', () => {
+      const result = getFindingsQuery(
+        { ...baseOptions, sort: [['rule.section', 'asc']] },
+        emptyRulesStates,
+        undefined
+      );
+
+      expect(result).not.toHaveProperty('runtime_mappings');
+    });
+  });
+});

x-pack/solutions/security/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts

@@ -21,8 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common';
 import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
 import type { BaseEsQuery } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
-import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
-import { CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS } from '../../../common/constants';
+import { getMultiFieldsSort } from '../../../../common/utils/findings_sort';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
 
@@ -41,21 +40,6 @@ interface FindingsAggs {
   count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
 }
 
-const getRuntimeMappingsFromSort = (sort: string[][]) => {
-  return sort
-    .filter(([field]) => CDR_MISCONFIGURATION_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field))
-    .reduce((acc, [field]) => {
-      const type: RuntimePrimitiveTypes = 'keyword';
-
-      return {
-        ...acc,
-        [field]: {
-          type,
-        },
-      };
-    }, {});
-};
-
 export const getFindingsQuery = (
   { query, sort }: UseFindingsOptions,
   rulesStates: CspBenchmarkRulesStates,
@@ -66,7 +50,6 @@ export const getFindingsQuery = (
   return {
     index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
     sort: getMultiFieldsSort(sort),
-    runtime_mappings: getRuntimeMappingsFromSort(sort),
     size: MAX_FINDINGS_TO_LOAD,
     aggs: getFindingsCountAggQuery(),
     ignore_unavailable: true,
@@ -92,44 +75,6 @@ export const getFindingsQuery = (
   };
 };
 
-const getMultiFieldsSort = (sort: string[][]) => {
-  return sort.map(([id, direction]) => {
-    return {
-      ...getSortField({ field: id, direction }),
-    };
-  });
-};
-
-/**
- * By default, ES will sort keyword fields in case-sensitive format, the
- * following fields are required to have a case-insensitive sorting.
- */
-const fieldsRequiredSortingByPainlessScript = [
-  'rule.section',
-  'resource.name',
-  'resource.sub_type',
-];
-
-/**
- * Generates Painless sorting if the given field is matched or returns default sorting
- * This painless script will sort the field in case-insensitive manner
- */
-const getSortField = ({ field, direction }: { field: string; direction: string }) => {
-  if (fieldsRequiredSortingByPainlessScript.includes(field)) {
-    return {
-      _script: {
-        type: 'string',
-        order: direction,
-        script: {
-          source: `doc["${field}"].value.toLowerCase()`,
-          lang: 'painless',
-        },
-      },
-    };
-  }
-  return { [field]: direction };
-};
-
 export const useLatestFindings = (options: UseFindingsOptions) => {
   const {
     data,

x-pack/solutions/security/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx

@@ -22,7 +22,6 @@ import {
 import type { FindingsGroupingAggregation } from '@kbn/cloud-security-posture';
 import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
 import {
-  CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS,
   FINDINGS_GROUPING_OPTIONS,
   LOCAL_STORAGE_FINDINGS_GROUPING_KEY,
 } from '../../../common/constants';
@@ -115,28 +114,6 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
   return aggMetrics;
 };
 
-/**
- * Get runtime mappings for the given group field
- * Some fields require additional runtime mappings to aggregate additional information
- * Fallback to keyword type to support custom fields grouping
- */
-const getRuntimeMappingsByGroupField = (
-  field: string
-): Record<string, { type: 'keyword' }> | undefined => {
-  if (CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS?.[field]) {
-    return CDR_MISCONFIGURATION_GROUPING_RUNTIME_MAPPING_FIELDS[field].reduce(
-      (acc, runtimeField) => ({
-        ...acc,
-        [runtimeField]: {
-          type: 'keyword',
-        },
-      }),
-      {}
-    );
-  }
-  return {};
-};
-
 /**
  * Type Guard for checking if the given source is a FindingsRootGroupingAggregation
  */
@@ -214,7 +191,6 @@ export const useLatestFindingsGrouping = ({
     size: pageSize,
     sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
     statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
-    runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
     rootAggregations: [
       {
         failedFindings: {

x-pack/solutions/security/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx

@@ -24,11 +24,7 @@ import {
 import type { BaseEsQuery } from '@kbn/cloud-security-posture';
 import { showErrorToast } from '@kbn/cloud-security-posture';
 import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
-import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
-import {
-  CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS,
-  VULNERABILITY_FIELDS,
-} from '../../../common/constants';
+import { VULNERABILITY_FIELDS } from '../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
 type LatestFindingsRequest = IKibanaSearchRequest<SearchRequest>;
@@ -52,34 +48,18 @@ const getMultiFieldsSort = (sort: string[][]) => {
     }
 
     return {
-      [id]: direction,
+      [id]: { order: direction, unmapped_type: 'keyword' },
     };
   });
 };
 
-const getRuntimeMappingsFromSort = (sort: string[][]) => {
-  return sort
-    .filter(([field]) => CDR_VULNERABILITY_DATA_TABLE_RUNTIME_MAPPING_FIELDS.includes(field))
-    .reduce((acc, [field]) => {
-      const type: RuntimePrimitiveTypes = 'keyword';
-
-      return {
-        ...acc,
-        [field]: {
-          type,
-        },
-      };
-    }, {});
-};
-
 export const getVulnerabilitiesQuery = (
   { query, sort }: VulnerabilitiesQuery,
   pageParam: number
 ) => ({
   index: CDR_VULNERABILITIES_INDEX_PATTERN,
   ignore_unavailable: true,
   sort: getMultiFieldsSort(sort),
-  runtime_mappings: getRuntimeMappingsFromSort(sort),
   size: MAX_FINDINGS_TO_LOAD,
   query: {
     ...query,