From 5bf16b99896ebdc4f3b58cf4f6262f09306aa502 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Wed, 9 Oct 2024 14:15:09 -0700 Subject: [PATCH 1/6] handle missing mapping --- packages/kbn-grouping/src/containers/query/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-grouping/src/containers/query/index.ts b/packages/kbn-grouping/src/containers/query/index.ts index b31b0701de4f9..b7468a7d44232 100644 --- a/packages/kbn-grouping/src/containers/query/index.ts +++ b/packages/kbn-grouping/src/containers/query/index.ts @@ -56,8 +56,8 @@ export const getGroupingQuery = ({ type: 'keyword', script: { source: - // when size()==0, emits a uniqueValue as the value to represent this group else join by uniqueValue. - "if (doc[params['selectedGroup']].size()==0) { emit(params['uniqueValue']) }" + + // when doc misses the field, or it's size()==0, emits a uniqueValue as the value to represent this group else join by uniqueValue. + "if (!doc.containsKey(params['selectedGroup']) || doc[params['selectedGroup']].empty || doc[params['selectedGroup']].size() == 0) { emit(params['uniqueValue']) }" + // Else, join the values with uniqueValue. We cannot simply emit the value like doc[params['selectedGroup']].value, // the runtime field will only return the first value in an array. // The docs advise that if the field has multiple values, "Scripts can call the emit method multiple times to emit multiple values." From a1271e5b318d9a805db34050b11022907a7bb6af Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Wed, 9 Oct 2024 14:15:31 -0700 Subject: [PATCH 2/6] WIP: adding mapping --- .../hooks/use_latest_vulnerabilities_grouping.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 516cbed0c3975..b02322ac0b9f8 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -163,6 +163,11 @@ export const useLatestVulnerabilitiesGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: { + 'cloud.provider': { + type: 'keyword', + }, + }, }); const { data, isFetching } = useGroupedVulnerabilities({ From 1c7884f7e5f2fe6392aa4f0d5a1b7bad27c039eb Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Wed, 9 Oct 2024 21:48:51 -0700 Subject: [PATCH 3/6] handle misconfiguration grouping fields with missing mapping --- .../latest_findings/use_latest_findings.ts | 16 ++++++ .../use_latest_findings_grouping.tsx | 52 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts index f6f27e15ee7a4..955f5a45a9743 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts @@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common'; import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest'; import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture'; import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils'; @@ -39,6 +40,20 @@ interface FindingsAggs { count: estypes.AggregationsMultiBucketAggregateBase; } +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getFindingsQuery = ( { query, sort }: UseFindingsOptions, rulesStates: CspBenchmarkRulesStates, @@ -49,6 +64,7 @@ export const getFindingsQuery = ( return { index: CDR_MISCONFIGURATIONS_INDEX_PATTERN, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, aggs: getFindingsCountAggQuery(), ignore_unavailable: true, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index cc409fb95024d..d9718c672654b 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -114,6 +114,52 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: + return { + 'resource.id': { + type: 'keyword', + }, + 'resource.sub_type': { + type: 'keyword', + }, + 'resource.type': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.RULE_NAME: + return { + 'rule.benchmark.version': { + type: 'keyword', + }, + }; + case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: + return { + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a FindingsRootGroupingAggregation */ @@ -189,6 +235,12 @@ export const useLatestFindingsGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), + runtimeMappings: { + ...getRuntimeMappingsByGroupField(currentSelectedGroup), + 'result.evaluation': { + type: 'keyword', + }, + }, rootAggregations: [ { failedFindings: { From e8e5f20340efe4f017b5350618db9ca6c4943e07 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Wed, 9 Oct 2024 21:49:09 -0700 Subject: [PATCH 4/6] handle vulnerabilities grouping fields with missing mapping --- .../hooks/use_latest_vulnerabilities.tsx | 21 ++++++++++ .../use_latest_vulnerabilities_grouping.tsx | 38 +++++++++++++++---- .../utils/custom_sort_script.ts | 8 +++- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx index 0d0ea9ba5a22f..0b9cf6978c258 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx @@ -23,6 +23,7 @@ import { } from '@kbn/cloud-security-posture-common'; import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture'; import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest'; +import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { VULNERABILITY_FIELDS } from '../../../common/constants'; import { useKibana } from '../../../common/hooks/use_kibana'; import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script'; @@ -52,6 +53,25 @@ const getMultiFieldsSort = (sort: string[][]) => { }); }; +const getRuntimeMappingsFromSort = (sort: string[][]) => { + return sort.reduce((acc, [field]) => { + // TODO: Add proper type for all fields available in the field selector + const type: RuntimePrimitiveTypes = + field === VULNERABILITY_FIELDS.SCORE_BASE + ? 'double' + : field === '@timestamp' + ? 'date' + : 'keyword'; + + return { + ...acc, + [field]: { + type, + }, + }; + }, {}); +}; + export const getVulnerabilitiesQuery = ( { query, sort }: VulnerabilitiesQuery, pageParam: number @@ -59,6 +79,7 @@ export const getVulnerabilitiesQuery = ( index: CDR_VULNERABILITIES_INDEX_PATTERN, ignore_unavailable: true, sort: getMultiFieldsSort(sort), + runtime_mappings: getRuntimeMappingsFromSort(sort), size: MAX_FINDINGS_TO_LOAD, query: { ...query, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index b02322ac0b9f8..66b3b06793e53 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -88,12 +88,40 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { ...aggMetrics, getTermAggregation('cloudProvider', VULNERABILITY_FIELDS.CLOUD_PROVIDER), ]; - case VULNERABILITY_GROUPING_OPTIONS.CVE: - return [...aggMetrics, getTermAggregation('description', VULNERABILITY_FIELDS.DESCRIPTION)]; } return aggMetrics; }; +/** + * Get runtime mappings for the given group field + * Some fields require additional runtime mappings to aggregate additional information + * Fallback to keyword type to support custom fields grouping + */ +const getRuntimeMappingsByGroupField = ( + field: string +): Record | undefined => { + switch (field) { + case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { + type: 'keyword', + }, + }; + case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: + return { + [VULNERABILITY_FIELDS.RESOURCE_ID]: { + type: 'keyword', + }, + }; + default: + return { + [field]: { + type: 'keyword', + }, + }; + } +}; + /** * Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation */ @@ -163,11 +191,7 @@ export const useLatestVulnerabilitiesGrouping = ({ size: pageSize, sort: [{ groupByField: { order: 'desc' } }], statsAggregations: getAggregationsByGroupField(currentSelectedGroup), - runtimeMappings: { - 'cloud.provider': { - type: 'keyword', - }, - }, + runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup), }); const { data, isFetching } = useGroupedVulnerabilities({ diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts index 780cd539305b3..e517d622e71c5 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts @@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) = type: 'string', order: direction, script: { - source: `doc["${field}"].value.toLowerCase()`, + source: ` + if (doc.containsKey('${field}') && !doc['${field}'].empty) { + return doc['${field}'].value.toLowerCase(); + } else { + return ""; + } + `, lang: 'painless', }, }, From a388273a6a440c912b76561dff0d924373eb362a Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Thu, 10 Oct 2024 00:35:29 -0700 Subject: [PATCH 5/6] reverting package changes --- .../src/containers/query/index.ts | 4 ++-- .../use_latest_findings_grouping.tsx | 20 +++++++++++++++++++ .../use_latest_vulnerabilities_grouping.tsx | 17 ++++++++++++++++ .../alerts_table/grouping_settings/mock.ts | 2 +- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/kbn-grouping/src/containers/query/index.ts b/packages/kbn-grouping/src/containers/query/index.ts index b7468a7d44232..b31b0701de4f9 100644 --- a/packages/kbn-grouping/src/containers/query/index.ts +++ b/packages/kbn-grouping/src/containers/query/index.ts @@ -56,8 +56,8 @@ export const getGroupingQuery = ({ type: 'keyword', script: { source: - // when doc misses the field, or it's size()==0, emits a uniqueValue as the value to represent this group else join by uniqueValue. - "if (!doc.containsKey(params['selectedGroup']) || doc[params['selectedGroup']].empty || doc[params['selectedGroup']].size() == 0) { emit(params['uniqueValue']) }" + + // when size()==0, emits a uniqueValue as the value to represent this group else join by uniqueValue. + "if (doc[params['selectedGroup']].size()==0) { emit(params['uniqueValue']) }" + // Else, join the values with uniqueValue. We cannot simply emit the value like doc[params['selectedGroup']].value, // the runtime field will only return the first value in an array. // The docs advise that if the field has multiple values, "Scripts can call the emit method multiple times to emit multiple values." diff --git a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx index d9718c672654b..6482d864347a1 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx @@ -125,6 +125,9 @@ const getRuntimeMappingsByGroupField = ( switch (field) { case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME: return { + [FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, 'resource.id': { type: 'keyword', }, @@ -137,13 +140,30 @@ const getRuntimeMappingsByGroupField = ( }; case FINDINGS_GROUPING_OPTIONS.RULE_NAME: return { + [FINDINGS_GROUPING_OPTIONS.RULE_NAME]: { + type: 'keyword', + }, 'rule.benchmark.version': { type: 'keyword', }, }; case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: + return { + [FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, + 'rule.benchmark.name': { + type: 'keyword', + }, + 'rule.benchmark.id': { + type: 'keyword', + }, + }; case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME: return { + [FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: { + type: 'keyword', + }, 'rule.benchmark.name': { type: 'keyword', }, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx index 66b3b06793e53..3c52590f8fd80 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx @@ -88,6 +88,8 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => { ...aggMetrics, getTermAggregation('cloudProvider', VULNERABILITY_FIELDS.CLOUD_PROVIDER), ]; + case VULNERABILITY_GROUPING_OPTIONS.CVE: + return [...aggMetrics, getTermAggregation('description', VULNERABILITY_FIELDS.DESCRIPTION)]; } return aggMetrics; }; @@ -103,16 +105,31 @@ const getRuntimeMappingsByGroupField = ( switch (field) { case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME: return { + [VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: { + type: 'keyword', + }, [VULNERABILITY_FIELDS.CLOUD_PROVIDER]: { type: 'keyword', }, }; case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME: return { + [VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: { + type: 'keyword', + }, [VULNERABILITY_FIELDS.RESOURCE_ID]: { type: 'keyword', }, }; + case VULNERABILITY_GROUPING_OPTIONS.CVE: + return { + [VULNERABILITY_GROUPING_OPTIONS.CVE]: { + type: 'keyword', + }, + [VULNERABILITY_FIELDS.DESCRIPTION]: { + type: 'keyword', + }, + }; default: return { [field]: { diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts index 93fbb90ede9e5..4f04303456d0b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts @@ -102,7 +102,7 @@ export const getQuery = ( type: 'keyword', script: { source: - "if (doc[params['selectedGroup']].size()==0) { emit(params['uniqueValue']) } else { emit(doc[params['selectedGroup']].join(params['uniqueValue']))}", + "if (!doc.containsKey(params['selectedGroup']) || doc[params['selectedGroup']].empty || doc[params['selectedGroup']].size() == 0) { emit(params['uniqueValue']) } else { emit(doc[params['selectedGroup']].join(params['uniqueValue']))}", params: { selectedGroup, uniqueValue, From 2da8aa5ad20fd8768b5c4616c0ef107a93203606 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Thu, 10 Oct 2024 00:36:19 -0700 Subject: [PATCH 6/6] reverting test changes --- .../components/alerts_table/grouping_settings/mock.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts index 4f04303456d0b..93fbb90ede9e5 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/mock.ts @@ -102,7 +102,7 @@ export const getQuery = ( type: 'keyword', script: { source: - "if (!doc.containsKey(params['selectedGroup']) || doc[params['selectedGroup']].empty || doc[params['selectedGroup']].size() == 0) { emit(params['uniqueValue']) } else { emit(doc[params['selectedGroup']].join(params['uniqueValue']))}", + "if (doc[params['selectedGroup']].size()==0) { emit(params['uniqueValue']) } else { emit(doc[params['selectedGroup']].join(params['uniqueValue']))}", params: { selectedGroup, uniqueValue,