From ff4febbf490758bf72ed4f5fe1668265d96faedb Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Wed, 7 May 2025 07:25:41 -0700 Subject: [PATCH] [ResponseOps] Update ES|QL ES query rule for context.grouping action variable (#220302) Follow on from this PR that was merged, https://github.com/elastic/kibana/pull/213550 ## Summary This PR updates the ES|QL grouping processing to work with the `context.grouping` action variable. ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### To verify 1. Create an ES|QL rule with grouping. 2. In "active" and "recovered" action message, use `context.grouping` variable 3. Verify that both "active" and "recovered" alert notifications contain correct information Example if grouping on `host.name` and `container.id`: ``` { "grouping": "{{context.grouping}}", "host.name": "{{context.grouping.host.name}}", "container.id": "{{context.grouping.container.id}}" } ``` (cherry picked from commit d72c62c4c0536944fd17eb0a6ff9d83aea72ab61) --- .../common/es_query/esql_query_utils.test.ts | 57 +++++++++++++++---- .../common/es_query/esql_query_utils.ts | 5 +- .../builtin_alert_types/es_query/esql_only.ts | 23 +++++++- 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.test.ts b/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.test.ts index 0f38c8cb200ff..48394d8da3a2d 100644 --- a/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.test.ts +++ b/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.test.ts @@ -116,7 +116,7 @@ describe('ESQL query utils', () => { buckets: [ { doc_count: 1, - key: '1.8.0', + key: ['1.8.0'], topHitsAgg: { hits: { hits: [ @@ -135,7 +135,7 @@ describe('ESQL query utils', () => { }, { doc_count: 1, - key: '1.2.0', + key: ['1.2.0'], topHitsAgg: { hits: { hits: [ @@ -161,6 +161,7 @@ describe('ESQL query utils', () => { }, isCountAgg: false, isGroupAgg: true, + termField: ['ecs.version'], }); expect(rows).toEqual([ { @@ -200,7 +201,7 @@ describe('ESQL query utils', () => { buckets: [ { doc_count: 1, - key: '1.8.0', + key: ['1.8.0'], topHitsAgg: { hits: { hits: [ @@ -219,7 +220,7 @@ describe('ESQL query utils', () => { }, { doc_count: 2, - key: '1.2.0', + key: ['1.2.0'], topHitsAgg: { hits: { hits: [ @@ -254,6 +255,7 @@ describe('ESQL query utils', () => { }, isCountAgg: false, isGroupAgg: true, + termField: ['ecs.version'], }); expect(duplicateAlertIds?.size).toBe(1); }); @@ -323,7 +325,17 @@ describe('ESQL query utils', () => { buckets: [ { doc_count: 1, - key: '2023-07-12T13:32:04.174Z,1.8.0,www.elastic.co,test,US,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,info,test message,/app-search', + key: [ + '2023-07-12T13:32:04.174Z', + '1.8.0', + 'www.elastic.co', + 'test', + 'US', + 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', + 'info', + 'test message', + '/app-search', + ], topHitsAgg: { hits: { hits: [ @@ -350,7 +362,18 @@ describe('ESQL query utils', () => { }, { doc_count: 1, - key: '2025-07-12T13:32:04.174Z,1.2.0,400,artifacts.elastic.co,test,US,Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1,info,test message,/app-search', + key: [ + '2025-07-12T13:32:04.174Z', + '1.2.0', + '400', + 'artifacts.elastic.co', + 'test', + 'US', + 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', + 'info', + 'test message', + '/app-search', + ], topHitsAgg: { hits: { hits: [ @@ -387,6 +410,18 @@ describe('ESQL query utils', () => { }, isCountAgg: false, isGroupAgg: true, + termField: [ + '@timestamp', + 'ecs.version', + 'error.code', + 'host', + 'name', + 'geo.dest', + 'agent', + 'tags', + 'message', + 'request', + ], }); expect(longAlertIds?.size).toBe(1); }); @@ -406,7 +441,7 @@ describe('ESQL query utils', () => { buckets: [ { doc_count: 1, - key: '400', + key: ['400'], topHitsAgg: { hits: { hits: [ @@ -432,6 +467,7 @@ describe('ESQL query utils', () => { }, isCountAgg: false, isGroupAgg: true, + termField: ['error.code'], }); expect(rows).toEqual([ { @@ -466,7 +502,7 @@ describe('ESQL query utils', () => { buckets: [ { doc_count: 1, - key: '1.8.0', + key: ['1.8.0'], topHitsAgg: { hits: { hits: [ @@ -485,7 +521,7 @@ describe('ESQL query utils', () => { }, { doc_count: 1, - key: '400,1.2.0', + key: ['400', '1.2.0'], topHitsAgg: { hits: { hits: [ @@ -504,7 +540,7 @@ describe('ESQL query utils', () => { }, { doc_count: 1, - key: '1.2.0', + key: ['1.2.0'], topHitsAgg: { hits: { hits: [ @@ -530,6 +566,7 @@ describe('ESQL query utils', () => { }, isCountAgg: false, isGroupAgg: true, + termField: ['error.code', 'ecs.version'], }); expect(rows).toEqual([ { diff --git a/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.ts b/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.ts index d8904a4b39905..58ae0ccea76f6 100644 --- a/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.ts +++ b/x-pack/platform/plugins/shared/stack_alerts/common/es_query/esql_query_utils.ts @@ -106,6 +106,7 @@ export const toGroupedEsqlQueryHits = ( const duplicateAlertIds: Set = new Set(); const longAlertIds: Set = new Set(); const rows: EsqlDocument[] = []; + const mappedAlertIds: Record> = {}; const groupedHits = table.values.reduce>((acc, row) => { const document = rowToDocument(table.columns, row); const mappedAlertId = alertIdFields.filter((a) => !isNil(document[a])).map((a) => document[a]); @@ -121,6 +122,7 @@ export const toGroupedEsqlQueryHits = ( acc[alertId].push(hit); } else { acc[alertId] = [hit]; + mappedAlertIds[alertId] = mappedAlertId; } rows.push({ [ALERT_ID_COLUMN]: alertId, ...document }); @@ -135,7 +137,7 @@ export const toGroupedEsqlQueryHits = ( groupAgg: { buckets: entries(groupedHits).map(([key, value]) => { return { - key, + key: mappedAlertIds[key], doc_count: value.length, topHitsAgg: { hits: { @@ -158,6 +160,7 @@ export const toGroupedEsqlQueryHits = ( hits: { hits: [] }, aggregations, }, + termField: alertIdFields, }, duplicateAlertIds, longAlertIds, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts index 2ca2f2cfd3d06..861906a1944ab 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts @@ -128,15 +128,17 @@ export default function ruleTests({ getService }: FtrProviderContext) { const messagePattern = /Document count is 1 in the last 30s for group-\d. Alert when greater than 0./; const conditionPattern = /Query matched documents for group "group-\d"/; + const groupPattern = /{"group":"group-\d"}/; for (let i = 0; i < docs.length; i++) { const doc = docs[i]; - const { hits } = doc._source; + const { hits, grouping } = doc._source; const { name, title, message } = doc._source.params; expect(name).to.be('always fire'); expect(title).to.match(titlePattern); expect(message).to.match(messagePattern); expect(hits).not.to.be.empty(); + expect(grouping).to.match(groupPattern); } const aadDocs = await getAllAADDocs(3, ALERT_INSTANCE_ID); @@ -176,15 +178,18 @@ export default function ruleTests({ getService }: FtrProviderContext) { /Query matched documents for group "[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"/; const idPattern = /[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}/; + const groupPattern = + /{"_id":"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"}/; for (let i = 0; i < docs.length; i++) { const doc = docs[i]; - const { hits } = doc._source; + const { hits, grouping } = doc._source; const { name, title, message } = doc._source.params; expect(name).to.be('always fire'); expect(title).to.match(titlePattern); expect(message).to.match(messagePattern); expect(hits).not.to.be.empty(); + expect(grouping).to.match(groupPattern); } const aadDocs = await getAllAADDocs(2); @@ -224,11 +229,23 @@ export default function ruleTests({ getService }: FtrProviderContext) { for (let i = 0; i < docs.length; i++) { const doc = docs[i]; const { hits } = doc._source; + const grouping = JSON.parse(doc._source.grouping); const { name, title, message } = doc._source.params; expect(name).to.be('always fire'); expect(title).to.match(titlePattern); expect(message).to.match(messagePattern); expect(hits).not.to.be.empty(); + expect(grouping['@timestamp']).to.be.ok(); + expect(grouping.date).to.be.ok(); + expect(grouping.date_epoch_millis).to.be.ok(); + expect(grouping.group).to.be.ok(); + expect(grouping.host.hostname).to.be.ok(); + expect(grouping.host.id).to.be.ok(); + expect(grouping.host.name).to.be.ok(); + expect(grouping.reference).to.be.ok(); + expect(grouping.source).to.be.ok(); + expect(grouping.testedValue).to.be.ok(); + expect(grouping.testedValueFloat).to.be.ok(); } const aadDocs = await getAllAADDocs(3); @@ -578,6 +595,7 @@ export default function ruleTests({ getService }: FtrProviderContext) { hits: '{{context.hits}}', date: '{{{context.date}}}', previousTimestamp: '{{{state.latestTimestamp}}}', + grouping: '{{context.grouping}}', }, ], }, @@ -599,6 +617,7 @@ export default function ruleTests({ getService }: FtrProviderContext) { }, hits: '{{context.hits}}', date: '{{{context.date}}}', + grouping: '{{context.grouping}}', }, ], },