diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts index 94ba544e4c96f..1d85ffcb1f714 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/stack_schema.ts @@ -11,6 +11,7 @@ import * as rt from 'io-ts'; import { Either } from 'fp-ts/lib/Either'; import { AlertSchema } from './alert_schema'; +import { EcsSchema } from './ecs_schema'; const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/; export const IsoDateString = new rt.Type( 'IsoDateString', @@ -77,6 +78,6 @@ const StackAlertOptional = rt.partial({ }); // prettier-ignore -export const StackAlertSchema = rt.intersection([StackAlertRequired, StackAlertOptional, AlertSchema]); +export const StackAlertSchema = rt.intersection([StackAlertRequired, StackAlertOptional, AlertSchema, EcsSchema]); // prettier-ignore export type StackAlert = rt.TypeOf; diff --git a/x-pack/plugins/stack_alerts/public/rule_types/es_query/constants.ts b/x-pack/plugins/stack_alerts/public/rule_types/es_query/constants.ts index 7d01043bfbc21..e40f373cb95fc 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/es_query/constants.ts +++ b/x-pack/plugins/stack_alerts/public/rule_types/es_query/constants.ts @@ -22,7 +22,6 @@ export const DEFAULT_VALUES = { GROUP_BY: 'all', EXCLUDE_PREVIOUS_HITS: false, CAN_SELECT_MULTI_TERMS: true, - SOURCE_FIELDS: [], }; export const SERVERLESS_DEFAULT_VALUES = { SIZE: 10, diff --git a/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx b/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx index febae77ad0f22..3ff2b70522e9a 100644 --- a/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx +++ b/x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx @@ -6,7 +6,7 @@ */ import React, { useState, Fragment, useEffect, useCallback } from 'react'; -import { debounce, get } from 'lodash'; +import { get } from 'lodash'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiFieldNumber, @@ -16,7 +16,6 @@ import { EuiSelect, EuiSpacer, } from '@elastic/eui'; -import { getESQLQueryColumns } from '@kbn/esql-utils'; import { getFields, RuleTypeParamsExpressionProps } from '@kbn/triggers-actions-ui-plugin/public'; import { TextBasedLangEditor } from '@kbn/text-based-languages/public'; import { fetchFieldsFromESQL } from '@kbn/text-based-editor'; @@ -24,14 +23,12 @@ import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils'; import type { AggregateQuery } from '@kbn/es-query'; import { parseDuration } from '@kbn/alerting-plugin/common'; import { - FieldOption, firstFieldOption, getTimeFieldOptions, getTimeOptions, parseAggregationResults, } from '@kbn/triggers-actions-ui-plugin/public/common'; import { DataView } from '@kbn/data-views-plugin/common'; -import { SourceFields } from '../../components/source_fields_select'; import { EsQueryRuleParams, EsQueryRuleMetaData, SearchType } from '../types'; import { DEFAULT_VALUES, SERVERLESS_DEFAULT_VALUES } from '../constants'; import { useTriggerUiActionServices } from '../util'; @@ -42,8 +39,8 @@ import { rowToDocument, toEsQueryHits, transformDatatableToEsqlTable } from '../ export const EsqlQueryExpression: React.FC< RuleTypeParamsExpressionProps, EsQueryRuleMetaData> > = ({ ruleParams, setRuleParams, setRuleProperty, errors }) => { - const { expressions, http, fieldFormats, isServerless, data } = useTriggerUiActionServices(); - const { esqlQuery, timeWindowSize, timeWindowUnit, timeField, sourceFields } = ruleParams; + const { expressions, http, fieldFormats, isServerless } = useTriggerUiActionServices(); + const { esqlQuery, timeWindowSize, timeWindowUnit, timeField } = ruleParams; const [currentRuleParams, setCurrentRuleParams] = useState< EsQueryRuleParams @@ -61,12 +58,12 @@ export const EsqlQueryExpression: React.FC< groupBy: DEFAULT_VALUES.GROUP_BY, termSize: DEFAULT_VALUES.TERM_SIZE, searchType: SearchType.esqlQuery, - sourceFields: sourceFields ?? DEFAULT_VALUES.SOURCE_FIELDS, + // The sourceFields param is ignored for the ES|QL type + sourceFields: [], }); const [query, setQuery] = useState({ esql: '' }); const [timeFieldOptions, setTimeFieldOptions] = useState([firstFieldOption]); const [detectTimestamp, setDetectTimestamp] = useState(false); - const [esFields, setEsFields] = useState([]); const [isLoading, setIsLoading] = useState(false); const setParam = useCallback( @@ -86,7 +83,6 @@ export const EsqlQueryExpression: React.FC< if (esqlQuery) { if (esqlQuery.esql) { refreshTimeFields(esqlQuery); - refreshEsFields(esqlQuery, false); } } if (timeField) { @@ -180,32 +176,6 @@ export const EsqlQueryExpression: React.FC< setDetectTimestamp(hasTimestamp); }; - const refreshEsFields = async (q: AggregateQuery, resetSourceFields: boolean = true) => { - let fields: FieldOption[] = []; - try { - const columns = await getESQLQueryColumns({ - esqlQuery: `${get(q, 'esql')}`, - search: data.search.search, - }); - if (columns.length) { - fields = columns.map((c) => ({ - name: c.id, - type: c.meta.type, - normalizedType: c.meta.type, - searchable: true, - aggregatable: true, - })); - } - } catch (error) { - /** ignore error */ - } - - if (resetSourceFields) { - setParam('sourceFields', undefined); - } - setEsFields(fields); - }; - return ( { + onTextLangQueryChange={(q: AggregateQuery) => { setQuery(q); setParam('esqlQuery', q); refreshTimeFields(q); - refreshEsFields(q); - }, 1000)} + }} expandCodeEditor={() => true} isCodeEditorExpanded={true} onTextLangQuerySubmit={async () => {}} @@ -236,14 +205,6 @@ export const EsqlQueryExpression: React.FC< isLoading={isLoading} /> - - setParam('sourceFields', selectedSourceFields) - } - esFields={esFields} - sourceFields={sourceFields} - errors={errors.sourceFields} - /> = { }, }, shouldWrite: true, + useEcs: true, }; diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.test.ts index 1d7096d20140e..b23cddc0eaab7 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.test.ts @@ -7,7 +7,7 @@ import { OnlyEsqlQueryRuleParams } from '../types'; import { Comparator } from '../../../../common/comparator_types'; -import { getEsqlQuery } from './fetch_esql_query'; +import { getEsqlQuery, getSourceFields } from './fetch_esql_query'; const getTimeRange = () => { const date = Date.now(); @@ -98,4 +98,30 @@ describe('fetchEsqlQuery', () => { `); }); }); + + describe('getSourceFields', () => { + it('should generate the correct source fields', async () => { + const sourceFields = getSourceFields({ + columns: [ + { name: '@timestamp', type: 'date' }, + { name: 'ecs.version', type: 'keyword' }, + { name: 'error.code', type: 'keyword' }, + ], + values: [['2023-07-12T13:32:04.174Z', '1.8.0', null]], + }); + + expect(sourceFields).toMatchInlineSnapshot(` + Array [ + Object { + "label": "ecs.version", + "searchPath": "ecs.version", + }, + Object { + "label": "error.code", + "searchPath": "error.code", + }, + ] + `); + }); + }); }); diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts index 87ae2c1123547..5ecb258b0579f 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_esql_query.ts @@ -5,9 +5,11 @@ * 2.0. */ +import { intersectionBy } from 'lodash'; import { parseAggregationResults } from '@kbn/triggers-actions-ui-plugin/common'; import { SharePluginStart } from '@kbn/share-plugin/server'; import { IScopedClusterClient, Logger } from '@kbn/core/server'; +import { ecsFieldMap, alertFieldMap } from '@kbn/alerts-as-data-utils'; import { OnlyEsqlQueryRuleParams } from '../types'; import { EsqlTable, toEsQueryHits } from '../../../../common'; @@ -47,6 +49,8 @@ export async function fetchEsqlQuery({ path: '/_query', body: query, }); + const hits = toEsQueryHits(response); + const sourceFields = getSourceFields(response); const link = `${publicBaseUrl}${spacePrefix}/app/management/insightsAndAlerting/triggersActions/rule/${ruleId}`; @@ -60,10 +64,10 @@ export async function fetchEsqlQuery({ took: 0, timed_out: false, _shards: { failed: 0, successful: 0, total: 0 }, - hits: toEsQueryHits(response), + hits, }, resultLimit: alertLimit, - sourceFieldsParams: params.sourceFields, + sourceFieldsParams: sourceFields, generateSourceFieldsFromHits: true, }), index: null, @@ -98,3 +102,17 @@ export const getEsqlQuery = ( }; return query; }; + +export const getSourceFields = (results: EsqlTable) => { + const resultFields = results.columns.map((c) => ({ + label: c.name, + searchPath: c.name, + })); + const alertFields = Object.keys(alertFieldMap); + const ecsFields = Object.keys(ecsFieldMap) + // exclude the alert fields that we don't want to override + .filter((key) => !alertFields.includes(key)) + .map((key) => ({ label: key, searchPath: key })); + + return intersectionBy(resultFields, ecsFields, 'label'); +}; diff --git a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts index e22d4959c6093..b881aa7609954 100644 --- a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts +++ b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.test.ts @@ -843,9 +843,9 @@ describe('parseAggregationResults', () => { sampleEsqlSourceFieldsHit, ], sourceFields: { - 'host.hostname': ['host-1'], - 'host.id': ['1'], - 'host.name': ['host-1'], + 'host.hostname': ['host-1', 'host-1', 'host-1', 'host-1'], + 'host.id': ['1', '1', '1', '1'], + 'host.name': ['host-1', 'host-1', 'host-1', 'host-1'], }, }, ], diff --git a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts index 6fe7040abacc3..756f79dceec97 100644 --- a/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts +++ b/x-pack/plugins/triggers_actions_ui/common/data/lib/parse_aggregation_results.ts @@ -87,10 +87,10 @@ export const parseAggregationResults = ({ sourceFieldsParams.forEach((field) => { if (generateSourceFieldsFromHits) { - const fieldsSet = new Set(); + const fieldsSet: string[] = []; groupBucket.topHitsAgg.hits.hits.forEach((hit: SearchHit<{ [key: string]: string }>) => { if (hit._source && hit._source[field.label]) { - fieldsSet.add(hit._source[field.label]); + fieldsSet.push(hit._source[field.label]); } }); sourceFields[field.label] = Array.from(fieldsSet); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts index f193af1b81703..56934c5d5f8a3 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/builtin_alert_types/es_query/esql_only.ts @@ -22,7 +22,6 @@ import { RULE_INTERVAL_MILLIS, RULE_INTERVAL_SECONDS, RULE_TYPE_ID, - SourceField, } from './common'; import { createDataStream, deleteDataStream } from '../../../create_test_data'; @@ -39,12 +38,6 @@ export default function ruleTests({ getService }: FtrProviderContext) { getAllAADDocs, } = getRuleServices(getService); - const sourceFields = [ - { label: 'host.hostname', searchPath: 'host.hostname.keyword' }, - { label: 'host.id', searchPath: 'host.id' }, - { label: 'host.name', searchPath: 'host.name' }, - ]; - describe('rule', async () => { let endDate: string; let connectorId: string; @@ -81,13 +74,11 @@ export default function ruleTests({ getService }: FtrProviderContext) { name: 'never fire', esqlQuery: 'from .kibana-alerting-test-data | stats c = count(date) by host.hostname, host.name, host.id | where c < 0', - sourceFields, }); await createRule({ name: 'always fire', esqlQuery: 'from .kibana-alerting-test-data | stats c = count(date) by host.hostname, host.name, host.id | where c > -1', - sourceFields, }); const docs = await waitForDocs(2); @@ -225,13 +216,11 @@ export default function ruleTests({ getService }: FtrProviderContext) { name: 'never fire', esqlQuery: 'from test-data-stream | stats c = count(@timestamp) by host.hostname, host.name, host.id | where c < 0', - sourceFields, }); await createRule({ name: 'always fire', esqlQuery: 'from test-data-stream | stats c = count(@timestamp) by host.hostname, host.name, host.id | where c > -1', - sourceFields, }); const messagePattern = /Document count is \d+ in the last 20s. Alert when greater than 0./; @@ -397,7 +386,6 @@ export default function ruleTests({ getService }: FtrProviderContext) { groupBy?: string; termField?: string; termSize?: number; - sourceFields?: SourceField[]; } async function createRule(params: CreateRuleParams): Promise { @@ -469,7 +457,7 @@ export default function ruleTests({ getService }: FtrProviderContext) { termSize: params.termSize, timeField: params.timeField || 'date', esqlQuery: { esql: params.esqlQuery }, - sourceFields: params.sourceFields, + sourceFields: [], }, }) .expect(200); diff --git a/x-pack/test/api_integration/apis/maps/maps_telemetry.ts b/x-pack/test/api_integration/apis/maps/maps_telemetry.ts index 8f5c9ae95f8af..acd846f77c982 100644 --- a/x-pack/test/api_integration/apis/maps/maps_telemetry.ts +++ b/x-pack/test/api_integration/apis/maps/maps_telemetry.ts @@ -36,8 +36,8 @@ export default function ({ getService }: FtrProviderContext) { return fieldStat.name === 'geo_point'; } ); - expect(geoPointFieldStats.count).to.be(39); - expect(geoPointFieldStats.index_count).to.be(10); + expect(geoPointFieldStats.count).to.be(47); + expect(geoPointFieldStats.index_count).to.be(11); const geoShapeFieldStats = apiResponse.cluster_stats.indices.mappings.field_types.find( (fieldStat: estypes.ClusterStatsFieldTypes) => {