[8.19] [Rules] KQL-to-DSL conversion without data view produces incorrect queries for keyword fields for Metric threshold rule (#260046)

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_rule.ts

@@ -6,6 +6,7 @@
  */
 
 import type { ElasticsearchClient } from '@kbn/core/server';
+import type { DataViewBase } from '@kbn/es-query';
 import moment from 'moment';
 import type { Logger } from '@kbn/logging';
 import { isCustom } from './metric_expression_params';
@@ -44,6 +45,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate
   compositeSize: number,
   alertOnGroupDisappear: boolean,
   logger: Logger,
+  dataView?: DataViewBase,
   lastPeriodEnd?: number,
   timeframe?: { start?: number; end: number },
   missingGroups: MissingGroupsRecord[] = []
@@ -72,6 +74,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate
         alertOnGroupDisappear,
         calculatedTimerange,
         logger,
+        dataView,
         lastPeriodEnd
       );
 

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/lib/get_data.ts

@@ -7,6 +7,7 @@
 
 import type { SearchResponse, AggregationsAggregate } from '@elastic/elasticsearch/lib/api/types';
 import type { ElasticsearchClient } from '@kbn/core/server';
+import type { DataViewBase } from '@kbn/es-query';
 import type { Logger } from '@kbn/logging';
 import type { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common';
 import { COMPARATORS } from '@kbn/alerting-comparators';
@@ -121,6 +122,7 @@ export const getData = async (
   alertOnGroupDisappear: boolean,
   timeframe: { start: number; end: number },
   logger: Logger,
+  dataView?: DataViewBase,
   lastPeriodEnd?: number,
   previousResults: GetDataResponse = {},
   afterKey?: Record<string, string>
@@ -195,6 +197,7 @@ export const getData = async (
           alertOnGroupDisappear,
           timeframe,
           logger,
+          dataView,
           lastPeriodEnd,
           previous,
           nextAfterKey
@@ -275,7 +278,8 @@ export const getData = async (
       groupBy,
       filterQuery,
       afterKey,
-      fieldsExisted
+      fieldsExisted,
+      dataView
     ),
   };
   logger.trace(() => `Request: ${JSON.stringify(request)}`);

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts

@@ -7,6 +7,7 @@
 
 import moment from 'moment';
 import { COMPARATORS } from '@kbn/alerting-comparators';
+import type { DataViewBase } from '@kbn/es-query';
 import type { MetricExpressionParams } from '../../../../../common/alerting/metrics';
 import { Aggregators } from '../../../../../common/alerting/metrics';
 import { getElasticsearchMetricQuery } from './metric_query';
@@ -49,6 +50,61 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
     });
   });
 
+  describe('when using a custom aggregation with a wildcard KQL filter on a keyword field', () => {
+    const dataView: DataViewBase = {
+      title: 'metrics-*',
+      fields: [{ name: 'machine.os.keyword', type: 'string', esTypes: ['keyword'] }],
+    };
+
+    const customParams: MetricExpressionParams = {
+      aggType: Aggregators.CUSTOM,
+      timeUnit: 'm',
+      timeSize: 1,
+      threshold: [0],
+      comparator: COMPARATORS.GREATER_THAN,
+      customMetrics: [
+        {
+          name: 'A',
+          aggType: Aggregators.COUNT,
+          filter: 'machine.os.keyword: *win 7*',
+        },
+      ],
+    };
+
+    const searchBodyWithDataView = getElasticsearchMetricQuery(
+      customParams,
+      timeframe,
+      100,
+      true,
+      void 0,
+      void 0,
+      void 0,
+      void 0,
+      void 0,
+      dataView
+    );
+
+    const searchBodyWithoutDataView = getElasticsearchMetricQuery(
+      customParams,
+      timeframe,
+      100,
+      true
+    );
+
+    test('generates a wildcard query when dataView is provided', () => {
+      const filterAgg =
+        searchBodyWithDataView.aggs.all.aggs.currentPeriod.aggs.aggregatedValue_A.filter;
+      expect(JSON.stringify(filterAgg)).not.toContain('query_string');
+      expect(JSON.stringify(filterAgg)).toContain('wildcard');
+    });
+
+    test('generates a query_string when no dataView is provided', () => {
+      const filterAgg =
+        searchBodyWithoutDataView.aggs.all.aggs.currentPeriod.aggs.aggregatedValue_A.filter;
+      expect(JSON.stringify(filterAgg)).toContain('query_string');
+    });
+  });
+
   describe('when passed a filterQuery', () => {
     const filterQuery =
       // This is adapted from a real-world query that previously broke alerts

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -6,6 +6,7 @@
  */
 
 import moment from 'moment';
+import type { DataViewBase } from '@kbn/es-query';
 import { isCustom, isNotCountOrCustom } from './metric_expression_params';
 import type { MetricExpressionParams } from '../../../../../common/alerting/metrics';
 import { Aggregators } from '../../../../../common/alerting/metrics';
@@ -84,7 +85,8 @@ export const getElasticsearchMetricQuery = (
   groupBy?: string | string[],
   filterQuery?: string,
   afterKey?: Record<string, string>,
-  fieldsExisted?: Record<string, boolean> | null
+  fieldsExisted?: Record<string, boolean> | null,
+  dataView?: DataViewBase
 ) => {
   const { aggType } = metricParams;
   if (isNotCountOrCustom(metricParams) && !metricParams.metric) {
@@ -108,7 +110,8 @@ export const getElasticsearchMetricQuery = (
       ? createCustomMetricsAggregations(
           'aggregatedValue',
           metricParams.customMetrics,
-          metricParams.equation
+          metricParams.equation,
+          dataView
         )
       : {
           aggregatedValue: {

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts

@@ -11,6 +11,7 @@ import { getThresholds } from '../common/get_values';
 import { set } from '@kbn/safer-lodash-set';
 import { COMPARATORS } from '@kbn/alerting-comparators';
 import type {
+  CustomMetricExpressionParams,
   CountMetricExpressionParams,
   NonCountMetricExpressionParams,
 } from '../../../../common/alerting/metrics';
@@ -66,6 +67,15 @@ const mockMetricsExplorerLocator = {
   getRedirectUrl: jest.fn(),
 };
 
+const mockDataView = {
+  title: 'metrics-*,metricbeat-*',
+  fields: [{ name: 'host.name', type: 'string', esTypes: ['keyword'] }],
+};
+
+const mockDataViewsService = {
+  getFieldsForWildcard: jest.fn().mockResolvedValue(mockDataView.fields),
+};
+
 const mockOptions = {
   executionId: '',
   startedAt: mockNow,
@@ -115,7 +125,8 @@ describe('The metric threshold rule type', () => {
     jest.setSystemTime();
   });
   beforeEach(() => {
-    jest.resetAllMocks();
+    jest.clearAllMocks();
+    services.getDataViews.mockResolvedValue(mockDataViewsService);
 
     mockAssetDetailsLocator.getRedirectUrl.mockImplementation(
       ({ assetId, assetType, assetDetails }: AssetDetailsLocatorParams) =>
@@ -306,6 +317,44 @@ describe('The metric threshold rule type', () => {
     });
   });
 
+  describe('data view fetching', () => {
+    test('does not fetch a data view when the rule has no custom count filters', async () => {
+      setEvaluationResults([]);
+
+      await executor({
+        ...mockOptions,
+        services,
+        params: {
+          criteria: [baseNonCountCriterion],
+        },
+      });
+
+      expect(services.getDataViews).not.toHaveBeenCalled();
+      expect(jest.requireMock('./lib/evaluate_rule').evaluateRule.mock.calls[0][6]).toBeUndefined();
+    });
+
+    test('fetches a data view when the rule uses a filtered custom count metric', async () => {
+      setEvaluationResults([]);
+
+      await executor({
+        ...mockOptions,
+        services,
+        params: {
+          criteria: [baseCustomCriterion],
+        },
+      });
+
+      expect(services.getDataViews).toHaveBeenCalledTimes(1);
+      expect(mockDataViewsService.getFieldsForWildcard).toHaveBeenCalledWith({
+        pattern: 'metrics-*,metricbeat-*',
+        allowNoIndex: true,
+      });
+      expect(jest.requireMock('./lib/evaluate_rule').evaluateRule.mock.calls[0][6]).toEqual(
+        mockDataView
+      );
+    });
+  });
+
   describe('querying with a groupBy parameter', () => {
     const execute = (
       comparator: COMPARATORS,
@@ -899,7 +948,7 @@ describe('The metric threshold rule type', () => {
         stateResult2
       );
       expect(stateResult3.missingGroups).toEqual([{ key: 'b', bucketKey: { groupBy0: 'b' } }]);
-      expect(mockedEvaluateRule.mock.calls[2][8]).toEqual([
+      expect(mockedEvaluateRule.mock.calls[2][9]).toEqual([
         { bucketKey: { groupBy0: 'b' }, key: 'b' },
       ]);
     });
@@ -3447,6 +3496,7 @@ const mockLibs: any = {
       return Promise.resolve({
         id: sourceId,
         configuration: {
+          metricAlias: 'metrics-*,metricbeat-*',
           logIndices: {
             type: 'index_pattern',
             indexPatternId: 'some-id',
@@ -3508,3 +3558,18 @@ const baseCountCriterion = {
   threshold: [0],
   comparator: COMPARATORS.GREATER_THAN,
 } as CountMetricExpressionParams;
+
+const baseCustomCriterion = {
+  aggType: Aggregators.CUSTOM,
+  timeSize: 1,
+  timeUnit: 'm',
+  threshold: [0],
+  comparator: COMPARATORS.GREATER_THAN,
+  customMetrics: [
+    {
+      name: 'A',
+      aggType: Aggregators.COUNT,
+      filter: 'host.name: *foo*',
+    },
+  ],
+} as CustomMetricExpressionParams;

x-pack/solutions/observability/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts

@@ -31,9 +31,11 @@ import {
   getGroupByObject,
   type Group,
 } from '@kbn/alerting-rule-utils';
+import type { DataViewBase } from '@kbn/es-query';
 import { convertToBuiltInComparators } from '@kbn/observability-plugin/common/utils/convert_legacy_outside_comparator';
 import { getOriginalActionGroup } from '../../../utils/get_original_action_group';
-import { AlertStates } from '../../../../common/alerting/metrics';
+import { Aggregators, AlertStates } from '../../../../common/alerting/metrics';
+import type { MetricExpressionParams } from '../../../../common/alerting/metrics';
 import { createFormatter } from '../../../../common/formatters';
 import type { InfraBackendLibs, InfraLocators } from '../../infra_types';
 import {
@@ -59,6 +61,7 @@ import type { EvaluatedRuleParams, Evaluation } from './lib/evaluate_rule';
 import { evaluateRule } from './lib/evaluate_rule';
 import type { MissingGroupsRecord } from './lib/check_missing_group';
 import { convertStringsToMissingGroupsRecord } from './lib/convert_strings_to_missing_groups_record';
+import { isCustom } from './lib/metric_expression_params';
 
 export type MetricThresholdAlert = Omit<
   ObservabilityMetricsAlert,
@@ -256,13 +259,31 @@ export const createMetricThresholdExecutor =
           )
         : [];
 
+    let dataView: DataViewBase | undefined;
+    if (shouldCreateDataView(criteria)) {
+      const dataViewsService = await services.getDataViews();
+      try {
+        const fields = await dataViewsService.getFieldsForWildcard({
+          pattern: config.metricAlias,
+          allowNoIndex: true,
+        });
+        dataView = {
+          title: config.metricAlias,
+          fields,
+        };
+      } catch (e) {
+        // ignore — dataView stays undefined and toElasticsearchQuery degrades gracefully
+      }
+    }
+
     const alertResults = await evaluateRule(
       services.scopedClusterClient.asCurrentUser,
       params as EvaluatedRuleParams,
       config,
       compositeSize,
       alertOnGroupDisappear,
       logger,
+      dataView,
       state.lastRunTimestamp,
       { end: startedAt.valueOf() },
       convertStringsToMissingGroupsRecord(previousMissingGroups)
@@ -560,6 +581,17 @@ const mapToConditionsLookup = (
     return result;
   }, {} as Record<string, unknown>);
 
+const shouldCreateDataView = (criteria: MetricExpressionParams[]) =>
+  criteria.some((criterion) => {
+    if (!isCustom(criterion)) {
+      return false;
+    }
+
+    return criterion.customMetrics.some(
+      (customMetric) => customMetric.aggType === Aggregators.COUNT && customMetric.filter != null
+    );
+  });
+
 const formatAlertResult = <AlertResult>(
   alertResult: {
     metric: string;