[ML] AIOps: Fixes issue where some queries cause filters to not be applied

x-pack/plugins/aiops/public/application/utils/search_utils.ts

@@ -13,23 +13,14 @@ import type { IUiSettingsClient } from '@kbn/core/public';
 import { getEsQueryConfig, SearchSource } from '@kbn/data-plugin/common';
 import type { SavedSearch } from '@kbn/saved-search-plugin/public';
 import type { FilterManager } from '@kbn/data-plugin/public';
-import { isQuery, mapAndFlattenFilters } from '@kbn/data-plugin/public';
-import type { Query, Filter, AggregateQuery } from '@kbn/es-query';
-import {
-  fromKueryExpression,
-  toElasticsearchQuery,
-  buildQueryFromFilters,
-  buildEsQuery,
-} from '@kbn/es-query';
+import { mapAndFlattenFilters } from '@kbn/data-plugin/public';
+import type { Query, Filter } from '@kbn/es-query';
+import { buildEsQuery } from '@kbn/es-query';
 import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { SimpleSavedObject } from '@kbn/core/public';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
-import {
-  getDefaultDSLQuery,
-  type SearchQueryLanguage,
-  SEARCH_QUERY_LANGUAGE,
-} from '@kbn/ml-query-utils';
+import { getDefaultDSLQuery, type SearchQueryLanguage } from '@kbn/ml-query-utils';
 
 export type SavedSearchSavedObject = SimpleSavedObject<any>;
 
@@ -67,51 +58,6 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObjec
   return parsed;
 }
 
-/**
- * Create an Elasticsearch query that combines both lucene/kql query string and filters
- * Should also form a valid query if only the query or filters is provided
- */
-export function createMergedEsQuery(
-  query?: Query | AggregateQuery,
-  filters?: Filter[],
-  dataView?: DataView,
-  uiSettings?: IUiSettingsClient
-) {
-  let combinedQuery: QueryDslQueryContainer = getDefaultDSLQuery();
-
-  // FIXME: Add support for AggregateQuery type #150091
-  if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) {
-    const ast = fromKueryExpression(query.query);
-    if (query.query !== '') {
-      combinedQuery = toElasticsearchQuery(ast, dataView);
-    }
-    if (combinedQuery.bool !== undefined) {
-      const filterQuery = buildQueryFromFilters(filters, dataView);
-
-      if (!Array.isArray(combinedQuery.bool.filter)) {
-        combinedQuery.bool.filter =
-          combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter];
-      }
-
-      if (!Array.isArray(combinedQuery.bool.must_not)) {
-        combinedQuery.bool.must_not =
-          combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not];
-      }
-
-      combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter];
-      combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not];
-    }
-  } else {
-    combinedQuery = buildEsQuery(
-      dataView,
-      query ? [query] : [],
-      filters ? filters : [],
-      uiSettings ? getEsQueryConfig(uiSettings) : undefined
-    );
-  }
-  return combinedQuery;
-}
-
 function getSavedSearchSource(savedSearch: SavedSearch) {
   return savedSearch &&
     'searchSource' in savedSearch &&
@@ -174,11 +120,11 @@ export function getEsQueryFromSavedSearch({
   if (!savedSearch && userQuery) {
     if (filterManager && userFilters) filterManager.addFilters(userFilters);
 
-    const combinedQuery = createMergedEsQuery(
+    const combinedQuery = buildEsQuery(
+      dataView,
       userQuery,
       Array.isArray(userFilters) ? userFilters : [],
-      dataView,
-      uiSettings
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
     );
 
     return {
@@ -199,11 +145,11 @@ export function getEsQueryFromSavedSearch({
     if (filterManager) filterManager.setFilters(currentFilters);
     if (globalFilters) filterManager?.addFilters(globalFilters);
 
-    const combinedQuery = createMergedEsQuery(
+    const combinedQuery = buildEsQuery(
+      dataView,
       currentQuery,
       filterManager ? filterManager?.getFilters() : currentFilters,
-      dataView,
-      uiSettings
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
     );
 
     return {

x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx

@@ -9,20 +9,18 @@ import type { FC, PropsWithChildren } from 'react';
 import React, { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { type DataViewField } from '@kbn/data-views-plugin/public';
 import { startWith } from 'rxjs';
-import type { Filter, Query } from '@kbn/es-query';
+import { buildEsQuery, type Filter, type Query } from '@kbn/es-query';
 import { usePageUrlState } from '@kbn/ml-url-state';
 import { useTimefilter } from '@kbn/ml-date-picker';
 import { ES_FIELD_TYPES } from '@kbn/field-types';
 import { type QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
 import type { TimeBuckets, TimeBucketsInterval } from '@kbn/ml-time-buckets';
 import { useTimeBuckets } from '@kbn/ml-time-buckets';
 import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
+import { getEsQueryConfig } from '@kbn/data-service';
 import { useFilterQueryUpdates } from '../../hooks/use_filters_query';
 import { type ChangePointType, DEFAULT_AGG_FUNCTION } from './constants';
-import {
-  createMergedEsQuery,
-  getEsQueryFromSavedSearch,
-} from '../../application/utils/search_utils';
+import { getEsQueryFromSavedSearch } from '../../application/utils/search_utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { useDataSource } from '../../hooks/use_data_source';
 
@@ -261,7 +259,12 @@ export const ChangePointDetectionContextProvider: FC<PropsWithChildren<unknown>>
   );
 
   const combinedQuery = useMemo(() => {
-    const mergedQuery = createMergedEsQuery(resultQuery, resultFilters, dataView, uiSettings);
+    const mergedQuery = buildEsQuery(
+      dataView,
+      resultQuery,
+      resultFilters,
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
+    );
     const to = searchBounds.max?.valueOf();
     const from = searchBounds.min?.valueOf();
     const timeRange = to !== undefined && from !== undefined ? { from, to } : undefined;

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx

@@ -12,7 +12,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, useEuiPaddingSize } from '@elasti
 import type { DataViewField } from '@kbn/data-views-plugin/public';
 import { i18n } from '@kbn/i18n';
 import type { Filter } from '@kbn/es-query';
-import { buildEmptyFilter } from '@kbn/es-query';
+import { buildEmptyFilter, buildEsQuery } from '@kbn/es-query';
 import { usePageUrlState } from '@kbn/ml-url-state';
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 
@@ -24,11 +24,11 @@ import type { EmbeddablePatternAnalysisInput } from '@kbn/aiops-log-pattern-anal
 import { css } from '@emotion/react';
 import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state';
 import useMountedState from 'react-use/lib/useMountedState';
+import { getEsQueryConfig } from '@kbn/data-service';
 import {
   type LogCategorizationPageUrlState,
   getDefaultLogCategorizationAppState,
 } from '../../../application/url_state/log_pattern_analysis';
-import { createMergedEsQuery } from '../../../application/utils/search_utils';
 import { useData } from '../../../hooks/use_data';
 import { useSearch } from '../../../hooks/use_search';
 import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
@@ -90,7 +90,12 @@ export const LogCategorizationDiscover: FC<LogCategorizationEmbeddableProps> = (
   const [stateFromUrl] = usePageUrlState<LogCategorizationPageUrlState>(
     'logCategorization',
     getDefaultLogCategorizationAppState({
-      searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings),
+      searchQuery: buildEsQuery(
+        dataView,
+        query ?? [],
+        filters ?? [],
+        uiSettings ? getEsQueryConfig(uiSettings) : undefined
+      ),
     })
   );
   const [selectedCategories, setSelectedCategories] = useState<Category[]>([]);

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx

@@ -10,7 +10,7 @@ import React, { useState, useEffect, useCallback } from 'react';
 
 import { i18n } from '@kbn/i18n';
 import type { Filter } from '@kbn/es-query';
-import { buildEmptyFilter } from '@kbn/es-query';
+import { buildEmptyFilter, buildEsQuery } from '@kbn/es-query';
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 
 import type { Category } from '@kbn/aiops-log-pattern-analysis/types';
@@ -21,11 +21,11 @@ import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state';
 import { AIOPS_ANALYSIS_RUN_ORIGIN } from '@kbn/aiops-common/constants';
 import datemath from '@elastic/datemath';
 import useMountedState from 'react-use/lib/useMountedState';
+import { getEsQueryConfig } from '@kbn/data-service';
 import { useFilterQueryUpdates } from '../../../hooks/use_filters_query';
 import type { PatternAnalysisProps } from '../../../shared_components/pattern_analysis';
 import { useSearch } from '../../../hooks/use_search';
 import { getDefaultLogCategorizationAppState } from '../../../application/url_state/log_pattern_analysis';
-import { createMergedEsQuery } from '../../../application/utils/search_utils';
 import { useData } from '../../../hooks/use_data';
 import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
 
@@ -86,7 +86,12 @@ export const LogCategorizationEmbeddable: FC<LogCategorizationEmbeddableProps> =
   });
 
   const appState = getDefaultLogCategorizationAppState({
-    searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings),
+    searchQuery: buildEsQuery(
+      dataView,
+      query ?? [],
+      filters ?? [],
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
+    ),
     filters,
   });
   const { searchQuery } = useSearch({ dataView, savedSearch: savedSearch ?? null }, appState, true);

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx

@@ -33,11 +33,12 @@ import type { CategorizationAdditionalFilter } from '@kbn/aiops-log-pattern-anal
 import type { Category } from '@kbn/aiops-log-pattern-analysis/types';
 
 import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state';
+import { buildEsQuery } from '@kbn/es-query';
+import { getEsQueryConfig } from '@kbn/data-service';
 import {
   type LogCategorizationPageUrlState,
   getDefaultLogCategorizationAppState,
 } from '../../application/url_state/log_pattern_analysis';
-import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
@@ -100,7 +101,12 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
   const [stateFromUrl] = usePageUrlState<LogCategorizationPageUrlState>(
     'logCategorization',
     getDefaultLogCategorizationAppState({
-      searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings),
+      searchQuery: buildEsQuery(
+        dataView,
+        query ?? [],
+        filters ?? [],
+        uiSettings ? getEsQueryConfig(uiSettings) : undefined
+      ),
     })
   );
   const [selectedCategories, setSelectedCategories] = useState<Category[]>([]);

x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx

@@ -9,13 +9,13 @@ import type { FC } from 'react';
 import React, { useEffect, useState } from 'react';
 import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import type { Query, Filter } from '@kbn/es-query';
+import { type Query, type Filter, buildEsQuery } from '@kbn/es-query';
 import type { TimeRange } from '@kbn/es-query';
 import type { DataViewField } from '@kbn/data-views-plugin/public';
 import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { getEsQueryConfig } from '@kbn/data-service';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { useDataSource } from '../../hooks/use_data_source';
-import { createMergedEsQuery } from '../../application/utils/search_utils';
 interface Props {
   searchString: Query['query'];
   searchQuery: Query['query'];
@@ -66,11 +66,11 @@ export const SearchPanel: FC<Props> = ({ searchString, searchQueryLanguage, setS
         queryManager.filterManager.setFilters(mergedFilters);
       }
 
-      const combinedQuery = createMergedEsQuery(
+      const combinedQuery = buildEsQuery(
+        dataView,
         mergedQuery,
         queryManager.filterManager.getFilters() ?? [],
-        dataView,
-        uiSettings
+        uiSettings ? getEsQueryConfig(uiSettings) : undefined
       );
 
       setSearchParams({

x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx

@@ -9,6 +9,8 @@ import type { FC } from 'react';
 import React, { useEffect, useMemo } from 'react';
 import { css } from '@emotion/react';
 import { CHANGE_POINT_DETECTION_VIEW_TYPE } from '@kbn/aiops-change-point-detection/constants';
+import { getEsQueryConfig } from '@kbn/data-service';
+import { buildEsQuery } from '@kbn/es-query';
 import type { ChangePointDetectionProps } from '../../shared_components/change_point_detection';
 import { ChangePointsTable } from '../../components/change_point_detection/change_points_table';
 import {
@@ -18,7 +20,6 @@ import {
 import { useFilterQueryUpdates } from '../../hooks/use_filters_query';
 import { useDataSource } from '../../hooks/use_data_source';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
-import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { useChangePointResults } from '../../components/change_point_detection/use_change_point_agg_request';
 import { ChartsGrid } from '../../components/change_point_detection/charts_grid';
 import { NoChangePointsWarning } from '../../components/change_point_detection/no_change_points_warning';
@@ -62,15 +63,13 @@ export const ChartGridEmbeddableWrapper: FC<ChangePointDetectionProps> = ({
   const { uiSettings } = useAiopsAppContext();
 
   const combinedQuery = useMemo(() => {
-    const mergedQuery = createMergedEsQuery(query, filters, dataView, uiSettings);
-    if (!Array.isArray(mergedQuery.bool?.filter)) {
-      if (!mergedQuery.bool) {
-        mergedQuery.bool = {};
-      }
-      mergedQuery.bool.filter = [];
-    }
-
-    mergedQuery.bool!.filter.push({
+    const mergedQuery = buildEsQuery(
+      dataView,
+      query,
+      filters,
+      uiSettings ? getEsQueryConfig(uiSettings) : undefined
+    );
+    mergedQuery.bool.filter.push({
       range: {
         [dataView.timeFieldName!]: {
           from: searchBounds.min?.valueOf(),

x-pack/plugins/aiops/public/hooks/use_search.ts

@@ -9,14 +9,14 @@ import { useMemo } from 'react';
 
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { SavedSearch } from '@kbn/saved-search-plugin/public';
-import { isQuery } from '@kbn/data-plugin/public';
+import { getEsQueryConfig, isQuery } from '@kbn/data-plugin/public';
 
+import { buildEsQuery } from '@kbn/es-query';
 import { getEsQueryFromSavedSearch } from '../application/utils/search_utils';
 import {
   isDefaultSearchQuery,
   type AiOpsIndexBasedAppState,
 } from '../application/url_state/common';
-import { createMergedEsQuery } from '../application/utils/search_utils';
 
 import { useAiopsAppContext } from './use_aiops_app_context';
 
@@ -66,7 +66,12 @@ export const useSearch = (
         (isDefaultSearchQuery(searchQuery) || searchQuery === undefined) &&
         isQuery(query)
       ) {
-        searchQuery = createMergedEsQuery(query, aiopsListState.filters, dataView, uiSettings);
+        searchQuery = buildEsQuery(
+          dataView,
+          query,
+          aiopsListState.filters ?? [],
+          uiSettings ? getEsQueryConfig(uiSettings) : undefined
+        );
       }
 
       return {