diff --git a/packages/kbn-generate-csv/src/generate_csv.test.ts b/packages/kbn-generate-csv/src/generate_csv.test.ts index 5123e75d8bf68..5a419ef4741c6 100644 --- a/packages/kbn-generate-csv/src/generate_csv.test.ts +++ b/packages/kbn-generate-csv/src/generate_csv.test.ts @@ -73,6 +73,7 @@ describe('CsvGenerator', () => { const mockSearchSourceService: jest.Mocked = { create: jest.fn().mockReturnValue(searchSourceMock), + createLazy: jest.fn().mockReturnValue(searchSourceMock), createEmpty: jest.fn().mockReturnValue(searchSourceMock), telemetry: jest.fn(), inject: jest.fn(), diff --git a/src/plugins/data/common/search/search_source/create_search_source.test.ts b/src/plugins/data/common/search/search_source/create_search_source.test.ts index 709b065c1ec41..916b503cc4cad 100644 --- a/src/plugins/data/common/search/search_source/create_search_source.test.ts +++ b/src/plugins/data/common/search/search_source/create_search_source.test.ts @@ -8,7 +8,7 @@ import { createSearchSource as createSearchSourceFactory } from './create_search_source'; import { SearchSourceDependencies } from './search_source'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, DataViewLazy } from '@kbn/data-views-plugin/common'; import type { Filter } from '@kbn/es-query'; describe('createSearchSource', () => { @@ -24,6 +24,10 @@ describe('createSearchSource', () => { search: jest.fn(), onResponse: (req, res) => res, scriptedFieldsEnabled: true, + dataViews: { + getMetaFields: jest.fn(), + getShortDotsEnable: jest.fn(), + } as unknown as DataViewsContract, }; indexPatternContractMock = { @@ -104,4 +108,63 @@ describe('createSearchSource', () => { language: 'lucene', }); }); + + it('uses DataViews.get', async () => { + const dataViewMock: DataView = { + toSpec: jest.fn().mockReturnValue(Promise.resolve({})), + getSourceFiltering: jest.fn().mockReturnValue({ + excludes: [], + }), + } as unknown as DataView; + const get = jest.fn().mockReturnValue(Promise.resolve(dataViewMock)); + const getDataViewLazy = jest.fn(); + indexPatternContractMock = { + get, + getDataViewLazy, + } as unknown as jest.Mocked; + + createSearchSource = createSearchSourceFactory(indexPatternContractMock, dependencies); + + await createSearchSource({ + index: '123-456', + highlightAll: true, + query: { + query: '', + language: 'kuery', + }, + }); + expect(get).toHaveBeenCalledWith('123-456'); + expect(getDataViewLazy).not.toHaveBeenCalled(); + }); + + it('uses DataViews.getDataViewLazy when flag is passed', async () => { + const dataViewLazyMock: DataViewLazy = { + toSpec: jest.fn().mockReturnValue(Promise.resolve({})), + getSourceFiltering: jest.fn().mockReturnValue({ + excludes: [], + }), + } as unknown as DataViewLazy; + const get = jest.fn(); + const getDataViewLazy = jest.fn().mockReturnValue(Promise.resolve(dataViewLazyMock)); + indexPatternContractMock = { + get, + getDataViewLazy, + } as unknown as jest.Mocked; + + createSearchSource = createSearchSourceFactory(indexPatternContractMock, dependencies); + + await createSearchSource( + { + index: '123-456', + highlightAll: true, + query: { + query: '', + language: 'kuery', + }, + }, + true + ); + expect(get).not.toHaveBeenCalled(); + expect(getDataViewLazy).toHaveBeenCalledWith('123-456'); + }); }); diff --git a/src/plugins/data/common/search/search_source/create_search_source.ts b/src/plugins/data/common/search/search_source/create_search_source.ts index 77dfcff156ab7..ecee9a34c454f 100644 --- a/src/plugins/data/common/search/search_source/create_search_source.ts +++ b/src/plugins/data/common/search/search_source/create_search_source.ts @@ -6,7 +6,8 @@ * Side Public License, v 1. */ -import { DataViewsContract } from '@kbn/data-views-plugin/common'; +import { DataViewsContract, DataView, DataViewLazy } from '@kbn/data-views-plugin/common'; +import { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common'; import { migrateLegacyQuery } from './migrate_legacy_query'; import { SearchSource, SearchSourceDependencies } from './search_source'; import { SerializedSearchSourceFields } from '../..'; @@ -33,7 +34,11 @@ export const createSearchSource = ( indexPatterns: DataViewsContract, searchSourceDependencies: SearchSourceDependencies ) => { - const createFields = async (searchSourceFields: SerializedSearchSourceFields = {}) => { + let dataViewLazy: DataViewLazy | undefined; + const createFields = async ( + searchSourceFields: SerializedSearchSourceFields = {}, + useDataViewLazy = false + ) => { const { index, parent, ...restOfFields } = searchSourceFields; const fields: SearchSourceFields = { ...restOfFields, @@ -41,10 +46,31 @@ export const createSearchSource = ( // hydrating index pattern if (searchSourceFields.index) { - if (typeof searchSourceFields.index === 'string') { - fields.index = await indexPatterns.get(searchSourceFields.index); + if (!useDataViewLazy) { + fields.index = + typeof searchSourceFields.index === 'string' + ? await indexPatterns.get(searchSourceFields.index) + : await indexPatterns.create(searchSourceFields.index); } else { - fields.index = await indexPatterns.create(searchSourceFields.index); + dataViewLazy = + typeof searchSourceFields.index === 'string' + ? await indexPatterns.getDataViewLazy(searchSourceFields.index) + : await indexPatterns.createDataViewLazy(searchSourceFields.index); + + const [spec, shortDotsEnable, metaFields] = await Promise.all([ + dataViewLazy.toSpec(), + searchSourceDependencies.dataViews.getShortDotsEnable(), + searchSourceDependencies.dataViews.getMetaFields(), + ]); + + const dataView = new DataView({ + spec, + // field format functionality is not used within search source + fieldFormats: {} as FieldFormatsStartCommon, + shortDotsEnable, + metaFields, + }); + fields.index = dataView; } } @@ -55,8 +81,11 @@ export const createSearchSource = ( return fields; }; - const createSearchSourceFn = async (searchSourceFields: SerializedSearchSourceFields = {}) => { - const fields = await createFields(searchSourceFields); + const createSearchSourceFn = async ( + searchSourceFields: SerializedSearchSourceFields = {}, + useDataViewLazy?: boolean + ) => { + const fields = await createFields(searchSourceFields, !!useDataViewLazy); const searchSource = new SearchSource(fields, searchSourceDependencies); // todo: move to migration script .. create issue @@ -65,6 +94,11 @@ export const createSearchSource = ( if (typeof query !== 'undefined') { searchSource.setField('query', migrateLegacyQuery(query)); } + // using the dataViewLazy check as a type guard + if (useDataViewLazy && dataViewLazy) { + const dataViewFields = await searchSource.loadDataViewFields(dataViewLazy); + fields.index?.fields.replaceAll(Object.values(dataViewFields).map((fld) => fld.toSpec())); + } return searchSource; }; diff --git a/src/plugins/data/common/search/search_source/mocks.ts b/src/plugins/data/common/search/search_source/mocks.ts index a3ffa23c2819f..3889e593ac869 100644 --- a/src/plugins/data/common/search/search_source/mocks.ts +++ b/src/plugins/data/common/search/search_source/mocks.ts @@ -9,6 +9,7 @@ import { of } from 'rxjs'; import type { MockedKeys } from '@kbn/utility-types-jest'; import { uiSettingsServiceMock } from '@kbn/core/public/mocks'; +import { DataViewsContract } from '@kbn/data-views-plugin/common'; import { SearchSource, SearchSourceDependencies } from './search_source'; import { ISearchStartSearchSource, ISearchSource, SearchSourceFields } from './types'; @@ -37,10 +38,12 @@ export const searchSourceInstanceMock: MockedKeys = { toExpressionAst: jest.fn(), getActiveIndexFilter: jest.fn(), parseActiveIndexPatternFromQueryString: jest.fn(), + loadDataViewFields: jest.fn(), }; export const searchSourceCommonMock: jest.Mocked = { create: jest.fn().mockReturnValue(searchSourceInstanceMock), + createLazy: jest.fn().mockReturnValue(searchSourceInstanceMock), createEmpty: jest.fn().mockReturnValue(searchSourceInstanceMock), telemetry: jest.fn(), getAllMigrations: jest.fn(), @@ -71,4 +74,8 @@ export const createSearchSourceMock = ( ), onResponse: jest.fn().mockImplementation((req, res) => res), scriptedFieldsEnabled: true, + dataViews: { + getMetaFields: jest.fn(), + getShortDotsEnable: jest.fn(), + } as unknown as DataViewsContract, }); diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 9f66e55be9f21..a28a863989af7 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -7,7 +7,7 @@ */ import Rx, { firstValueFrom, lastValueFrom, of, throwError } from 'rxjs'; -import type { DataView } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; import { buildExpression, ExpressionAstExpression } from '@kbn/expressions-plugin/common'; import type { MockedKeys } from '@kbn/utility-types-jest'; import type { ISearchGeneric } from '@kbn/search-types'; @@ -95,6 +95,10 @@ describe('SearchSource', () => { search: mockSearchMethod, onResponse: jest.fn().mockImplementation((_, res) => res), scriptedFieldsEnabled: true, + dataViews: { + getMetaFields: jest.fn(), + getShortDotsEnable: jest.fn(), + } as unknown as jest.Mocked, }; searchSource = new SearchSource({}, searchSourceDependencies); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index bcedc77846d50..97d545d51e960 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -77,13 +77,15 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { buildEsQuery, Filter, + fromKueryExpression, isOfQueryType, isPhraseFilter, isPhrasesFilter, + getKqlFieldNames, } from '@kbn/es-query'; import { fieldWildcardFilter } from '@kbn/kibana-utils-plugin/common'; import { getHighlightRequest } from '@kbn/field-formats-plugin/common'; -import type { DataView } from '@kbn/data-views-plugin/common'; +import { DataView, DataViewLazy, DataViewsContract } from '@kbn/data-views-plugin/common'; import { ExpressionAstExpression, buildExpression, @@ -134,6 +136,7 @@ export const searchSourceRequiredUiSettings = [ export interface SearchSourceDependencies extends FetchHandlers { aggs: AggsStart; search: ISearchGeneric; + dataViews: DataViewsContract; scriptedFieldsEnabled: boolean; } @@ -712,7 +715,7 @@ export class SearchSource { } private readonly getFieldName = (fld: SearchFieldValue): string => - typeof fld === 'string' ? fld : (fld.field as string); + typeof fld === 'string' ? fld : (fld?.field as string); private getFieldsWithoutSourceFilters( index: DataView | undefined, @@ -778,6 +781,47 @@ export class SearchSource { return field; } + public async loadDataViewFields(dataView: DataViewLazy) { + const request = this.mergeProps(this, { body: {} }); + let fields = dataView.timeFieldName ? [dataView.timeFieldName] : []; + const sort = this.getField('sort'); + if (sort) { + const sortArr = Array.isArray(sort) ? sort : [sort]; + for (const s of sortArr) { + const keys = Object.keys(s); + fields = fields.concat(keys); + } + } + for (const query of request.query) { + if (query.query) { + const nodes = fromKueryExpression(query.query); + const queryFields = getKqlFieldNames(nodes); + fields = fields.concat(queryFields); + } + } + const filters = request.filters; + if (filters) { + const filtersArr = Array.isArray(filters) ? filters : [filters]; + for (const f of filtersArr) { + fields = fields.concat(f.meta.key); + } + } + fields = fields.filter((f) => Boolean(f)); + + if (dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length) { + // if source filtering is enabled, we need to fetch all the fields + return (await dataView.getFields({ fieldName: ['*'] })).getFieldMapSorted(); + } else if (fields.length) { + return ( + await dataView.getFields({ + fieldName: fields, + }) + ).getFieldMapSorted(); + } + // no fields needed to be loaded for query + return {}; + } + private flatten() { const { getConfig } = this.dependencies; const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? []; diff --git a/src/plugins/data/common/search/search_source/search_source_service.test.ts b/src/plugins/data/common/search/search_source/search_source_service.test.ts index 8e03f56fe0421..49ffa85cee09a 100644 --- a/src/plugins/data/common/search/search_source/search_source_service.test.ts +++ b/src/plugins/data/common/search/search_source/search_source_service.test.ts @@ -20,6 +20,10 @@ describe('SearchSource service', () => { search: jest.fn(), onResponse: jest.fn(), scriptedFieldsEnabled: true, + dataViews: { + getMetaFields: jest.fn(), + getShortDotsEnable: jest.fn(), + } as unknown as DataViewsContract, }; }); @@ -32,6 +36,7 @@ describe('SearchSource service', () => { expect(Object.keys(start)).toEqual([ 'create', + 'createLazy', 'createEmpty', 'extract', 'inject', diff --git a/src/plugins/data/common/search/search_source/search_source_service.ts b/src/plugins/data/common/search/search_source/search_source_service.ts index 986689cfe3d73..3305c0dde166b 100644 --- a/src/plugins/data/common/search/search_source/search_source_service.ts +++ b/src/plugins/data/common/search/search_source/search_source_service.ts @@ -61,6 +61,10 @@ export class SearchSourceService { * creates searchsource based on serialized search source fields */ create: createSearchSource(indexPatterns, dependencies), + createLazy: (searchSourceFields: SerializedSearchSourceFields = {}) => { + const fn = createSearchSource(indexPatterns, dependencies); + return fn(searchSourceFields, true); + }, /** * creates an enpty search source */ diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts index 2bd38cb8c00f6..4499e14dac67a 100644 --- a/src/plugins/data/common/search/search_source/types.ts +++ b/src/plugins/data/common/search/search_source/types.ts @@ -34,6 +34,8 @@ export interface ISearchStartSearchSource * @param fields */ create: (fields?: SerializedSearchSourceFields) => Promise; + + createLazy: (fields?: SerializedSearchSourceFields) => Promise; /** * creates empty {@link SearchSource} */ diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 10d84a8db93a1..0bdc33cb59303 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -264,6 +264,7 @@ export class SearchService implements Plugin { aggs, getConfig: uiSettings.get.bind(uiSettings), search, + dataViews: indexPatterns, onResponse: (request, response, options) => { if (!options.disableWarningToasts) { const { rawResponse } = response; diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 8d43e28b108e7..4de7cec4dade6 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -316,6 +316,7 @@ export class SearchService implements Plugin { getConfig: (key: string): T => uiSettingsCache[key], search: this.asScoped(request).search, onResponse: (req, res) => res, + dataViews: scopedIndexPatterns, scriptedFieldsEnabled: true, }; diff --git a/src/plugins/data_views/common/data_views/data_view_lazy.ts b/src/plugins/data_views/common/data_views/data_view_lazy.ts index a6aee7a33f3ba..0fbb7f2ae21e0 100644 --- a/src/plugins/data_views/common/data_views/data_view_lazy.ts +++ b/src/plugins/data_views/common/data_views/data_view_lazy.ts @@ -108,7 +108,7 @@ export class DataViewLazy extends AbstractDataView { return { getFieldMap: () => fieldMap, - getFieldMapSorted: () => { + getFieldMapSorted: (): Record => { if (!hasBeenSorted) { fieldMapSorted = chain(fieldMap).toPairs().sortBy(0).fromPairs().value(); hasBeenSorted = true; diff --git a/src/plugins/saved_search/common/service/get_saved_searches.test.ts b/src/plugins/saved_search/common/service/get_saved_searches.test.ts index 5b4462c77aa07..6572d6addbbbb 100644 --- a/src/plugins/saved_search/common/service/get_saved_searches.test.ts +++ b/src/plugins/saved_search/common/service/get_saved_searches.test.ts @@ -123,6 +123,7 @@ describe('getSavedSearch', () => { "getSearchRequestBody": [MockFunction], "getSerializedFields": [MockFunction], "history": Array [], + "loadDataViewFields": [MockFunction], "onRequestStart": [MockFunction], "parseActiveIndexPatternFromQueryString": [MockFunction], "removeField": [MockFunction], @@ -231,6 +232,7 @@ describe('getSavedSearch', () => { "getSearchRequestBody": [MockFunction], "getSerializedFields": [MockFunction], "history": Array [], + "loadDataViewFields": [MockFunction], "onRequestStart": [MockFunction], "parseActiveIndexPatternFromQueryString": [MockFunction], "removeField": [MockFunction], diff --git a/src/plugins/saved_search/common/service/saved_searches_utils.test.ts b/src/plugins/saved_search/common/service/saved_searches_utils.test.ts index 3972f38caa5b5..ce8c073dd27c9 100644 --- a/src/plugins/saved_search/common/service/saved_searches_utils.test.ts +++ b/src/plugins/saved_search/common/service/saved_searches_utils.test.ts @@ -65,6 +65,10 @@ describe('saved_searches_utils', () => { "aggs": Object { "createAggConfigs": [MockFunction], }, + "dataViews": Object { + "getMetaFields": [MockFunction], + "getShortDotsEnable": [MockFunction], + }, "getConfig": [MockFunction], "onResponse": [MockFunction], "scriptedFieldsEnabled": true, diff --git a/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts b/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts index 812ff0ece7d45..b1a030bb6d3ca 100644 --- a/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts +++ b/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts @@ -219,6 +219,7 @@ describe('getSavedSearchAttributeService', () => { "getSearchRequestBody": [MockFunction], "getSerializedFields": [MockFunction], "history": Array [], + "loadDataViewFields": [MockFunction], "onRequestStart": [MockFunction], "parseActiveIndexPatternFromQueryString": [MockFunction], "removeField": [MockFunction], diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.test.ts index f20492c37e911..e64cd443fadf6 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.test.ts @@ -84,7 +84,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { searchSource, filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource( searchSourceInstance, dataViewMock, params, @@ -124,7 +124,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { searchSource, filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource( searchSourceInstance, dataViewMock, params, @@ -189,7 +189,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { searchSource, filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource( searchSourceInstance, dataViewMock, params, @@ -229,7 +229,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { searchSource, filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource( searchSourceInstance, dataViewMock, params, @@ -275,7 +275,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { searchSource } = updateSearchSource( + const { searchSource } = await updateSearchSource( searchSourceInstance, dataViewMock, params, @@ -346,7 +346,7 @@ describe('fetchSearchSourceQuery', () => { const searchSourceInstance = createSearchSourceMock({ index: dataViewMock }); const { dateStart, dateEnd } = getTimeRange(); - const { filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { filterToExcludeHitsFromPreviousRun } = await updateSearchSource( searchSourceInstance, dataViewMock, params, diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts index 29564ad4b3863..e0f6613fb5a10 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts @@ -56,11 +56,10 @@ export async function fetchSearchSourceQuery({ const { logger, searchSourceClient } = services; const isGroupAgg = isGroupAggregation(params.termField); const isCountAgg = isCountAggregation(params.aggType); - - const initialSearchSource = await searchSourceClient.create(params.searchConfiguration); + const initialSearchSource = await searchSourceClient.createLazy(params.searchConfiguration); const index = initialSearchSource.getField('index') as DataView; - const { searchSource, filterToExcludeHitsFromPreviousRun } = updateSearchSource( + const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource( initialSearchSource, index, params, @@ -102,7 +101,7 @@ export async function fetchSearchSourceQuery({ }; } -export function updateSearchSource( +export async function updateSearchSource( searchSource: ISearchSource, index: DataView, params: OnlySearchSourceRuleParams, @@ -110,9 +109,9 @@ export function updateSearchSource( dateStart: string, dateEnd: string, alertLimit?: number -): { searchSource: ISearchSource; filterToExcludeHitsFromPreviousRun: Filter | null } { +): Promise<{ searchSource: ISearchSource; filterToExcludeHitsFromPreviousRun: Filter | null }> { const isGroupAgg = isGroupAggregation(params.termField); - const timeField = index.getTimeField(); + const timeField = await index.getTimeField(); if (!timeField) { throw new Error(`Data view with ID ${index.id} no longer contains a time field.`);