[search source] ES Query rule loads fewer fields on query execution

packages/kbn-generate-csv/src/generate_csv.test.ts

@@ -73,6 +73,7 @@ describe('CsvGenerator', () => {
 
   const mockSearchSourceService: jest.Mocked<ISearchStartSearchSource> = {
     create: jest.fn().mockReturnValue(searchSourceMock),
+    createLazy: jest.fn().mockReturnValue(searchSourceMock),
     createEmpty: jest.fn().mockReturnValue(searchSourceMock),
     telemetry: jest.fn(),
     inject: jest.fn(),

src/plugins/data/common/search/search_source/create_search_source.test.ts

@@ -8,7 +8,7 @@
 
 import { createSearchSource as createSearchSourceFactory } from './create_search_source';
 import { SearchSourceDependencies } from './search_source';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewsContract, DataViewLazy } from '@kbn/data-views-plugin/common';
 import type { Filter } from '@kbn/es-query';
 
 describe('createSearchSource', () => {
@@ -24,6 +24,10 @@ describe('createSearchSource', () => {
       search: jest.fn(),
       onResponse: (req, res) => res,
       scriptedFieldsEnabled: true,
+      dataViews: {
+        getMetaFields: jest.fn(),
+        getShortDotsEnable: jest.fn(),
+      } as unknown as DataViewsContract,
     };
 
     indexPatternContractMock = {
@@ -104,4 +108,63 @@ describe('createSearchSource', () => {
       language: 'lucene',
     });
   });
+
+  it('uses DataViews.get', async () => {
+    const dataViewMock: DataView = {
+      toSpec: jest.fn().mockReturnValue(Promise.resolve({})),
+      getSourceFiltering: jest.fn().mockReturnValue({
+        excludes: [],
+      }),
+    } as unknown as DataView;
+    const get = jest.fn().mockReturnValue(Promise.resolve(dataViewMock));
+    const getDataViewLazy = jest.fn();
+    indexPatternContractMock = {
+      get,
+      getDataViewLazy,
+    } as unknown as jest.Mocked<DataViewsContract>;
+
+    createSearchSource = createSearchSourceFactory(indexPatternContractMock, dependencies);
+
+    await createSearchSource({
+      index: '123-456',
+      highlightAll: true,
+      query: {
+        query: '',
+        language: 'kuery',
+      },
+    });
+    expect(get).toHaveBeenCalledWith('123-456');
+    expect(getDataViewLazy).not.toHaveBeenCalled();
+  });
+
+  it('uses DataViews.getDataViewLazy when flag is passed', async () => {
+    const dataViewLazyMock: DataViewLazy = {
+      toSpec: jest.fn().mockReturnValue(Promise.resolve({})),
+      getSourceFiltering: jest.fn().mockReturnValue({
+        excludes: [],
+      }),
+    } as unknown as DataViewLazy;
+    const get = jest.fn();
+    const getDataViewLazy = jest.fn().mockReturnValue(Promise.resolve(dataViewLazyMock));
+    indexPatternContractMock = {
+      get,
+      getDataViewLazy,
+    } as unknown as jest.Mocked<DataViewsContract>;
+
+    createSearchSource = createSearchSourceFactory(indexPatternContractMock, dependencies);
+
+    await createSearchSource(
+      {
+        index: '123-456',
+        highlightAll: true,
+        query: {
+          query: '',
+          language: 'kuery',
+        },
+      },
+      true
+    );
+    expect(get).not.toHaveBeenCalled();
+    expect(getDataViewLazy).toHaveBeenCalledWith('123-456');
+  });
 });

src/plugins/data/common/search/search_source/create_search_source.ts

@@ -6,7 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { DataViewsContract } from '@kbn/data-views-plugin/common';
+import { DataViewsContract, DataView, DataViewLazy } from '@kbn/data-views-plugin/common';
+import { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common';
 import { migrateLegacyQuery } from './migrate_legacy_query';
 import { SearchSource, SearchSourceDependencies } from './search_source';
 import { SerializedSearchSourceFields } from '../..';
@@ -33,18 +34,43 @@ export const createSearchSource = (
   indexPatterns: DataViewsContract,
   searchSourceDependencies: SearchSourceDependencies
 ) => {
-  const createFields = async (searchSourceFields: SerializedSearchSourceFields = {}) => {
+  let dataViewLazy: DataViewLazy | undefined;
+  const createFields = async (
+    searchSourceFields: SerializedSearchSourceFields = {},
+    useDataViewLazy = false
+  ) => {
     const { index, parent, ...restOfFields } = searchSourceFields;
     const fields: SearchSourceFields = {
       ...restOfFields,
     };
 
     // hydrating index pattern
     if (searchSourceFields.index) {
-      if (typeof searchSourceFields.index === 'string') {
-        fields.index = await indexPatterns.get(searchSourceFields.index);
+      if (!useDataViewLazy) {
+        fields.index =
+          typeof searchSourceFields.index === 'string'
+            ? await indexPatterns.get(searchSourceFields.index)
+            : await indexPatterns.create(searchSourceFields.index);
       } else {
-        fields.index = await indexPatterns.create(searchSourceFields.index);
+        dataViewLazy =
+          typeof searchSourceFields.index === 'string'
+            ? await indexPatterns.getDataViewLazy(searchSourceFields.index)
+            : await indexPatterns.createDataViewLazy(searchSourceFields.index);
+
+        const [spec, shortDotsEnable, metaFields] = await Promise.all([
+          dataViewLazy.toSpec(),
+          searchSourceDependencies.dataViews.getShortDotsEnable(),
+          searchSourceDependencies.dataViews.getMetaFields(),
+        ]);
+
+        const dataView = new DataView({
+          spec,
+          // field format functionality is not used within search source
+          fieldFormats: {} as FieldFormatsStartCommon,
+          shortDotsEnable,
+          metaFields,
+        });
+        fields.index = dataView;
       }
     }
 
@@ -55,8 +81,11 @@ export const createSearchSource = (
     return fields;
   };
 
-  const createSearchSourceFn = async (searchSourceFields: SerializedSearchSourceFields = {}) => {
-    const fields = await createFields(searchSourceFields);
+  const createSearchSourceFn = async (
+    searchSourceFields: SerializedSearchSourceFields = {},
+    useDataViewLazy?: boolean
+  ) => {
+    const fields = await createFields(searchSourceFields, !!useDataViewLazy);
     const searchSource = new SearchSource(fields, searchSourceDependencies);
 
     // todo: move to migration script .. create issue
@@ -65,6 +94,11 @@ export const createSearchSource = (
     if (typeof query !== 'undefined') {
       searchSource.setField('query', migrateLegacyQuery(query));
     }
+    // using the dataViewLazy check as a type guard
+    if (useDataViewLazy && dataViewLazy) {
+      const dataViewFields = await searchSource.loadDataViewFields(dataViewLazy);
+      fields.index?.fields.replaceAll(Object.values(dataViewFields).map((fld) => fld.toSpec()));
+    }
 
     return searchSource;
   };

src/plugins/data/common/search/search_source/mocks.ts

@@ -9,6 +9,7 @@
 import { of } from 'rxjs';
 import type { MockedKeys } from '@kbn/utility-types-jest';
 import { uiSettingsServiceMock } from '@kbn/core/public/mocks';
+import { DataViewsContract } from '@kbn/data-views-plugin/common';
 
 import { SearchSource, SearchSourceDependencies } from './search_source';
 import { ISearchStartSearchSource, ISearchSource, SearchSourceFields } from './types';
@@ -37,10 +38,12 @@ export const searchSourceInstanceMock: MockedKeys<ISearchSource> = {
   toExpressionAst: jest.fn(),
   getActiveIndexFilter: jest.fn(),
   parseActiveIndexPatternFromQueryString: jest.fn(),
+  loadDataViewFields: jest.fn(),
 };
 
 export const searchSourceCommonMock: jest.Mocked<ISearchStartSearchSource> = {
   create: jest.fn().mockReturnValue(searchSourceInstanceMock),
+  createLazy: jest.fn().mockReturnValue(searchSourceInstanceMock),
   createEmpty: jest.fn().mockReturnValue(searchSourceInstanceMock),
   telemetry: jest.fn(),
   getAllMigrations: jest.fn(),
@@ -71,4 +74,8 @@ export const createSearchSourceMock = (
       ),
     onResponse: jest.fn().mockImplementation((req, res) => res),
     scriptedFieldsEnabled: true,
+    dataViews: {
+      getMetaFields: jest.fn(),
+      getShortDotsEnable: jest.fn(),
+    } as unknown as DataViewsContract,
   });

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -7,7 +7,7 @@
  */
 
 import Rx, { firstValueFrom, lastValueFrom, of, throwError } from 'rxjs';
-import type { DataView } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
 import { buildExpression, ExpressionAstExpression } from '@kbn/expressions-plugin/common';
 import type { MockedKeys } from '@kbn/utility-types-jest';
 import type { ISearchGeneric } from '@kbn/search-types';
@@ -95,6 +95,10 @@ describe('SearchSource', () => {
       search: mockSearchMethod,
       onResponse: jest.fn().mockImplementation((_, res) => res),
       scriptedFieldsEnabled: true,
+      dataViews: {
+        getMetaFields: jest.fn(),
+        getShortDotsEnable: jest.fn(),
+      } as unknown as jest.Mocked<DataViewsContract>,
     };
 
     searchSource = new SearchSource({}, searchSourceDependencies);

src/plugins/data/common/search/search_source/search_source.ts

@@ -77,13 +77,15 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import {
   buildEsQuery,
   Filter,
+  fromKueryExpression,
   isOfQueryType,
   isPhraseFilter,
   isPhrasesFilter,
+  getKqlFieldNames,
 } from '@kbn/es-query';
 import { fieldWildcardFilter } from '@kbn/kibana-utils-plugin/common';
 import { getHighlightRequest } from '@kbn/field-formats-plugin/common';
-import type { DataView } from '@kbn/data-views-plugin/common';
+import { DataView, DataViewLazy, DataViewsContract } from '@kbn/data-views-plugin/common';
 import {
   ExpressionAstExpression,
   buildExpression,
@@ -134,6 +136,7 @@ export const searchSourceRequiredUiSettings = [
 export interface SearchSourceDependencies extends FetchHandlers {
   aggs: AggsStart;
   search: ISearchGeneric;
+  dataViews: DataViewsContract;
   scriptedFieldsEnabled: boolean;
 }
 
@@ -712,7 +715,7 @@ export class SearchSource {
   }
 
   private readonly getFieldName = (fld: SearchFieldValue): string =>
-    typeof fld === 'string' ? fld : (fld.field as string);
+    typeof fld === 'string' ? fld : (fld?.field as string);
 
   private getFieldsWithoutSourceFilters(
     index: DataView | undefined,
@@ -778,6 +781,47 @@ export class SearchSource {
     return field;
   }
 
+  public async loadDataViewFields(dataView: DataViewLazy) {
+    const request = this.mergeProps(this, { body: {} });
+    let fields = dataView.timeFieldName ? [dataView.timeFieldName] : [];
+    const sort = this.getField('sort');
+    if (sort) {
+      const sortArr = Array.isArray(sort) ? sort : [sort];
+      for (const s of sortArr) {
+        const keys = Object.keys(s);
+        fields = fields.concat(keys);
+      }
+    }
+    for (const query of request.query) {
+      if (query.query) {
+        const nodes = fromKueryExpression(query.query);
+        const queryFields = getKqlFieldNames(nodes);
+        fields = fields.concat(queryFields);
+      }
+    }
+    const filters = request.filters;
+    if (filters) {
+      const filtersArr = Array.isArray(filters) ? filters : [filters];
+      for (const f of filtersArr) {
+        fields = fields.concat(f.meta.key);
+      }
+    }
+    fields = fields.filter((f) => Boolean(f));
+
+    if (dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length) {
+      // if source filtering is enabled, we need to fetch all the fields
+      return (await dataView.getFields({ fieldName: ['*'] })).getFieldMapSorted();
+    } else if (fields.length) {
+      return (
+        await dataView.getFields({
+          fieldName: fields,
+        })
+      ).getFieldMapSorted();
+    }
+    // no fields needed to be loaded for query
+    return {};
+  }
+
   private flatten() {
     const { getConfig } = this.dependencies;
     const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? [];

src/plugins/data/common/search/search_source/search_source_service.test.ts

@@ -20,6 +20,10 @@ describe('SearchSource service', () => {
       search: jest.fn(),
       onResponse: jest.fn(),
       scriptedFieldsEnabled: true,
+      dataViews: {
+        getMetaFields: jest.fn(),
+        getShortDotsEnable: jest.fn(),
+      } as unknown as DataViewsContract,
     };
   });
 
@@ -32,6 +36,7 @@ describe('SearchSource service', () => {
 
       expect(Object.keys(start)).toEqual([
         'create',
+        'createLazy',
         'createEmpty',
         'extract',
         'inject',

src/plugins/data/common/search/search_source/search_source_service.ts

@@ -61,6 +61,10 @@ export class SearchSourceService {
        * creates searchsource based on serialized search source fields
        */
       create: createSearchSource(indexPatterns, dependencies),
+      createLazy: (searchSourceFields: SerializedSearchSourceFields = {}) => {
+        const fn = createSearchSource(indexPatterns, dependencies);
+        return fn(searchSourceFields, true);
+      },
       /**
        * creates an enpty search source
        */

src/plugins/data/common/search/search_source/types.ts

@@ -34,6 +34,8 @@ export interface ISearchStartSearchSource
    * @param fields
    */
   create: (fields?: SerializedSearchSourceFields) => Promise<ISearchSource>;
+
+  createLazy: (fields?: SerializedSearchSourceFields) => Promise<ISearchSource>;
   /**
    * creates empty {@link SearchSource}
    */

src/plugins/data/public/search/search_service.ts

@@ -264,6 +264,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
       aggs,
       getConfig: uiSettings.get.bind(uiSettings),
       search,
+      dataViews: indexPatterns,
       onResponse: (request, response, options) => {
         if (!options.disableWarningToasts) {
           const { rawResponse } = response;

src/plugins/data/server/search/search_service.ts

@@ -316,6 +316,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
             getConfig: <T = any>(key: string): T => uiSettingsCache[key],
             search: this.asScoped(request).search,
             onResponse: (req, res) => res,
+            dataViews: scopedIndexPatterns,
             scriptedFieldsEnabled: true,
           };
 

src/plugins/data_views/common/data_views/data_view_lazy.ts

@@ -108,7 +108,7 @@ export class DataViewLazy extends AbstractDataView {
 
     return {
       getFieldMap: () => fieldMap,
-      getFieldMapSorted: () => {
+      getFieldMapSorted: (): Record<string, DataViewField> => {
         if (!hasBeenSorted) {
           fieldMapSorted = chain(fieldMap).toPairs().sortBy(0).fromPairs().value();
           hasBeenSorted = true;

src/plugins/saved_search/common/service/get_saved_searches.test.ts

@@ -123,6 +123,7 @@ describe('getSavedSearch', () => {
           "getSearchRequestBody": [MockFunction],
           "getSerializedFields": [MockFunction],
           "history": Array [],
+          "loadDataViewFields": [MockFunction],
           "onRequestStart": [MockFunction],
           "parseActiveIndexPatternFromQueryString": [MockFunction],
           "removeField": [MockFunction],
@@ -231,6 +232,7 @@ describe('getSavedSearch', () => {
           "getSearchRequestBody": [MockFunction],
           "getSerializedFields": [MockFunction],
           "history": Array [],
+          "loadDataViewFields": [MockFunction],
           "onRequestStart": [MockFunction],
           "parseActiveIndexPatternFromQueryString": [MockFunction],
           "removeField": [MockFunction],

src/plugins/saved_search/common/service/saved_searches_utils.test.ts

@@ -65,6 +65,10 @@ describe('saved_searches_utils', () => {
               "aggs": Object {
                 "createAggConfigs": [MockFunction],
               },
+              "dataViews": Object {
+                "getMetaFields": [MockFunction],
+                "getShortDotsEnable": [MockFunction],
+              },
               "getConfig": [MockFunction],
               "onResponse": [MockFunction],
               "scriptedFieldsEnabled": true,