[7.x] [Search Source] Request date fields explicitly (#91478)

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -345,8 +345,13 @@ describe('SearchSource', () => {
       });
 
       test('allows you to override computed fields if you provide a format', async () => {
+        const indexPatternFields = indexPattern.fields;
+        indexPatternFields.getByType = (type) => {
+          return [];
+        };
         searchSource.setField('index', ({
           ...indexPattern,
+          fields: indexPatternFields,
           getComputedFields: () => ({
             storedFields: [],
             scriptFields: {},
@@ -379,6 +384,11 @@ describe('SearchSource', () => {
       test('injects a date format for computed docvalue fields while merging other properties', async () => {
         searchSource.setField('index', ({
           ...indexPattern,
+          fields: {
+            getByType: () => {
+              return [];
+            },
+          },
           getComputedFields: () => ({
             storedFields: [],
             scriptFields: {},
@@ -625,7 +635,7 @@ describe('SearchSource', () => {
         searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']);
 
         const request = await searchSource.getSearchRequestBody();
-        expect(request.fields).toEqual(['hello', '@timestamp', 'bar']);
+        expect(request.fields).toEqual(['hello', '@timestamp', 'bar', 'date']);
         expect(request.script_fields).toEqual({ hello: {} });
         expect(request.stored_fields).toEqual(['@timestamp', 'bar']);
       });
@@ -681,6 +691,60 @@ describe('SearchSource', () => {
       });
     });
 
+    describe('handling date fields', () => {
+      test('adds date format to any date field', async () => {
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: '@timestamp' }],
+          }),
+          fields: {
+            getByType: () => [{ name: '@timestamp', esTypes: ['date_nanos'] }],
+          },
+          getSourceFiltering: () => ({ excludes: [] }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['*']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([
+          '*',
+          { field: '@timestamp', format: 'strict_date_optional_time_nanos' },
+        ]);
+      });
+
+      test('adds date format to any date field except the one excluded by source filters', async () => {
+        const indexPatternFields = indexPattern.fields;
+        // @ts-ignore
+        indexPatternFields.getByType = (type) => {
+          return [
+            { name: '@timestamp', esTypes: ['date_nanos'] },
+            { name: 'custom_date', esTypes: ['date'] },
+          ];
+        };
+        searchSource.setField('index', ({
+          ...indexPattern,
+          getComputedFields: () => ({
+            storedFields: [],
+            scriptFields: {},
+            docvalueFields: [{ field: '@timestamp' }, { field: 'custom_date' }],
+          }),
+          fields: indexPatternFields,
+          getSourceFiltering: () => ({ excludes: ['custom_date'] }),
+        } as unknown) as IndexPattern);
+        searchSource.setField('fields', ['*']);
+
+        const request = await searchSource.getSearchRequestBody();
+        expect(request.fields).toEqual([
+          { field: 'foo-bar' },
+          { field: 'field1' },
+          { field: 'field2' },
+          { field: '@timestamp', format: 'strict_date_optional_time_nanos' },
+        ]);
+      });
+    });
+
     describe(`#setField('index')`, () => {
       describe('auto-sourceFiltering', () => {
         describe('new index pattern assigned', () => {

src/plugins/data/common/search/search_source/search_source.ts

@@ -59,7 +59,7 @@
  */
 
 import { setWith } from '@elastic/safer-lodash-set';
-import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual } from 'lodash';
+import { uniqueId, keyBy, pick, difference, omit, isFunction, isEqual, uniqWith } from 'lodash';
 import { map, switchMap, tap } from 'rxjs/operators';
 import { defer, from } from 'rxjs';
 import { isObject } from 'rxjs/internal-compatibility';
@@ -545,6 +545,37 @@ export class SearchSource {
       }));
   }
 
+  private getFieldFromDocValueFieldsOrIndexPattern(
+    docvaluesIndex: Record<string, object>,
+    fld: SearchFieldValue,
+    index?: IndexPattern
+  ) {
+    if (typeof fld === 'string') {
+      return fld;
+    }
+    const fieldName = this.getFieldName(fld);
+    const field = {
+      ...docvaluesIndex[fieldName],
+      ...fld,
+    };
+    if (!index) {
+      return field;
+    }
+    const { fields } = index;
+    const dateFields = fields.getByType('date');
+    const dateField = dateFields.find((indexPatternField) => indexPatternField.name === fieldName);
+    if (!dateField) {
+      return field;
+    }
+    const { esTypes } = dateField;
+    if (esTypes?.includes('date_nanos')) {
+      field.format = 'strict_date_optional_time_nanos';
+    } else if (esTypes?.includes('date')) {
+      field.format = 'strict_date_optional_time';
+    }
+    return field;
+  }
+
   private flatten() {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
@@ -654,22 +685,25 @@ export class SearchSource {
         // if items that are in the docvalueFields are provided, we should
         // inject the format from the computed fields if one isn't given
         const docvaluesIndex = keyBy(filteredDocvalueFields, 'field');
-        body.fields = this.getFieldsWithoutSourceFilters(index, body.fields).map(
-          (fld: SearchFieldValue) => {
-            const fieldName = this.getFieldName(fld);
-            if (Object.keys(docvaluesIndex).includes(fieldName)) {
-              // either provide the field object from computed docvalues,
-              // or merge the user-provided field with the one in docvalues
-              return typeof fld === 'string'
-                ? docvaluesIndex[fld]
-                : {
-                    ...docvaluesIndex[fieldName],
-                    ...fld,
-                  };
-            }
-            return fld;
+        const bodyFields = this.getFieldsWithoutSourceFilters(index, body.fields);
+        body.fields = uniqWith(
+          bodyFields.concat(filteredDocvalueFields),
+          (fld1: SearchFieldValue, fld2: SearchFieldValue) => {
+            const field1Name = this.getFieldName(fld1);
+            const field2Name = this.getFieldName(fld2);
+            return field1Name === field2Name;
           }
-        );
+        ).map((fld: SearchFieldValue) => {
+          const fieldName = this.getFieldName(fld);
+          if (Object.keys(docvaluesIndex).includes(fieldName)) {
+            // either provide the field object from computed docvalues,
+            // or merge the user-provided field with the one in docvalues
+            return typeof fld === 'string'
+              ? docvaluesIndex[fld]
+              : this.getFieldFromDocValueFieldsOrIndexPattern(docvaluesIndex, fld, index);
+          }
+          return fld;
+        });
       }
     } else {
       body.fields = filteredDocvalueFields;