diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index 321a102e8d782..b721c9157fe16 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -29,3 +29,4 @@ export const CONTEXT_STEP_SETTING = 'context:step'; export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields'; export const DOC_TABLE_LEGACY = 'doc_table:legacy'; export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch'; +export const SEARCH_FIELDS_FROM_SOURCE = 'discover:searchFieldsFromSource'; diff --git a/src/fixtures/stubbed_saved_object_index_pattern.ts b/src/plugins/discover/public/__mocks__/stubbed_saved_object_index_pattern.ts similarity index 94% rename from src/fixtures/stubbed_saved_object_index_pattern.ts rename to src/plugins/discover/public/__mocks__/stubbed_saved_object_index_pattern.ts index 261e451db5452..a85734edba274 100644 --- a/src/fixtures/stubbed_saved_object_index_pattern.ts +++ b/src/plugins/discover/public/__mocks__/stubbed_saved_object_index_pattern.ts @@ -18,7 +18,7 @@ */ // @ts-expect-error -import stubbedLogstashFields from './logstash_fields'; +import stubbedLogstashFields from '../../../../fixtures/logstash_fields'; const mockLogstashFields = stubbedLogstashFields(); diff --git a/src/plugins/discover/public/application/angular/context/api/_stubs.js b/src/plugins/discover/public/application/angular/context/api/_stubs.js index d82189db60935..17d45756af148 100644 --- a/src/plugins/discover/public/application/angular/context/api/_stubs.js +++ b/src/plugins/discover/public/application/angular/context/api/_stubs.js @@ -47,6 +47,7 @@ export function createSearchSourceStub(hits, timeField) { searchSourceStub.setParent = sinon.spy(() => searchSourceStub); searchSourceStub.setField = sinon.spy(() => searchSourceStub); + searchSourceStub.removeField = sinon.spy(() => searchSourceStub); searchSourceStub.getField = sinon.spy((key) => { const previousSetCall = searchSourceStub.setField.withArgs(key).lastCall; diff --git a/src/plugins/discover/public/application/angular/context/api/anchor.js b/src/plugins/discover/public/application/angular/context/api/anchor.js index 4df5ba989f798..31c106b95cbe5 100644 --- a/src/plugins/discover/public/application/angular/context/api/anchor.js +++ b/src/plugins/discover/public/application/angular/context/api/anchor.js @@ -20,7 +20,7 @@ import _ from 'lodash'; import { i18n } from '@kbn/i18n'; -export function fetchAnchorProvider(indexPatterns, searchSource) { +export function fetchAnchorProvider(indexPatterns, searchSource, useNewFieldsApi = false) { return async function fetchAnchor(indexPatternId, anchorId, sort) { const indexPattern = await indexPatterns.get(indexPatternId); searchSource @@ -41,7 +41,10 @@ export function fetchAnchorProvider(indexPatterns, searchSource) { language: 'lucene', }) .setField('sort', sort); - + if (useNewFieldsApi) { + searchSource.removeField('fieldsFromSource'); + searchSource.setField('fields', ['*']); + } const response = await searchSource.fetch(); if (_.get(response, ['hits', 'total'], 0) < 1) { diff --git a/src/plugins/discover/public/application/angular/context/api/anchor.test.js b/src/plugins/discover/public/application/angular/context/api/anchor.test.js index 993aefc4f59e3..d54b38c466a5c 100644 --- a/src/plugins/discover/public/application/angular/context/api/anchor.test.js +++ b/src/plugins/discover/public/application/angular/context/api/anchor.test.js @@ -144,4 +144,29 @@ describe('context app', function () { }); }); }); + + describe('useNewFields API', () => { + let fetchAnchor; + let searchSourceStub; + + beforeEach(() => { + searchSourceStub = createSearchSourceStub([{ _id: 'hit1' }]); + fetchAnchor = fetchAnchorProvider(createIndexPatternsStub(), searchSourceStub, true); + }); + + it('should request fields if useNewFieldsApi set', function () { + searchSourceStub._stubHits = [{ property1: 'value1' }, { property2: 'value2' }]; + + return fetchAnchor('INDEX_PATTERN_ID', 'id', [ + { '@timestamp': 'desc' }, + { _doc: 'desc' }, + ]).then(() => { + const setFieldsSpy = searchSourceStub.setField.withArgs('fields'); + const removeFieldsSpy = searchSourceStub.removeField.withArgs('fieldsFromSource'); + expect(setFieldsSpy.calledOnce).toBe(true); + expect(removeFieldsSpy.calledOnce).toBe(true); + expect(setFieldsSpy.firstCall.args[1]).toEqual(['*']); + }); + }); + }); }); diff --git a/src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js b/src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js index 4c0515906a494..ea181782470fa 100644 --- a/src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js +++ b/src/plugins/discover/public/application/angular/context/api/context.predecessors.test.js @@ -227,4 +227,81 @@ describe('context app', function () { }); }); }); + + describe('function fetchPredecessors with useNewFieldsApi set', function () { + let fetchPredecessors; + let mockSearchSource; + + beforeEach(() => { + mockSearchSource = createContextSearchSourceStub([], '@timestamp', MS_PER_DAY * 8); + + setServices({ + data: { + search: { + searchSource: { + create: jest.fn().mockImplementation(() => mockSearchSource), + }, + }, + }, + }); + + fetchPredecessors = ( + indexPatternId, + timeField, + sortDir, + timeValIso, + timeValNr, + tieBreakerField, + tieBreakerValue, + size + ) => { + const anchor = { + _source: { + [timeField]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], + }; + + return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs( + 'predecessors', + indexPatternId, + anchor, + timeField, + tieBreakerField, + sortDir, + size, + [] + ); + }; + }); + + it('should perform exactly one query when enough hits are returned', function () { + mockSearchSource._stubHits = [ + mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 2), + mockSearchSource._createStubHit(MS_PER_DAY * 3000 + 1), + mockSearchSource._createStubHit(MS_PER_DAY * 3000), + mockSearchSource._createStubHit(MS_PER_DAY * 2000), + mockSearchSource._createStubHit(MS_PER_DAY * 1000), + ]; + + return fetchPredecessors( + 'INDEX_PATTERN_ID', + '@timestamp', + 'desc', + ANCHOR_TIMESTAMP_3000, + MS_PER_DAY * 3000, + '_doc', + 0, + 3, + [] + ).then((hits) => { + const setFieldsSpy = mockSearchSource.setField.withArgs('fields'); + const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource'); + expect(mockSearchSource.fetch.calledOnce).toBe(true); + expect(removeFieldsSpy.calledOnce).toBe(true); + expect(setFieldsSpy.calledOnce).toBe(true); + expect(hits).toEqual(mockSearchSource._stubHits.slice(0, 3)); + }); + }); + }); }); diff --git a/src/plugins/discover/public/application/angular/context/api/context.successors.test.js b/src/plugins/discover/public/application/angular/context/api/context.successors.test.js index 285d39cd4d8a4..2c54de946c8d4 100644 --- a/src/plugins/discover/public/application/angular/context/api/context.successors.test.js +++ b/src/plugins/discover/public/application/angular/context/api/context.successors.test.js @@ -231,4 +231,81 @@ describe('context app', function () { }); }); }); + + describe('function fetchSuccessors with useNewFieldsApi set', function () { + let fetchSuccessors; + let mockSearchSource; + + beforeEach(() => { + mockSearchSource = createContextSearchSourceStub([], '@timestamp'); + + setServices({ + data: { + search: { + searchSource: { + create: jest.fn().mockImplementation(() => mockSearchSource), + }, + }, + }, + }); + + fetchSuccessors = ( + indexPatternId, + timeField, + sortDir, + timeValIso, + timeValNr, + tieBreakerField, + tieBreakerValue, + size + ) => { + const anchor = { + _source: { + [timeField]: timeValIso, + }, + sort: [timeValNr, tieBreakerValue], + }; + + return fetchContextProvider(createIndexPatternsStub(), true).fetchSurroundingDocs( + 'successors', + indexPatternId, + anchor, + timeField, + tieBreakerField, + sortDir, + size, + [] + ); + }; + }); + + it('should perform exactly one query when enough hits are returned', function () { + mockSearchSource._stubHits = [ + mockSearchSource._createStubHit(MS_PER_DAY * 5000), + mockSearchSource._createStubHit(MS_PER_DAY * 4000), + mockSearchSource._createStubHit(MS_PER_DAY * 3000), + mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 1), + mockSearchSource._createStubHit(MS_PER_DAY * 3000 - 2), + ]; + + return fetchSuccessors( + 'INDEX_PATTERN_ID', + '@timestamp', + 'desc', + ANCHOR_TIMESTAMP_3000, + MS_PER_DAY * 3000, + '_doc', + 0, + 3, + [] + ).then((hits) => { + expect(mockSearchSource.fetch.calledOnce).toBe(true); + expect(hits).toEqual(mockSearchSource._stubHits.slice(-3)); + const setFieldsSpy = mockSearchSource.setField.withArgs('fields'); + const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource'); + expect(removeFieldsSpy.calledOnce).toBe(true); + expect(setFieldsSpy.calledOnce).toBe(true); + }); + }); + }); }); diff --git a/src/plugins/discover/public/application/angular/context/api/context.ts b/src/plugins/discover/public/application/angular/context/api/context.ts index ba8cffd1d7558..903e4e0f1b485 100644 --- a/src/plugins/discover/public/application/angular/context/api/context.ts +++ b/src/plugins/discover/public/application/angular/context/api/context.ts @@ -40,7 +40,7 @@ const DAY_MILLIS = 24 * 60 * 60 * 1000; // look from 1 day up to 10000 days into the past and future const LOOKUP_OFFSETS = [0, 1, 7, 30, 365, 10000].map((days) => days * DAY_MILLIS); -function fetchContextProvider(indexPatterns: IndexPatternsContract) { +function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFieldsApi?: boolean) { return { fetchSurroundingDocs, }; @@ -89,7 +89,14 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) { break; } - const searchAfter = getEsQuerySearchAfter(type, documents, timeField, anchor, nanos); + const searchAfter = getEsQuerySearchAfter( + type, + documents, + timeField, + anchor, + nanos, + useNewFieldsApi + ); const sort = getEsQuerySort(timeField, tieBreakerField, sortDirToApply); @@ -116,6 +123,10 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract) { const { data } = getServices(); const searchSource = await data.search.searchSource.create(); + if (useNewFieldsApi) { + searchSource.removeField('fieldsFromSource'); + searchSource.setField('fields', ['*']); + } return searchSource .setParent(undefined) .setField('index', indexPattern) diff --git a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts index 24ac19a7e3bc3..348a0c04a84ad 100644 --- a/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts +++ b/src/plugins/discover/public/application/angular/context/api/utils/get_es_query_search_after.ts @@ -31,16 +31,30 @@ export function getEsQuerySearchAfter( documents: EsHitRecordList, timeFieldName: string, anchor: EsHitRecord, - nanoSeconds: string + nanoSeconds: string, + useNewFieldsApi?: boolean ): EsQuerySearchAfter { if (documents.length) { // already surrounding docs -> first or last record is used const afterTimeRecIdx = type === 'successors' && documents.length ? documents.length - 1 : 0; const afterTimeDoc = documents[afterTimeRecIdx]; - const afterTimeValue = nanoSeconds ? afterTimeDoc._source[timeFieldName] : afterTimeDoc.sort[0]; + let afterTimeValue = afterTimeDoc.sort[0]; + if (nanoSeconds) { + afterTimeValue = useNewFieldsApi + ? afterTimeDoc.fields[timeFieldName][0] + : afterTimeDoc._source[timeFieldName]; + } return [afterTimeValue, afterTimeDoc.sort[1]]; } // if data_nanos adapt timestamp value for sorting, since numeric value was rounded by browser // ES search_after also works when number is provided as string - return [nanoSeconds ? anchor._source[timeFieldName] : anchor.sort[0], anchor.sort[1]]; + const searchAfter = new Array(2) as EsQuerySearchAfter; + searchAfter[0] = anchor.sort[0]; + if (nanoSeconds) { + searchAfter[0] = useNewFieldsApi + ? anchor.fields[timeFieldName][0] + : anchor._source[timeFieldName]; + } + searchAfter[1] = anchor.sort[1]; + return searchAfter; } diff --git a/src/plugins/discover/public/application/angular/context/query/actions.js b/src/plugins/discover/public/application/angular/context/query/actions.js index d5c72d34006e2..42638cd90a1bb 100644 --- a/src/plugins/discover/public/application/angular/context/query/actions.js +++ b/src/plugins/discover/public/application/angular/context/query/actions.js @@ -27,11 +27,17 @@ import { fetchContextProvider } from '../api/context'; import { getQueryParameterActions } from '../query_parameters'; import { FAILURE_REASONS, LOADING_STATUS } from './index'; import { MarkdownSimple } from '../../../../../../kibana_react/public'; +import { SEARCH_FIELDS_FROM_SOURCE } from '../../../../../common'; export function QueryActionsProvider(Promise) { - const { filterManager, indexPatterns, data } = getServices(); - const fetchAnchor = fetchAnchorProvider(indexPatterns, data.search.searchSource.createEmpty()); - const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns); + const { filterManager, indexPatterns, data, uiSettings } = getServices(); + const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); + const fetchAnchor = fetchAnchorProvider( + indexPatterns, + data.search.searchSource.createEmpty(), + useNewFieldsApi + ); + const { fetchSurroundingDocs } = fetchContextProvider(indexPatterns, useNewFieldsApi); const { setPredecessorCount, setQueryParameters, setSuccessorCount } = getQueryParameterActions( filterManager, indexPatterns diff --git a/src/plugins/discover/public/application/angular/context_app.html b/src/plugins/discover/public/application/angular/context_app.html index 8dc3e5c87e504..3d731459ad8d7 100644 --- a/src/plugins/discover/public/application/angular/context_app.html +++ b/src/plugins/discover/public/application/angular/context_app.html @@ -17,5 +17,6 @@ successor-available="contextApp.state.rows.successors.length" successor-status="contextApp.state.loadingStatus.successors.status" on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows" + use-new-fields-api="contextApp.state.useNewFieldsApi" top-nav-menu="contextApp.topNavMenu" > diff --git a/src/plugins/discover/public/application/angular/context_app.js b/src/plugins/discover/public/application/angular/context_app.js index d9e2452eb8bd6..f18389df6d12d 100644 --- a/src/plugins/discover/public/application/angular/context_app.js +++ b/src/plugins/discover/public/application/angular/context_app.js @@ -18,7 +18,11 @@ */ import _ from 'lodash'; -import { CONTEXT_STEP_SETTING, CONTEXT_TIE_BREAKER_FIELDS_SETTING } from '../../../common'; +import { + CONTEXT_STEP_SETTING, + CONTEXT_TIE_BREAKER_FIELDS_SETTING, + SEARCH_FIELDS_FROM_SOURCE, +} from '../../../common'; import { getAngularModule, getServices } from '../../kibana_services'; import contextAppTemplate from './context_app.html'; import './context/components/action_bar'; @@ -59,9 +63,11 @@ function ContextAppController($scope, Private) { const { filterManager, indexPatterns, uiSettings, navigation } = getServices(); const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns); const queryActions = Private(QueryActionsProvider); + const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); this.state = createInitialState( parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10), - getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)) + getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)), + useNewFieldsApi ); this.topNavMenu = navigation.ui.TopNavMenu; @@ -127,7 +133,7 @@ function ContextAppController($scope, Private) { ); } -function createInitialState(defaultStepSize, tieBreakerField) { +function createInitialState(defaultStepSize, tieBreakerField, useNewFieldsApi) { return { queryParameters: createInitialQueryParametersState(defaultStepSize, tieBreakerField), rows: { @@ -137,5 +143,6 @@ function createInitialState(defaultStepSize, tieBreakerField) { successors: [], }, loadingStatus: createInitialLoadingStatusState(), + useNewFieldsApi, }; } diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index de3b7c6c1a326..6b552d92df0f0 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -64,6 +64,7 @@ import { DEFAULT_COLUMNS_SETTING, MODIFY_COLUMNS_ON_SWITCH, SAMPLE_SIZE_SETTING, + SEARCH_FIELDS_FROM_SOURCE, SEARCH_ON_PAGE_LOAD_SETTING, SORT_DEFAULT_ORDER_SETTING, } from '../../../common'; @@ -197,6 +198,8 @@ function discoverController($element, $route, $scope, $timeout, Promise) { $scope.searchSource, toastNotifications ); + $scope.useNewFieldsApi = !config.get(SEARCH_FIELDS_FROM_SOURCE); + //used for functional testing $scope.fetchCounter = 0; @@ -308,7 +311,8 @@ function discoverController($element, $route, $scope, $timeout, Promise) { nextIndexPattern, $scope.state.columns, $scope.state.sort, - config.get(MODIFY_COLUMNS_ON_SWITCH) + config.get(MODIFY_COLUMNS_ON_SWITCH), + $scope.useNewFieldsApi ); await setAppState(nextAppState); } @@ -415,19 +419,33 @@ function discoverController($element, $route, $scope, $timeout, Promise) { setBreadcrumbsTitle(savedSearch, chrome); + function removeSourceFromColumns(columns) { + return columns.filter((col) => col !== '_source'); + } + + function getDefaultColumns() { + const columns = [...savedSearch.columns]; + + if ($scope.useNewFieldsApi) { + return removeSourceFromColumns(columns); + } + if (columns.length > 0) { + return columns; + } + return [...config.get(DEFAULT_COLUMNS_SETTING)]; + } + function getStateDefaults() { const query = $scope.searchSource.getField('query') || data.query.queryString.getDefaultQuery(); const sort = getSortArray(savedSearch.sort, $scope.indexPattern); + const columns = getDefaultColumns(); const defaultState = { query, sort: !sort.length ? getDefaultSort($scope.indexPattern, config.get(SORT_DEFAULT_ORDER_SETTING, 'desc')) : sort, - columns: - savedSearch.columns.length > 0 - ? savedSearch.columns - : config.get(DEFAULT_COLUMNS_SETTING).slice(), + columns, index: $scope.indexPattern.id, interval: 'auto', filters: _.cloneDeep($scope.searchSource.getOwnField('filter')), @@ -739,10 +757,14 @@ function discoverController($element, $route, $scope, $timeout, Promise) { }; $scope.updateDataSource = () => { - updateSearchSource($scope.searchSource, { - indexPattern: $scope.indexPattern, + const { indexPattern, searchSource, useNewFieldsApi } = $scope; + const { columns, sort } = $scope.state; + updateSearchSource(searchSource, { + indexPattern, services, - sort: $scope.state.sort, + sort, + columns, + useNewFieldsApi, }); return Promise.resolve(); }; @@ -770,20 +792,20 @@ function discoverController($element, $route, $scope, $timeout, Promise) { }; $scope.addColumn = function addColumn(columnName) { + const { indexPattern, useNewFieldsApi } = $scope; if (capabilities.discover.save) { - const { indexPattern } = $scope; popularizeField(indexPattern, columnName, indexPatterns); } - const columns = columnActions.addColumn($scope.state.columns, columnName); + const columns = columnActions.addColumn($scope.state.columns, columnName, useNewFieldsApi); setAppState({ columns }); }; $scope.removeColumn = function removeColumn(columnName) { + const { indexPattern, useNewFieldsApi } = $scope; if (capabilities.discover.save) { - const { indexPattern } = $scope; popularizeField(indexPattern, columnName, indexPatterns); } - const columns = columnActions.removeColumn($scope.state.columns, columnName); + const columns = columnActions.removeColumn($scope.state.columns, columnName, useNewFieldsApi); // The state's sort property is an array of [sortByColumn,sortDirection] const sort = $scope.state.sort.length ? $scope.state.sort.filter((subArr) => subArr[0] !== columnName) diff --git a/src/plugins/discover/public/application/angular/discover_legacy.html b/src/plugins/discover/public/application/angular/discover_legacy.html index 3596c0a2519ed..9383980fd9fd6 100644 --- a/src/plugins/discover/public/application/angular/discover_legacy.html +++ b/src/plugins/discover/public/application/angular/discover_legacy.html @@ -29,6 +29,7 @@ top-nav-menu="topNavMenu" update-query="handleRefresh" update-saved-query-id="updateSavedQueryId" + use-new-fields-api="useNewFieldsApi" > diff --git a/src/plugins/discover/public/application/angular/doc_table/actions/columns.ts b/src/plugins/discover/public/application/angular/doc_table/actions/columns.ts index 8257c79af7e8a..1b6d8fcbc2544 100644 --- a/src/plugins/discover/public/application/angular/doc_table/actions/columns.ts +++ b/src/plugins/discover/public/application/angular/doc_table/actions/columns.ts @@ -21,28 +21,32 @@ * Helper function to provide a fallback to a single _source column if the given array of columns * is empty, and removes _source if there are more than 1 columns given * @param columns + * @param useNewFieldsApi should a new fields API be used */ -function buildColumns(columns: string[]) { +function buildColumns(columns: string[], useNewFieldsApi = false) { if (columns.length > 1 && columns.indexOf('_source') !== -1) { return columns.filter((col) => col !== '_source'); } else if (columns.length !== 0) { return columns; } - return ['_source']; + return useNewFieldsApi ? [] : ['_source']; } -export function addColumn(columns: string[], columnName: string) { +export function addColumn(columns: string[], columnName: string, useNewFieldsApi?: boolean) { if (columns.includes(columnName)) { return columns; } - return buildColumns([...columns, columnName]); + return buildColumns([...columns, columnName], useNewFieldsApi); } -export function removeColumn(columns: string[], columnName: string) { +export function removeColumn(columns: string[], columnName: string, useNewFieldsApi?: boolean) { if (!columns.includes(columnName)) { return columns; } - return buildColumns(columns.filter((col) => col !== columnName)); + return buildColumns( + columns.filter((col) => col !== columnName), + useNewFieldsApi + ); } export function moveColumn(columns: string[], columnName: string, newIndex: number) { diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx b/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx index b456fa0773b85..bb855373c910f 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { i18n } from '@kbn/i18n'; import { IndexPattern } from '../../../../../kibana_services'; export type SortOrder = [string, string]; @@ -62,17 +63,33 @@ export function getDisplayedColumns( if (!Array.isArray(columns) || typeof indexPattern !== 'object' || !indexPattern.getFieldByName) { return []; } - const columnProps = columns.map((column, idx) => { - const field = indexPattern.getFieldByName(column); - return { - name: column, - displayName: field ? field.displayName : column, - isSortable: field && field.sortable ? true : false, - isRemoveable: column !== '_source' || columns.length > 1, - colLeftIdx: idx - 1 < 0 ? -1 : idx - 1, - colRightIdx: idx + 1 >= columns.length ? -1 : idx + 1, - }; - }); + + const columnProps = + columns.length === 0 + ? [ + { + name: '__document__', + displayName: i18n.translate('discover.docTable.tableHeader.documentHeader', { + defaultMessage: 'Document', + }), + isSortable: false, + isRemoveable: false, + colLeftIdx: -1, + colRightIdx: -1, + }, + ] + : columns.map((column, idx) => { + const field = indexPattern.getFieldByName(column); + return { + name: column, + displayName: field?.displayName ?? column, + isSortable: !!(field && field.sortable), + isRemoveable: column !== '_source' || columns.length > 1, + colLeftIdx: idx - 1 < 0 ? -1 : idx - 1, + colRightIdx: idx + 1 >= columns.length ? -1 : idx + 1, + }; + }); + return !hideTimeField && indexPattern.timeFieldName ? [getTimeColumn(indexPattern.timeFieldName), ...columnProps] : columnProps; diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts b/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts index e45f18606e3fc..75206d6bf2e84 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_row.ts @@ -27,6 +27,7 @@ import cellTemplateHtml from '../components/table_row/cell.html'; import truncateByHeightTemplateHtml from '../components/table_row/truncate_by_height.html'; import { getServices } from '../../../../kibana_services'; import { getContextUrl } from '../../../helpers/get_context_url'; +import { formatRow } from '../../helpers'; const TAGS_WITH_WS = />\s+ { $el.after(''); @@ -139,19 +141,33 @@ export function createTableRowDirective($compile: ng.ICompileService) { ); } - $scope.columns.forEach(function (column: any) { - const isFilterable = mapping(column) && mapping(column).filterable && $scope.filter; + if ($scope.columns.length === 0 && $scope.useNewFieldsApi) { + const formatted = formatRow(row, indexPattern); newHtmls.push( cellTemplate({ timefield: false, - sourcefield: column === '_source', - formatted: _displayField(row, column, true), - filterable: isFilterable, - column, + sourcefield: true, + formatted, + filterable: false, + column: '__document__', }) ); - }); + } else { + $scope.columns.forEach(function (column: string) { + const isFilterable = mapping(column) && mapping(column).filterable && $scope.filter; + + newHtmls.push( + cellTemplate({ + timefield: false, + sourcefield: column === '_source', + formatted: _displayField(row, column, true), + filterable: isFilterable, + column, + }) + ); + }); + } let $cells = $el.children(); newHtmls.forEach(function (html, i) { diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_row/_cell.scss b/src/plugins/discover/public/application/angular/doc_table/components/table_row/_cell.scss index b73a2598070b5..22b6e0f29268b 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_row/_cell.scss +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_row/_cell.scss @@ -1,9 +1,5 @@ -.kbnDocTableCell__dataField { - white-space: pre-wrap; -} - .kbnDocTableCell__toggleDetails { - padding: 4px 0 0 0!important; + padding: $euiSizeXS 0 0 0!important; } .kbnDocTableCell__filter { diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_row/details.html b/src/plugins/discover/public/application/angular/doc_table/components/table_row/details.html index fd20bea8fb3df..bb443b880e217 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_row/details.html +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_row/details.html @@ -1,4 +1,4 @@ - +
diff --git a/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx b/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx index f191fa2dc89e8..0a162673eec82 100644 --- a/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx +++ b/src/plugins/discover/public/application/angular/doc_table/create_doc_table_react.tsx @@ -97,6 +97,7 @@ export interface DocTableLegacyProps { onMoveColumn?: (columns: string, newIdx: number) => void; onRemoveColumn?: (column: string) => void; sort?: string[][]; + useNewFieldsApi?: boolean; } export function DocTableLegacy(renderProps: DocTableLegacyProps) { @@ -118,6 +119,7 @@ export function DocTableLegacy(renderProps: DocTableLegacyProps) { on-move-column="onMoveColumn" on-remove-column="onRemoveColumn" render-complete + use-new-fields-api="useNewFieldsApi" sorting="sort">`, }, () => getServices().getEmbeddableInjector() diff --git a/src/plugins/discover/public/application/angular/doc_table/doc_table.html b/src/plugins/discover/public/application/angular/doc_table/doc_table.html index bb8cc4b9ee4c2..427893bd3e6fe 100644 --- a/src/plugins/discover/public/application/angular/doc_table/doc_table.html +++ b/src/plugins/discover/public/application/angular/doc_table/doc_table.html @@ -46,6 +46,7 @@ class="kbnDocTable__row" on-add-column="onAddColumn" on-remove-column="onRemoveColumn" + use-new-fields-api="useNewFieldsApi" > @@ -97,6 +98,7 @@ data-test-subj="docTableRow{{ row['$$_isAnchor'] ? ' docTableAnchorRow' : ''}}" on-add-column="onAddColumn" on-remove-column="onRemoveColumn" + use-new-fields-api="useNewFieldsApi" > diff --git a/src/plugins/discover/public/application/angular/doc_table/doc_table.ts b/src/plugins/discover/public/application/angular/doc_table/doc_table.ts index 735ee9f555740..2baf010b47c78 100644 --- a/src/plugins/discover/public/application/angular/doc_table/doc_table.ts +++ b/src/plugins/discover/public/application/angular/doc_table/doc_table.ts @@ -48,6 +48,7 @@ export function createDocTableDirective(pagerFactory: any, $filter: any) { onMoveColumn: '=?', onRemoveColumn: '=?', inspectorAdapters: '=?', + useNewFieldsApi: '<', }, link: ($scope: LazyScope, $el: JQuery) => { $scope.persist = { diff --git a/src/plugins/discover/public/application/angular/helpers/index.ts b/src/plugins/discover/public/application/angular/helpers/index.ts index 9bfba4de966be..cba50dfa58751 100644 --- a/src/plugins/discover/public/application/angular/helpers/index.ts +++ b/src/plugins/discover/public/application/angular/helpers/index.ts @@ -18,3 +18,4 @@ */ export { buildPointSeriesData } from './point_series'; +export { formatRow } from './row_formatter'; diff --git a/src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts b/src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts new file mode 100644 index 0000000000000..60ee1e4c2b68b --- /dev/null +++ b/src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { formatRow } from './row_formatter'; +import { stubbedSavedObjectIndexPattern } from '../../../__mocks__/stubbed_saved_object_index_pattern'; +import { IndexPattern } from '../../../../../data/common/index_patterns/index_patterns'; +import { fieldFormatsMock } from '../../../../../data/common/field_formats/mocks'; + +describe('Row formatter', () => { + const hit = { + foo: 'bar', + number: 42, + hello: '

World

', + also: 'with "quotes" or \'single quotes\'', + }; + + const createIndexPattern = () => { + const id = 'my-index'; + const { + type, + version, + attributes: { timeFieldName, fields, title }, + } = stubbedSavedObjectIndexPattern(id); + + return new IndexPattern({ + spec: { id, type, version, timeFieldName, fields, title }, + fieldFormats: fieldFormatsMock, + shortDotsEnable: false, + metaFields: [], + }); + }; + + const indexPattern = createIndexPattern(); + + const formatHitReturnValue = { + also: 'with \\"quotes\\" or 'single qoutes'', + number: '42', + foo: 'bar', + hello: '<h1>World</h1>', + }; + const formatHitMock = jest.fn().mockReturnValueOnce(formatHitReturnValue); + + beforeEach(() => { + // @ts-ignore + indexPattern.formatHit = formatHitMock; + }); + + it('formats document properly', () => { + expect(formatRow(hit, indexPattern).trim()).toBe( + '
also:
with \\"quotes\\" or 'single qoutes'
number:
42
foo:
bar
hello:
<h1>World</h1>
' + ); + }); +}); diff --git a/src/plugins/discover/public/application/angular/helpers/row_formatter.ts b/src/plugins/discover/public/application/angular/helpers/row_formatter.ts new file mode 100644 index 0000000000000..4ad50ef7621c5 --- /dev/null +++ b/src/plugins/discover/public/application/angular/helpers/row_formatter.ts @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { template } from 'lodash'; +import { IndexPattern } from '../../../kibana_services'; + +function noWhiteSpace(html: string) { + const TAGS_WITH_WS = />\s+<'); +} + +const templateHtml = ` +
+ <% defPairs.forEach(function (def) { %> +
<%- def[0] %>:
+
<%= def[1] %>
+ <%= ' ' %> + <% }); %> +
`; +export const doTemplate = template(noWhiteSpace(templateHtml)); + +export const formatRow = (hit: Record, indexPattern: IndexPattern) => { + const highlights = hit?.highlight ?? {}; + const formatted = indexPattern.formatHit(hit); + const highlightPairs: Array<[string, unknown]> = []; + const sourcePairs: Array<[string, unknown]> = []; + Object.entries(formatted).forEach(([key, val]) => { + const pairs = highlights[key] ? highlightPairs : sourcePairs; + pairs.push([key, val]); + }); + return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] }); +}; diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx index f519df8a0b80d..4ace823471c45 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx @@ -48,6 +48,7 @@ export interface ContextAppProps { onChangeSuccessorCount: (count: number) => void; predecessorStatus: string; successorStatus: string; + useNewFieldsApi?: boolean; } const PREDECESSOR_TYPE = 'predecessors'; @@ -87,7 +88,15 @@ export function ContextAppLegacy(renderProps: ContextAppProps) { }; const docTableProps = () => { - const { hits, filter, sorting, columns, indexPattern, minimumVisibleRows } = renderProps; + const { + hits, + filter, + sorting, + columns, + indexPattern, + minimumVisibleRows, + useNewFieldsApi, + } = renderProps; return { columns, indexPattern, @@ -95,6 +104,7 @@ export function ContextAppLegacy(renderProps: ContextAppProps) { rows: hits, onFilter: filter, sort: sorting.map((el) => [el]), + useNewFieldsApi, } as DocTableLegacyProps; }; diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts index dfb5d90c2befe..e52226bee3785 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts @@ -37,6 +37,7 @@ export function createContextAppLegacy(reactDirective: any) { ['successorAvailable', { watchDepth: 'reference' }], ['successorStatus', { watchDepth: 'reference' }], ['onChangeSuccessorCount', { watchDepth: 'reference' }], + ['useNewFieldsApi', { watchDepth: 'reference' }], ['topNavMenu', { watchDepth: 'reference' }], ]); } diff --git a/src/plugins/discover/public/application/components/create_discover_legacy_directive.ts b/src/plugins/discover/public/application/components/create_discover_legacy_directive.ts index 6e5d47be987d8..fc877cab00e0b 100644 --- a/src/plugins/discover/public/application/components/create_discover_legacy_directive.ts +++ b/src/plugins/discover/public/application/components/create_discover_legacy_directive.ts @@ -51,5 +51,6 @@ export function createDiscoverLegacyDirective(reactDirective: any) { ['topNavMenu', { watchDepth: 'reference' }], ['updateQuery', { watchDepth: 'reference' }], ['updateSavedQueryId', { watchDepth: 'reference' }], + ['useNewFieldsApi', { watchDepth: 'reference' }], ]); } diff --git a/src/plugins/discover/public/application/components/discover_legacy.tsx b/src/plugins/discover/public/application/components/discover_legacy.tsx index 436a145024437..402e686979d67 100644 --- a/src/plugins/discover/public/application/components/discover_legacy.tsx +++ b/src/plugins/discover/public/application/components/discover_legacy.tsx @@ -219,6 +219,7 @@ export interface DiscoverProps { * Function to update the actual savedQuery id */ updateSavedQueryId: (savedQueryId?: string) => void; + useNewFieldsApi?: boolean; } export const DocTableLegacyMemoized = React.memo((props: DocTableLegacyProps) => ( @@ -257,6 +258,7 @@ export function DiscoverLegacy({ topNavMenu, updateQuery, updateSavedQueryId, + useNewFieldsApi, }: DiscoverProps) { const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -278,6 +280,17 @@ export function DiscoverLegacy({ : undefined; const contentCentered = resultState === 'uninitialized'; + const getDisplayColumns = () => { + if (!state.columns) { + return []; + } + const columns = [...state.columns]; + if (useNewFieldsApi) { + return columns.filter((column) => column !== '_source'); + } + return columns.length === 0 ? ['_source'] : columns; + }; + return ( @@ -315,6 +328,7 @@ export function DiscoverLegacy({ setIndexPattern={setIndexPattern} isClosed={isSidebarClosed} trackUiMetric={trackUiMetric} + useNewFieldsApi={useNewFieldsApi} /> @@ -445,7 +459,7 @@ export function DiscoverLegacy({ {rows && rows.length && (
{rows.length === opts.sampleSize ? (
+ +
+ +
+ +
+ + + +
+
+
+
+
+
+ +`; diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field.scss b/src/plugins/discover/public/application/components/sidebar/discover_field.scss index 8e1dd41f66ab1..40bc58cef7023 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field.scss +++ b/src/plugins/discover/public/application/components/sidebar/discover_field.scss @@ -1,4 +1,10 @@ .dscSidebarItem__fieldPopoverPanel { - min-width: 260px; - max-width: 300px; + min-width: $euiSizeXXL * 6.5; + max-width: $euiSizeXXL * 7.5; +} + +.dscSidebarItem--multi { + .kbnFieldButton__button { + padding-left: 0; + } } diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field.test.tsx index 0957ee101bd27..d22ef7cdcc28c 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field.test.tsx @@ -82,7 +82,7 @@ function getComponent({ const props = { indexPattern, field: finalField, - getDetails: jest.fn(() => ({ buckets: [], error: '', exists: 1, total: true, columns: [] })), + getDetails: jest.fn(() => ({ buckets: [], error: '', exists: 1, total: 2, columns: [] })), onAddFilter: jest.fn(), onAddField: jest.fn(), onRemoveField: jest.fn(), diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field.tsx index f95e512dfb66e..b885bdab316b5 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field.tsx @@ -19,7 +19,7 @@ import './discover_field.scss'; import React, { useState } from 'react'; -import { EuiPopover, EuiPopoverTitle, EuiButtonIcon, EuiToolTip } from '@elastic/eui'; +import { EuiPopover, EuiPopoverTitle, EuiButtonIcon, EuiToolTip, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { UiCounterMetricType } from '@kbn/analytics'; import classNames from 'classnames'; @@ -28,6 +28,7 @@ import { FieldIcon, FieldButton } from '../../../../../kibana_react/public'; import { FieldDetails } from './types'; import { IndexPatternField, IndexPattern } from '../../../../../data/public'; import { getFieldTypeName } from './lib/get_field_type_name'; +import { DiscoverFieldDetailsFooter } from './discover_field_details_footer'; export interface DiscoverFieldProps { /** @@ -69,6 +70,8 @@ export interface DiscoverFieldProps { * @param eventName */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; + + multiFields?: Array<{ field: IndexPatternField; isSelected: boolean }>; } export function DiscoverField({ @@ -81,6 +84,7 @@ export function DiscoverField({ getDetails, selected, trackUiMetric, + multiFields, }: DiscoverFieldProps) { const addLabelAria = i18n.translate('discover.fieldChooser.discoverField.addButtonAriaLabel', { defaultMessage: 'Add {field} to table', @@ -96,8 +100,8 @@ export function DiscoverField({ const [infoIsOpen, setOpen] = useState(false); - const toggleDisplay = (f: IndexPatternField) => { - if (selected) { + const toggleDisplay = (f: IndexPatternField, isSelected: boolean) => { + if (isSelected) { onRemoveField(f.name); } else { onAddField(f.name); @@ -115,72 +119,100 @@ export function DiscoverField({ return str ? str.replace(/\./g, '.\u200B') : ''; } - const dscFieldIcon = ( - - ); + const getDscFieldIcon = (indexPatternField: IndexPatternField) => { + return ( + + ); + }; - const title = - field.displayName !== field.name ? `${field.name} (${field.displayName} )` : field.displayName; + const dscFieldIcon = getDscFieldIcon(field); + + const getTitle = (indexPatternField: IndexPatternField) => { + return indexPatternField.displayName !== indexPatternField.name + ? i18n.translate('discover.field.title', { + defaultMessage: '{fieldName} ({fieldDisplayName})', + values: { + fieldName: indexPatternField.name, + fieldDisplayName: indexPatternField.displayName, + }, + }) + : indexPatternField.displayName; + }; + + const getFieldName = (indexPatternField: IndexPatternField) => { + return ( + + {wrapOnDot(indexPatternField.displayName)} + + ); + }; + const fieldName = getFieldName(field); - const fieldName = ( - - {wrapOnDot(field.displayName)} - - ); const actionBtnClassName = classNames('dscSidebarItem__action', { ['dscSidebarItem__mobile']: alwaysShowActionButton, }); - let actionButton; - if (field.name !== '_source' && !selected) { - actionButton = ( - - ) => { - if (ev.type === 'click') { - ev.currentTarget.focus(); - } - ev.preventDefault(); - ev.stopPropagation(); - toggleDisplay(field); - }} - data-test-subj={`fieldToggle-${field.name}`} - aria-label={addLabelAria} - /> - - ); - } else if (field.name !== '_source' && selected) { - actionButton = ( - - ) => { - if (ev.type === 'click') { - ev.currentTarget.focus(); - } - ev.preventDefault(); - ev.stopPropagation(); - toggleDisplay(field); - }} - data-test-subj={`fieldToggle-${field.name}`} - aria-label={removeLabelAria} - /> - - ); - } + const getActionButton = (f: IndexPatternField, isSelected?: boolean) => { + if (f.name !== '_source' && !isSelected) { + return ( + + ) => { + if (ev.type === 'click') { + ev.currentTarget.focus(); + } + ev.preventDefault(); + ev.stopPropagation(); + toggleDisplay(f, false); + }} + data-test-subj={`fieldToggle-${f.name}`} + aria-label={addLabelAria} + /> + + ); + } else if (f.name !== '_source' && isSelected) { + return ( + + ) => { + if (ev.type === 'click') { + ev.currentTarget.focus(); + } + ev.preventDefault(); + ev.stopPropagation(); + toggleDisplay(f, isSelected); + }} + data-test-subj={`fieldToggle-${f.name}`} + aria-label={removeLabelAria} + /> + + ); + } + }; + + const actionButton = getActionButton(field, selected); if (field.type === '_source') { return ( @@ -195,6 +227,37 @@ export function DiscoverField({ ); } + const shouldRenderMultiFields = !!multiFields; + const renderMultiFields = () => { + if (!multiFields) { + return null; + } + return ( + + +
+ {i18n.translate('discover.fieldChooser.discoverField.multiFields', { + defaultMessage: 'Multi fields', + })} +
+
+ {multiFields.map((entry) => ( + {}} + dataTestSubj={`field-${entry.field.name}-showDetails`} + fieldIcon={getDscFieldIcon(entry.field)} + fieldAction={getActionButton(entry.field, entry.isSelected)} + fieldName={getFieldName(entry.field)} + key={entry.field.name} + /> + ))} +
+ ); + }; + return ( - - {' '} - {i18n.translate('discover.fieldChooser.discoverField.fieldTopValuesLabel', { - defaultMessage: 'Top 5 values', - })} - + {field.displayName} + +
+ {i18n.translate('discover.fieldChooser.discoverField.fieldTopValuesLabel', { + defaultMessage: 'Top 5 values', + })} +
+
{infoIsOpen && ( )} + {shouldRenderMultiFields ? ( + <> + {renderMultiFields()} + + + ) : null}
); } diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx index 0618e53d15dbb..8444f11ac912c 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_details.test.tsx @@ -38,7 +38,7 @@ const indexPattern = getStubIndexPattern( describe('discover sidebar field details', function () { const defaultProps = { indexPattern, - details: { buckets: [], error: '', exists: 1, total: true, columns: [] }, + details: { buckets: [], error: '', exists: 1, total: 2, columns: [] }, onAddFilter: jest.fn(), }; diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_details.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_details.tsx index 740de54ae0cf3..bf24337543037 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field_details.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_details.tsx @@ -17,7 +17,7 @@ * under the License. */ import React, { useState, useEffect } from 'react'; -import { EuiLink, EuiIconTip, EuiText, EuiPopoverFooter, EuiButton, EuiSpacer } from '@elastic/eui'; +import { EuiIconTip, EuiText, EuiButton, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; import { DiscoverFieldBucket } from './discover_field_bucket'; @@ -30,6 +30,7 @@ import { import { Bucket, FieldDetails } from './types'; import { IndexPatternField, IndexPattern } from '../../../../../data/public'; import './discover_field_details.scss'; +import { DiscoverFieldDetailsFooter } from './discover_field_details_footer'; interface DiscoverFieldDetailsProps { field: IndexPatternField; @@ -37,6 +38,7 @@ interface DiscoverFieldDetailsProps { details: FieldDetails; onAddFilter: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; + showFooter?: boolean; } export function DiscoverFieldDetails({ @@ -45,6 +47,7 @@ export function DiscoverFieldDetails({ details, onAddFilter, trackUiMetric, + showFooter = true, }: DiscoverFieldDetailsProps) { const warnings = getWarnings(field); const [showVisualizeLink, setShowVisualizeLink] = useState(false); @@ -118,27 +121,13 @@ export function DiscoverFieldDetails({ )}
- {!details.error && ( - - - {!indexPattern.metaFields.includes(field.name) && !field.scripted ? ( - onAddFilter('_exists_', field.name, '+')}> - {' '} - {details.exists} - - ) : ( - {details.exists} - )}{' '} - / {details.total}{' '} - - - + {!details.error && showFooter && ( + )} ); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.test.tsx new file mode 100644 index 0000000000000..028187569e977 --- /dev/null +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.test.tsx @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { findTestSubject } from '@elastic/eui/lib/test'; +// @ts-ignore +import stubbedLogstashFields from 'fixtures/logstash_fields'; +import { mountWithIntl } from '@kbn/test/jest'; +import { coreMock } from '../../../../../../core/public/mocks'; +import { IndexPatternField } from '../../../../../data/public'; +import { getStubIndexPattern } from '../../../../../data/public/test_utils'; +import { DiscoverFieldDetailsFooter } from './discover_field_details_footer'; + +const indexPattern = getStubIndexPattern( + 'logstash-*', + (cfg: any) => cfg, + 'time', + stubbedLogstashFields(), + coreMock.createSetup() +); + +describe('discover sidebar field details footer', function () { + const onAddFilter = jest.fn(); + const defaultProps = { + indexPattern, + details: { buckets: [], error: '', exists: 1, total: 2, columns: [] }, + onAddFilter, + }; + + function mountComponent(field: IndexPatternField) { + const compProps = { ...defaultProps, field }; + return mountWithIntl(); + } + + it('renders properly', function () { + const visualizableField = new IndexPatternField({ + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }); + const component = mountComponent(visualizableField); + expect(component).toMatchSnapshot(); + }); + + it('click on addFilter calls the function', function () { + const visualizableField = new IndexPatternField({ + name: 'bytes', + type: 'number', + esTypes: ['long'], + count: 10, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }); + const component = mountComponent(visualizableField); + const onAddButton = findTestSubject(component, 'onAddFilterButton'); + onAddButton.simulate('click'); + expect(onAddFilter).toHaveBeenCalledWith('_exists_', visualizableField.name, '+'); + }); +}); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.tsx new file mode 100644 index 0000000000000..58e91c85913a1 --- /dev/null +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_details_footer.tsx @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { EuiLink, EuiPopoverFooter, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { IndexPatternField } from '../../../../../data/common/index_patterns/fields'; +import { IndexPattern } from '../../../../../data/common/index_patterns/index_patterns'; +import { FieldDetails } from './types'; + +interface DiscoverFieldDetailsFooterProps { + field: IndexPatternField; + indexPattern: IndexPattern; + details: FieldDetails; + onAddFilter: (field: IndexPatternField | string, value: string, type: '+' | '-') => void; +} + +export function DiscoverFieldDetailsFooter({ + field, + indexPattern, + details, + onAddFilter, +}: DiscoverFieldDetailsFooterProps) { + return ( + + + {!indexPattern.metaFields.includes(field.name) && !field.scripted ? ( + onAddFilter('_exists_', field.name, '+')} + data-test-subj="onAddFilterButton" + > + + + ) : ( + + )} + + + ); +} diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx index 57cc45b3c3e9f..6c312924fb7b7 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx @@ -100,6 +100,10 @@ export interface DiscoverSidebarProps { * Callback function to select another index pattern */ setIndexPattern: (id: string) => void; + /** + * If on, fields are read from the fields API, not from source + */ + useNewFieldsApi?: boolean; /** * Metric tracking function * @param metricType @@ -127,9 +131,11 @@ export function DiscoverSidebar({ setFieldFilter, setIndexPattern, trackUiMetric, + useNewFieldsApi = false, useFlyout = false, }: DiscoverSidebarProps) { const [fields, setFields] = useState(null); + useEffect(() => { const newFields = getIndexPatternFieldList(selectedIndexPattern, fieldCounts); setFields(newFields); @@ -154,13 +160,10 @@ export function DiscoverSidebar({ selected: selectedFields, popular: popularFields, unpopular: unpopularFields, - } = useMemo(() => groupFields(fields, columns, popularLimit, fieldCounts, fieldFilter), [ - fields, - columns, - popularLimit, - fieldCounts, - fieldFilter, - ]); + } = useMemo( + () => groupFields(fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi), + [fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi] + ); const fieldTypes = useMemo(() => { const result = ['any']; @@ -174,6 +177,27 @@ export function DiscoverSidebar({ return result; }, [fields]); + const multiFields = useMemo(() => { + if (!useNewFieldsApi || !fields) { + return undefined; + } + const map = new Map>(); + fields.forEach((field) => { + const parent = field.spec?.subType?.multi?.parent; + if (!parent) { + return; + } + const multiField = { + field, + isSelected: selectedFields.includes(field), + }; + const value = map.get(parent) ?? []; + value.push(multiField); + map.set(parent, value); + }); + return map; + }, [fields, useNewFieldsApi, selectedFields]); + if (!selectedIndexPattern || !fields) { return null; } @@ -278,6 +302,7 @@ export function DiscoverSidebar({ getDetails={getDetailsByField} selected={true} trackUiMetric={trackUiMetric} + multiFields={multiFields?.get(field.name)} /> ); @@ -338,6 +363,7 @@ export function DiscoverSidebar({ onAddFilter={onAddFilter} getDetails={getDetailsByField} trackUiMetric={trackUiMetric} + multiFields={multiFields?.get(field.name)} /> ); @@ -366,6 +392,7 @@ export function DiscoverSidebar({ onAddFilter={onAddFilter} getDetails={getDetailsByField} trackUiMetric={trackUiMetric} + multiFields={multiFields?.get(field.name)} /> ); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx index 0413ebd17d71b..3000291fc23bb 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx @@ -103,6 +103,10 @@ export interface DiscoverSidebarResponsiveProps { * Shows index pattern and a button that displays the sidebar in a flyout */ useFlyout?: boolean; + /** + * Read from the Fields API + */ + useNewFieldsApi?: boolean; } /** diff --git a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts index 22cacae4c3b45..6cbfa03a070db 100644 --- a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts +++ b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts @@ -69,7 +69,8 @@ describe('group_fields', function () { ['currency'], 5, fieldCounts, - fieldFilterState + fieldFilterState, + false ); expect(actual).toMatchInlineSnapshot(` Object { @@ -118,6 +119,80 @@ describe('group_fields', function () { } `); }); + it('should group fields in selected, popular, unpopular group if they contain multifields', function () { + const category = { + name: 'category', + type: 'string', + esTypes: ['text'], + count: 1, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }; + const currency = { + name: 'currency', + displayName: 'currency', + kbnFieldType: { + esTypes: ['string', 'text', 'keyword', '_type', '_id'], + filterable: true, + name: 'string', + sortable: true, + }, + spec: { + esTypes: ['text'], + name: 'category', + }, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }; + const currencyKeyword = { + name: 'currency.keyword', + displayName: 'currency.keyword', + type: 'string', + esTypes: ['keyword'], + kbnFieldType: { + esTypes: ['string', 'text', 'keyword', '_type', '_id'], + filterable: true, + name: 'string', + sortable: true, + }, + spec: { + aggregatable: true, + esTypes: ['keyword'], + name: 'category.keyword', + readFromDocValues: true, + searchable: true, + shortDotsEnable: false, + subType: { + multi: { + parent: 'currency', + }, + }, + }, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: false, + }; + const fieldsToGroup = [category, currency, currencyKeyword]; + + const fieldFilterState = getDefaultFieldFilter(); + + const actual = groupFields( + fieldsToGroup as any, + ['currency'], + 5, + fieldCounts, + fieldFilterState, + true + ); + expect(actual.popular).toEqual([category]); + expect(actual.selected).toEqual([currency]); + expect(actual.unpopular).toEqual([]); + }); it('should sort selected fields by columns order ', function () { const fieldFilterState = getDefaultFieldFilter(); @@ -127,7 +202,8 @@ describe('group_fields', function () { ['customer_birth_date', 'currency', 'unknown'], 5, fieldCounts, - fieldFilterState + fieldFilterState, + false ); expect(actual1.selected.map((field) => field.name)).toEqual([ 'customer_birth_date', @@ -140,7 +216,8 @@ describe('group_fields', function () { ['currency', 'customer_birth_date', 'unknown'], 5, fieldCounts, - fieldFilterState + fieldFilterState, + false ); expect(actual2.selected.map((field) => field.name)).toEqual([ 'currency', diff --git a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx index c34becc97cb93..e6c3d0fe3ea42 100644 --- a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx +++ b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx @@ -33,7 +33,8 @@ export function groupFields( columns: string[], popularLimit: number, fieldCounts: Record, - fieldFilterState: FieldFilterState + fieldFilterState: FieldFilterState, + useNewFieldsApi: boolean ): GroupedFields { const result: GroupedFields = { selected: [], @@ -62,12 +63,17 @@ export function groupFields( if (!isFieldFiltered(field, fieldFilterState, fieldCounts)) { continue; } + const isSubfield = useNewFieldsApi && field.spec?.subType?.multi?.parent; if (columns.includes(field.name)) { result.selected.push(field); } else if (popular.includes(field.name) && field.type !== '_source') { - result.popular.push(field); + if (!isSubfield) { + result.popular.push(field); + } } else if (field.type !== '_source') { - result.unpopular.push(field); + if (!isSubfield) { + result.unpopular.push(field); + } } } // add columns, that are not part of the index pattern, to be removeable diff --git a/src/plugins/discover/public/application/components/sidebar/types.ts b/src/plugins/discover/public/application/components/sidebar/types.ts index d80662b65cc7b..4ec731e852ce3 100644 --- a/src/plugins/discover/public/application/components/sidebar/types.ts +++ b/src/plugins/discover/public/application/components/sidebar/types.ts @@ -25,7 +25,7 @@ export interface IndexPatternRef { export interface FieldDetails { error: string; exists: number; - total: boolean; + total: number; buckets: Bucket[]; columns: string[]; } diff --git a/src/plugins/discover/public/application/helpers/get_switch_index_pattern_app_state.ts b/src/plugins/discover/public/application/helpers/get_switch_index_pattern_app_state.ts index 458b9b7e066fd..5af4449a63e0b 100644 --- a/src/plugins/discover/public/application/helpers/get_switch_index_pattern_app_state.ts +++ b/src/plugins/discover/public/application/helpers/get_switch_index_pattern_app_state.ts @@ -29,7 +29,8 @@ export function getSwitchIndexPatternAppState( nextIndexPattern: IndexPattern, currentColumns: string[], currentSort: SortPairArr[], - modifyColumns: boolean = true + modifyColumns: boolean = true, + useNewFieldsApi: boolean = false ) { const nextColumns = modifyColumns ? currentColumns.filter( @@ -38,9 +39,11 @@ export function getSwitchIndexPatternAppState( ) : currentColumns; const nextSort = getSortArray(currentSort, nextIndexPattern); + const defaultColumns = useNewFieldsApi ? [] : ['_source']; + const columns = nextColumns.length ? nextColumns : defaultColumns; return { index: nextIndexPattern.id, - columns: nextColumns.length ? nextColumns : ['_source'], + columns, sort: nextSort, }; } diff --git a/src/plugins/discover/public/application/helpers/persist_saved_search.ts b/src/plugins/discover/public/application/helpers/persist_saved_search.ts index 8ec2012b5843e..2f373c34eb17d 100644 --- a/src/plugins/discover/public/application/helpers/persist_saved_search.ts +++ b/src/plugins/discover/public/application/helpers/persist_saved_search.ts @@ -49,6 +49,8 @@ export async function persistSavedSearch( indexPattern, services, sort: state.sort as SortOrder[], + columns: state.columns || [], + useNewFieldsApi: false, }); savedSearch.columns = state.columns || []; diff --git a/src/plugins/discover/public/application/helpers/update_search_source.test.ts b/src/plugins/discover/public/application/helpers/update_search_source.test.ts index 91832325432ef..615a414680469 100644 --- a/src/plugins/discover/public/application/helpers/update_search_source.test.ts +++ b/src/plugins/discover/public/application/helpers/update_search_source.test.ts @@ -44,8 +44,37 @@ describe('updateSearchSource', () => { } as unknown) as IUiSettingsClient, } as unknown) as DiscoverServices, sort: [] as SortOrder[], + columns: [], + useNewFieldsApi: false, }); expect(result.getField('index')).toEqual(indexPatternMock); expect(result.getField('size')).toEqual(sampleSize); + expect(result.getField('fields')).toBe(undefined); + }); + + test('updates a given search source with the usage of the new fields api', async () => { + const searchSourceMock = createSearchSourceMock({}); + const sampleSize = 250; + const result = updateSearchSource(searchSourceMock, { + indexPattern: indexPatternMock, + services: ({ + data: dataPluginMock.createStartContract(), + uiSettings: ({ + get: (key: string) => { + if (key === SAMPLE_SIZE_SETTING) { + return sampleSize; + } + return false; + }, + } as unknown) as IUiSettingsClient, + } as unknown) as DiscoverServices, + sort: [] as SortOrder[], + columns: [], + useNewFieldsApi: true, + }); + expect(result.getField('index')).toEqual(indexPatternMock); + expect(result.getField('size')).toEqual(sampleSize); + expect(result.getField('fields')).toEqual(['*']); + expect(result.getField('fieldsFromSource')).toBe(undefined); }); }); diff --git a/src/plugins/discover/public/application/helpers/update_search_source.ts b/src/plugins/discover/public/application/helpers/update_search_source.ts index 324dc8a48457a..46f1c9f626054 100644 --- a/src/plugins/discover/public/application/helpers/update_search_source.ts +++ b/src/plugins/discover/public/application/helpers/update_search_source.ts @@ -31,10 +31,14 @@ export function updateSearchSource( indexPattern, services, sort, + columns, + useNewFieldsApi, }: { indexPattern: IndexPattern; services: DiscoverServices; sort: SortOrder[]; + columns: string[]; + useNewFieldsApi: boolean; } ) { const { uiSettings, data } = services; @@ -50,5 +54,13 @@ export function updateSearchSource( .setField('sort', usedSort) .setField('query', data.query.queryString.getQuery() || null) .setField('filter', data.query.filterManager.getFilters()); + if (useNewFieldsApi) { + searchSource.removeField('fieldsFromSource'); + searchSource.setField('fields', ['*']); + } else { + searchSource.removeField('fields'); + const fieldNames = indexPattern.fields.map((field) => field.name); + searchSource.setField('fieldsFromSource', fieldNames); + } return searchSource; } diff --git a/src/plugins/discover/server/ui_settings.ts b/src/plugins/discover/server/ui_settings.ts index 425928385e64a..673f55c78a506 100644 --- a/src/plugins/discover/server/ui_settings.ts +++ b/src/plugins/discover/server/ui_settings.ts @@ -35,6 +35,7 @@ import { CONTEXT_TIE_BREAKER_FIELDS_SETTING, DOC_TABLE_LEGACY, MODIFY_COLUMNS_ON_SWITCH, + SEARCH_FIELDS_FROM_SOURCE, } from '../common'; export const uiSettings: Record = { @@ -198,4 +199,11 @@ export const uiSettings: Record = { name: 'discover:modifyColumnsOnSwitchTitle', }, }, + [SEARCH_FIELDS_FROM_SOURCE]: { + name: 'Read fields from _source', + description: `When enabled will load documents directly from \`_source\`. This is soon going to be deprecated. When disabled, will retrieve fields via the new Fields API in the high-level search service.`, + value: false, + category: ['discover'], + schema: schema.boolean(), + }, }; diff --git a/test/functional/apps/context/_date_nanos_custom_timestamp.js b/test/functional/apps/context/_date_nanos_custom_timestamp.js index 8fe08d13af0aa..8772b10a4b8c8 100644 --- a/test/functional/apps/context/_date_nanos_custom_timestamp.js +++ b/test/functional/apps/context/_date_nanos_custom_timestamp.js @@ -38,6 +38,7 @@ export default function ({ getService, getPageObjects }) { await kibanaServer.uiSettings.update({ 'context:defaultSize': `${TEST_DEFAULT_CONTEXT_SIZE}`, 'context:step': `${TEST_STEP_SIZE}`, + 'discover:searchFieldsFromSource': true, }); }); diff --git a/test/functional/apps/discover/_data_grid_doc_navigation.ts b/test/functional/apps/discover/_data_grid_doc_navigation.ts index 92d9893cab0b6..97b8eb564a256 100644 --- a/test/functional/apps/discover/_data_grid_doc_navigation.ts +++ b/test/functional/apps/discover/_data_grid_doc_navigation.ts @@ -56,7 +56,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(hasDocHit).to.be(true); }); - it('add filter should create an exists filter if value is null (#7189)', async function () { + // no longer relevant as null field won't be returned in the Fields API response + xit('add filter should create an exists filter if value is null (#7189)', async function () { await PageObjects.discover.waitUntilSearchingHasFinished(); // Filter special document await filterBar.addFilter('agent', 'is', 'Missing/Fields'); diff --git a/test/functional/apps/discover/_data_grid_field_data.ts b/test/functional/apps/discover/_data_grid_field_data.ts index 8224f59f7fabf..137c19149d274 100644 --- a/test/functional/apps/discover/_data_grid_field_data.ts +++ b/test/functional/apps/discover/_data_grid_field_data.ts @@ -53,7 +53,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('the search term should be highlighted in the field data', async function () { // marks is the style that highlights the text in yellow const marks = await PageObjects.discover.getMarks(); - expect(marks.length).to.be(25); + expect(marks.length).to.be(50); expect(marks.indexOf('php')).to.be(0); }); diff --git a/test/functional/apps/discover/_discover_fields_api.ts b/test/functional/apps/discover/_discover_fields_api.ts new file mode 100644 index 0000000000000..94cb4ed5fa52e --- /dev/null +++ b/test/functional/apps/discover/_discover_fields_api.ts @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import expect from '@kbn/expect'; +import { FtrProviderContext } from './ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const log = getService('log'); + const retry = getService('retry'); + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']); + const defaultSettings = { + defaultIndex: 'logstash-*', + 'discover:searchFieldsFromSource': false, + }; + describe('discover uses fields API test', function describeIndexTests() { + before(async function () { + log.debug('load kibana index with default index pattern'); + await esArchiver.load('discover'); + await esArchiver.loadIfNeeded('logstash_functional'); + await kibanaServer.uiSettings.replace(defaultSettings); + log.debug('discover'); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setDefaultAbsoluteRange(); + }); + + after(async () => { + await kibanaServer.uiSettings.replace({ 'discover:searchFieldsFromSource': true }); + }); + + it('should correctly display documents', async function () { + log.debug('check if Document title exists in the grid'); + expect(await PageObjects.discover.getDocHeader()).to.have.string('Document'); + const rowData = await PageObjects.discover.getDocTableIndex(1); + log.debug('check the newest doc timestamp in UTC (check diff timezone in last test)'); + expect(rowData.startsWith('Sep 22, 2015 @ 23:50:13.253')).to.be.ok(); + const expectedHitCount = '14,004'; + await retry.try(async function () { + expect(await PageObjects.discover.getHitCount()).to.be(expectedHitCount); + }); + }); + + it('adding a column removes a default column', async function () { + await PageObjects.discover.clickFieldListItemAdd('_score'); + expect(await PageObjects.discover.getDocHeader()).to.have.string('_score'); + expect(await PageObjects.discover.getDocHeader()).not.to.have.string('Document'); + }); + + it('removing a column adds a default column', async function () { + await PageObjects.discover.clickFieldListItemRemove('_score'); + expect(await PageObjects.discover.getDocHeader()).not.to.have.string('_score'); + expect(await PageObjects.discover.getDocHeader()).to.have.string('Document'); + }); + }); +} diff --git a/test/functional/apps/discover/_doc_navigation.ts b/test/functional/apps/discover/_doc_navigation.ts index 76612b255ac23..79632942cf04a 100644 --- a/test/functional/apps/discover/_doc_navigation.ts +++ b/test/functional/apps/discover/_doc_navigation.ts @@ -55,7 +55,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(hasDocHit).to.be(true); }); - it('add filter should create an exists filter if value is null (#7189)', async function () { + // no longer relevant as null field won't be returned in the Fields API response + xit('add filter should create an exists filter if value is null (#7189)', async function () { await PageObjects.discover.waitUntilSearchingHasFinished(); // Filter special document await filterBar.addFilter('agent', 'is', 'Missing/Fields'); diff --git a/test/functional/apps/discover/_field_data.ts b/test/functional/apps/discover/_field_data.ts index e08325a81a3e8..3811cde8a6367 100644 --- a/test/functional/apps/discover/_field_data.ts +++ b/test/functional/apps/discover/_field_data.ts @@ -36,6 +36,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await esArchiver.load('discover'); await kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*', + 'discover:searchFieldsFromSource': true, }); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); await PageObjects.common.navigateToApp('discover'); diff --git a/test/functional/apps/discover/_field_data_with_fields_api.ts b/test/functional/apps/discover/_field_data_with_fields_api.ts new file mode 100644 index 0000000000000..923a021f5fad6 --- /dev/null +++ b/test/functional/apps/discover/_field_data_with_fields_api.ts @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const retry = getService('retry'); + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const toasts = getService('toasts'); + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']); + + describe('discover tab with new fields API', function describeIndexTests() { + this.tags('includeFirefox'); + before(async function () { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.load('discover'); + await kibanaServer.uiSettings.replace({ + defaultIndex: 'logstash-*', + 'discover:searchFieldsFromSource': false, + }); + await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); + await PageObjects.common.navigateToApp('discover'); + }); + describe('field data', function () { + it('search php should show the correct hit count', async function () { + const expectedHitCount = '445'; + await retry.try(async function () { + await queryBar.setQuery('php'); + await queryBar.submitQuery(); + const hitCount = await PageObjects.discover.getHitCount(); + expect(hitCount).to.be(expectedHitCount); + }); + }); + + it('the search term should be highlighted in the field data', async function () { + // marks is the style that highlights the text in yellow + const marks = await PageObjects.discover.getMarks(); + expect(marks.length).to.be(100); + expect(marks.indexOf('php')).to.be(0); + }); + + it('search type:apache should show the correct hit count', async function () { + const expectedHitCount = '11,156'; + await queryBar.setQuery('type:apache'); + await queryBar.submitQuery(); + await retry.try(async function tryingForTime() { + const hitCount = await PageObjects.discover.getHitCount(); + expect(hitCount).to.be(expectedHitCount); + }); + }); + + it('doc view should show Time and Document columns', async function () { + const expectedHeader = 'Time Document'; + const Docheader = await PageObjects.discover.getDocHeader(); + expect(Docheader).to.be(expectedHeader); + }); + + it('doc view should sort ascending', async function () { + const expectedTimeStamp = 'Sep 20, 2015 @ 00:00:00.000'; + await PageObjects.discover.clickDocSortDown(); + + // we don't technically need this sleep here because the tryForTime will retry and the + // results will match on the 2nd or 3rd attempt, but that debug output is huge in this + // case and it can be avoided with just a few seconds sleep. + await PageObjects.common.sleep(2000); + await retry.try(async function tryingForTime() { + const rowData = await PageObjects.discover.getDocTableIndex(1); + + expect(rowData.startsWith(expectedTimeStamp)).to.be.ok(); + }); + }); + + it('a bad syntax query should show an error message', async function () { + const expectedError = + 'Expected ":", "<", "<=", ">", ">=", AND, OR, end of input, ' + + 'whitespace but "(" found.'; + await queryBar.setQuery('xxx(yyy))'); + await queryBar.submitQuery(); + const { message } = await toasts.getErrorToast(); + expect(message).to.contain(expectedError); + await toasts.dismissToast(); + }); + }); + }); +} diff --git a/test/functional/apps/discover/_large_string.ts b/test/functional/apps/discover/_large_string.ts index fe5613a4e3f19..e8ad6131bcc21 100644 --- a/test/functional/apps/discover/_large_string.ts +++ b/test/functional/apps/discover/_large_string.ts @@ -40,7 +40,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('verify the large string book present', async function () { const ExpectedDoc = - 'mybook:Project Gutenberg EBook of Hamlet, by William Shakespeare' + + '_id:1 _type: - _index:testlargestring _score:0' + + ' mybook:Project Gutenberg EBook of Hamlet, by William Shakespeare' + ' This eBook is for the use of anyone anywhere in the United States' + ' and most other parts of the world at no cost and with almost no restrictions whatsoever.' + ' You may copy it, give it away or re-use it under the terms of the' + diff --git a/test/functional/apps/discover/_shared_links.ts b/test/functional/apps/discover/_shared_links.ts index 51ea5f997e859..b15f5b0aae39f 100644 --- a/test/functional/apps/discover/_shared_links.ts +++ b/test/functional/apps/discover/_shared_links.ts @@ -88,7 +88,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { '/app/discover?_t=1453775307251#' + '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time' + ":(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + - "-23T18:31:44.000Z'))&_a=(columns:!(_source),filters:!(),index:'logstash-" + + "-23T18:31:44.000Z'))&_a=(columns:!(),filters:!(),index:'logstash-" + "*',interval:auto,query:(language:kuery,query:'')" + ",sort:!(!('@timestamp',desc)))"; const actualUrl = await PageObjects.share.getSharedUrl(); diff --git a/test/functional/apps/discover/_source_filters.ts b/test/functional/apps/discover/_source_filters.ts index 0af7c0ade79ba..d2ae02ef25de4 100644 --- a/test/functional/apps/discover/_source_filters.ts +++ b/test/functional/apps/discover/_source_filters.ts @@ -40,6 +40,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); + await kibanaServer.uiSettings.update({ + 'discover:searchFieldsFromSource': true, + }); + log.debug('discover'); await PageObjects.common.navigateToApp('discover'); diff --git a/test/functional/apps/discover/ftr_provider_context.d.ts b/test/functional/apps/discover/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..a4894e024b612 --- /dev/null +++ b/test/functional/apps/discover/ftr_provider_context.d.ts @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; +import { services } from '../../services'; +import { pageObjects } from '../../page_objects'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/test/functional/apps/discover/index.ts b/test/functional/apps/discover/index.ts index 450049af66abf..5fd49a1d35216 100644 --- a/test/functional/apps/discover/index.ts +++ b/test/functional/apps/discover/index.ts @@ -42,6 +42,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_filter_editor')); loadTestFile(require.resolve('./_errors')); loadTestFile(require.resolve('./_field_data')); + loadTestFile(require.resolve('./_field_data_with_fields_api')); loadTestFile(require.resolve('./_shared_links')); loadTestFile(require.resolve('./_sidebar')); loadTestFile(require.resolve('./_source_filters')); @@ -51,6 +52,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_date_nanos')); loadTestFile(require.resolve('./_date_nanos_mixed')); loadTestFile(require.resolve('./_indexpattern_without_timefield')); + loadTestFile(require.resolve('./_discover_fields_api')); loadTestFile(require.resolve('./_data_grid')); loadTestFile(require.resolve('./_data_grid_context')); loadTestFile(require.resolve('./_data_grid_field_data')); diff --git a/test/functional/fixtures/es_archiver/date_nanos/mappings.json b/test/functional/fixtures/es_archiver/date_nanos/mappings.json index bea82767f6cbb..f9ef429a0f97c 100644 --- a/test/functional/fixtures/es_archiver/date_nanos/mappings.json +++ b/test/functional/fixtures/es_archiver/date_nanos/mappings.json @@ -5,7 +5,8 @@ "mappings": { "properties": { "@timestamp": { - "type": "date_nanos" + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" } } }, diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json index c62918abced58..b29f6b111b06d 100644 --- a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json @@ -29,7 +29,8 @@ "mappings": { "properties": { "timestamp": { - "type": "date_nanos" + "type": "date_nanos", + "format": "strict_date_optional_time_nanos" } } }, diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f8c5d7327d20d..2c553f9b81d7c 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -1510,10 +1510,8 @@ "discover.embeddable.inspectorRequestDescription": "このリクエストはElasticsearchにクエリをかけ、検索データを取得します。", "discover.embeddable.search.displayName": "検索", "discover.fieldChooser.detailViews.emptyStringText": "空の文字列", - "discover.fieldChooser.detailViews.existsText": "存在する", "discover.fieldChooser.detailViews.filterOutValueButtonAriaLabel": "{field}を除外:\"{value}\"", "discover.fieldChooser.detailViews.filterValueButtonAriaLabel": "{field}を除外:\"{value}\"", - "discover.fieldChooser.detailViews.recordsText": "記録", "discover.fieldChooser.detailViews.visualizeLinkText": "可視化", "discover.fieldChooser.discoverField.addButtonAriaLabel": "{field}を表に追加", "discover.fieldChooser.discoverField.addFieldTooltip": "フィールドを列として追加", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 6dbf484150d99..7d0cef70b0930 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -1510,10 +1510,8 @@ "discover.embeddable.inspectorRequestDescription": "此请求将查询 Elasticsearch 以获取搜索的数据。", "discover.embeddable.search.displayName": "搜索", "discover.fieldChooser.detailViews.emptyStringText": "空字符串", - "discover.fieldChooser.detailViews.existsText": "存在于", "discover.fieldChooser.detailViews.filterOutValueButtonAriaLabel": "筛除 {field}:“{value}”", "discover.fieldChooser.detailViews.filterValueButtonAriaLabel": "筛留 {field}:“{value}”", - "discover.fieldChooser.detailViews.recordsText": "个记录", "discover.fieldChooser.detailViews.visualizeLinkText": "可视化", "discover.fieldChooser.discoverField.addButtonAriaLabel": "将 {field} 添加到表中", "discover.fieldChooser.discoverField.addFieldTooltip": "将字段添加为列", diff --git a/x-pack/test/functional/apps/security/doc_level_security_roles.js b/x-pack/test/functional/apps/security/doc_level_security_roles.js index 72f463be48fd5..0595322ad2d21 100644 --- a/x-pack/test/functional/apps/security/doc_level_security_roles.js +++ b/x-pack/test/functional/apps/security/doc_level_security_roles.js @@ -77,7 +77,7 @@ export default function ({ getService, getPageObjects }) { }); const rowData = await PageObjects.discover.getDocTableIndex(1); expect(rowData).to.be( - 'name:ABC Company region:EAST _id:doc1 _type: - _index:dlstest _score:0' + '_id:doc1 _type: - _index:dlstest _score:0 region.keyword:EAST name:ABC Company name.keyword:ABC Company region:EAST' ); }); after('logout', async () => { diff --git a/x-pack/test/functional/apps/security/field_level_security.js b/x-pack/test/functional/apps/security/field_level_security.js index 7b22d72885c9d..3f3984dd05a94 100644 --- a/x-pack/test/functional/apps/security/field_level_security.js +++ b/x-pack/test/functional/apps/security/field_level_security.js @@ -112,7 +112,7 @@ export default function ({ getService, getPageObjects }) { }); const rowData = await PageObjects.discover.getDocTableIndex(1); expect(rowData).to.be( - 'customer_ssn:444.555.6666 customer_name:ABC Company customer_region:WEST _id:2 _type: - _index:flstest _score:0' + '_id:2 _type: - _index:flstest _score:0 customer_name.keyword:ABC Company customer_ssn:444.555.6666 customer_region.keyword:WEST runtime_customer_ssn:444.555.6666 calculated at runtime customer_region:WEST customer_name:ABC Company customer_ssn.keyword:444.555.6666' ); }); @@ -126,7 +126,7 @@ export default function ({ getService, getPageObjects }) { }); const rowData = await PageObjects.discover.getDocTableIndex(1); expect(rowData).to.be( - 'customer_name:ABC Company customer_region:WEST _id:2 _type: - _index:flstest _score:0' + '_id:2 _type: - _index:flstest _score:0 customer_name.keyword:ABC Company customer_region.keyword:WEST customer_region:WEST customer_name:ABC Company' ); }); diff --git a/x-pack/test/functional/es_archives/security/flstest/data/mappings.json b/x-pack/test/functional/es_archives/security/flstest/data/mappings.json index 4f419e4b6ade4..0b970d5a3c1df 100644 --- a/x-pack/test/functional/es_archives/security/flstest/data/mappings.json +++ b/x-pack/test/functional/es_archives/security/flstest/data/mappings.json @@ -7,7 +7,8 @@ "runtime_customer_ssn": { "type": "keyword", "script": { - "source": "emit(doc['customer_ssn'].value + ' calculated at runtime')" + "lang": "painless", + "source": "if (doc['customer_ssn'].size() !== 0) { return emit(doc['customer_ssn'].value + ' calculated at runtime') }" } } }, @@ -37,7 +38,8 @@ "type": "keyword" } }, - "type": "text" + "type": "text", + "fielddata": true } } },