diff --git a/packages/kbn-es-query/index.ts b/packages/kbn-es-query/index.ts index a14724fc3a748..ac0a078e34d94 100644 --- a/packages/kbn-es-query/index.ts +++ b/packages/kbn-es-query/index.ts @@ -56,6 +56,7 @@ export { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery, getLanguageDisplayName, + cleanupESQLQueryForLensSuggestions, } from './src/es_query'; export { diff --git a/packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts b/packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts index ca98864aab446..504e6a5c93d44 100644 --- a/packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts +++ b/packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts @@ -12,6 +12,7 @@ import { getAggregateQueryMode, getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery, + cleanupESQLQueryForLensSuggestions, } from './es_aggregate_query'; describe('sql query helpers', () => { @@ -115,4 +116,18 @@ describe('sql query helpers', () => { expect(idxPattern9).toBe('foo-1, foo-2'); }); }); + + describe('cleanupESQLQueryForLensSuggestions', () => { + it('should not remove anything if a drop command is not present', () => { + expect(cleanupESQLQueryForLensSuggestions('from a | eval b = 1')).toBe('from a | eval b = 1'); + }); + + it('should remove multiple drop statement if present', () => { + expect( + cleanupESQLQueryForLensSuggestions( + 'from a | drop @timestamp | drop a | drop b | keep c | drop d' + ) + ).toBe('from a | keep c '); + }); + }); }); diff --git a/packages/kbn-es-query/src/es_query/es_aggregate_query.ts b/packages/kbn-es-query/src/es_query/es_aggregate_query.ts index 27a5e790569c8..f746505896360 100644 --- a/packages/kbn-es-query/src/es_query/es_aggregate_query.ts +++ b/packages/kbn-es-query/src/es_query/es_aggregate_query.ts @@ -66,3 +66,8 @@ export function getIndexPatternFromESQLQuery(esql?: string): string { } return ''; } + +export function cleanupESQLQueryForLensSuggestions(esql?: string): string { + const pipes = (esql || '').split('|'); + return pipes.filter((statement) => !/DROP\s/i.test(statement)).join('|'); +} diff --git a/packages/kbn-es-query/src/es_query/index.ts b/packages/kbn-es-query/src/es_query/index.ts index 22141a52e93f5..18009145a432f 100644 --- a/packages/kbn-es-query/src/es_query/index.ts +++ b/packages/kbn-es-query/src/es_query/index.ts @@ -20,6 +20,7 @@ export { getIndexPatternFromSQLQuery, getLanguageDisplayName, getIndexPatternFromESQLQuery, + cleanupESQLQueryForLensSuggestions, } from './es_aggregate_query'; export { fromCombinedFilter } from './from_combined_filter'; export type { diff --git a/packages/kbn-unified-data-table/__mocks__/data_view_without_timefield.ts b/packages/kbn-unified-data-table/__mocks__/data_view_without_timefield.ts new file mode 100644 index 0000000000000..cc07103a54486 --- /dev/null +++ b/packages/kbn-unified-data-table/__mocks__/data_view_without_timefield.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { DataView } from '@kbn/data-views-plugin/public'; +import { buildDataViewMock } from '@kbn/discover-utils/src/__mocks__'; + +const fields = [ + { + name: '_index', + type: 'string', + scripted: false, + filterable: true, + }, + { + name: 'timestamp', + displayName: 'timestamp', + type: 'date', + scripted: false, + filterable: true, + aggregatable: true, + sortable: true, + }, + { + name: 'message', + displayName: 'message', + type: 'string', + scripted: false, + filterable: false, + }, + { + name: 'extension', + displayName: 'extension', + type: 'string', + scripted: false, + filterable: true, + aggregatable: true, + }, + { + name: 'bytes', + displayName: 'bytes', + type: 'number', + scripted: false, + filterable: true, + aggregatable: true, + }, + { + name: 'scripted', + displayName: 'scripted', + type: 'number', + scripted: true, + filterable: false, + }, +] as DataView['fields']; + +export const dataViewWithoutTimefieldMock = buildDataViewMock({ + name: 'index-pattern-without-timefield', + fields, + timeFieldName: undefined, +}); diff --git a/packages/kbn-unified-data-table/src/components/data_table.tsx b/packages/kbn-unified-data-table/src/components/data_table.tsx index 4ce88e52e6c8d..1f05601fd6892 100644 --- a/packages/kbn-unified-data-table/src/components/data_table.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table.tsx @@ -56,7 +56,12 @@ import { getDisplayedColumns } from '../utils/columns'; import { convertValueToString } from '../utils/convert_value_to_string'; import { getRowsPerPageOptions } from '../utils/rows_per_page'; import { getRenderCellValueFn } from '../utils/get_render_cell_value'; -import { getEuiGridColumns, getLeadControlColumns, getVisibleColumns } from './data_table_columns'; +import { + getEuiGridColumns, + getLeadControlColumns, + getVisibleColumns, + hasSourceTimeFieldValue, +} from './data_table_columns'; import { UnifiedDataTableContext } from '../table_context'; import { getSchemaDetectors } from './data_table_schema'; import { DataTableDocumentToolbarBtn } from './data_table_document_selection'; @@ -617,9 +622,15 @@ export const UnifiedDataTable = ({ [dataView, onFieldEdited, services.dataViewFieldEditor] ); + const shouldShowTimeField = useMemo( + () => + hasSourceTimeFieldValue(displayedColumns, dataView, columnTypes, showTimeCol, isPlainRecord), + [dataView, displayedColumns, isPlainRecord, showTimeCol, columnTypes] + ); + const visibleColumns = useMemo( - () => getVisibleColumns(displayedColumns, dataView, showTimeCol), - [dataView, displayedColumns, showTimeCol] + () => getVisibleColumns(displayedColumns, dataView, shouldShowTimeField), + [dataView, displayedColumns, shouldShowTimeField] ); const getCellValue = useCallback( diff --git a/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx b/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx index 38cbdb5aeb63a..e1d7082353e1c 100644 --- a/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx @@ -7,8 +7,14 @@ */ import { dataViewMock } from '@kbn/discover-utils/src/__mocks__'; -import { getEuiGridColumns, getVisibleColumns } from './data_table_columns'; +import type { DataView } from '@kbn/data-views-plugin/public'; +import { + getEuiGridColumns, + getVisibleColumns, + hasSourceTimeFieldValue, +} from './data_table_columns'; import { dataViewWithTimefieldMock } from '../../__mocks__/data_view_with_timefield'; +import { dataViewWithoutTimefieldMock } from '../../__mocks__/data_view_without_timefield'; import { dataTableContextMock } from '../../__mocks__/table_context'; import { servicesMock } from '../../__mocks__/services'; @@ -110,6 +116,133 @@ describe('Data table columns', function () { }); }); + describe('hasSourceTimeFieldValue', () => { + function buildColumnTypes(dataView: DataView) { + const columnTypes: Record = {}; + for (const field of dataView.fields) { + columnTypes[field.name] = ''; + } + return columnTypes; + } + + describe('dataView with timeField', () => { + it('should forward showTimeCol if no _source columns is passed', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['extension', 'message'], + dataViewWithTimefieldMock, + buildColumnTypes(dataViewWithTimefieldMock), + showTimeCol, + false + ) + ).toBe(showTimeCol); + } + }); + + it('should forward showTimeCol if no _source columns is passed, text-based datasource', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['extension', 'message'], + dataViewWithTimefieldMock, + buildColumnTypes(dataViewWithTimefieldMock), + showTimeCol, + true + ) + ).toBe(showTimeCol); + } + }); + + it('should forward showTimeCol if _source column is passed', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['_source'], + dataViewWithTimefieldMock, + buildColumnTypes(dataViewWithTimefieldMock), + showTimeCol, + false + ) + ).toBe(showTimeCol); + } + }); + + it('should return true if _source column is passed, text-based datasource', () => { + // ... | DROP @timestamp test case + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['_source'], + dataViewWithTimefieldMock, + buildColumnTypes(dataViewWithTimefieldMock), + showTimeCol, + true + ) + ).toBe(true); + } + }); + }); + + describe('dataView without timeField', () => { + it('should forward showTimeCol if no _source columns is passed', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['extension', 'message'], + dataViewWithoutTimefieldMock, + buildColumnTypes(dataViewWithoutTimefieldMock), + showTimeCol, + false + ) + ).toBe(showTimeCol); + } + }); + + it('should forward showTimeCol if no _source columns is passed, text-based datasource', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['extension', 'message'], + dataViewWithoutTimefieldMock, + buildColumnTypes(dataViewWithoutTimefieldMock), + showTimeCol, + true + ) + ).toBe(showTimeCol); + } + }); + + it('should forward showTimeCol if _source column is passed', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['_source'], + dataViewWithoutTimefieldMock, + buildColumnTypes(dataViewWithoutTimefieldMock), + showTimeCol, + false + ) + ).toBe(showTimeCol); + } + }); + + it('should return false if _source column is passed, text-based datasource', () => { + for (const showTimeCol of [true, false]) { + expect( + hasSourceTimeFieldValue( + ['_source'], + dataViewWithoutTimefieldMock, + buildColumnTypes(dataViewWithoutTimefieldMock), + showTimeCol, + true + ) + ).toBe(showTimeCol); + } + }); + }); + }); + describe('column tokens', () => { it('returns eui grid columns with tokens', async () => { const actual = getEuiGridColumns({ diff --git a/packages/kbn-unified-data-table/src/components/data_table_columns.tsx b/packages/kbn-unified-data-table/src/components/data_table_columns.tsx index 274b1148df4eb..f7a3d41bf330e 100644 --- a/packages/kbn-unified-data-table/src/components/data_table_columns.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table_columns.tsx @@ -267,6 +267,20 @@ export function getEuiGridColumns({ ); } +export function hasSourceTimeFieldValue( + columns: string[], + dataView: DataView, + columnTypes: DataTableColumnTypes | undefined, + showTimeCol: boolean, + isPlainRecord: boolean +) { + const timeFieldName = dataView.timeFieldName; + if (!isPlainRecord || !columns.includes('_source') || !timeFieldName || !columnTypes) { + return showTimeCol; + } + return timeFieldName in columnTypes; +} + export function getVisibleColumns(columns: string[], dataView: DataView, showTimeCol: boolean) { const timeFieldName = dataView.timeFieldName; diff --git a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts index 7bd7e6f21ed5e..119356af6f63f 100644 --- a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts +++ b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts @@ -143,6 +143,45 @@ describe('useLensSuggestions', () => { }); }); + test('should return histogramSuggestion even if the ESQL query contains a DROP @timestamp statement', async () => { + const firstMockReturn = undefined; + const secondMockReturn = allSuggestionsMock; + const lensSuggestionsApi = jest + .fn() + .mockReturnValueOnce(firstMockReturn) // will return to firstMockReturn object firstly + .mockReturnValueOnce(secondMockReturn); // will return to secondMockReturn object secondly + + renderHook(() => { + return useLensSuggestions({ + dataView: dataViewMock, + query: { esql: 'from the-data-view | DROP @timestamp | limit 100' }, + isPlainRecord: true, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + data: dataMock, + lensSuggestionsApi, + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + }); + }); + expect(lensSuggestionsApi).toHaveBeenLastCalledWith( + expect.objectContaining({ + query: { esql: expect.stringMatching('from the-data-view | limit 100 ') }, + }), + expect.anything(), + ['lnsDatatable'] + ); + }); + test('should not return histogramSuggestion if no suggestions returned by the api and transformational commands', async () => { const firstMockReturn = undefined; const secondMockReturn = allSuggestionsMock; diff --git a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts index 5441e08a28c71..063e1b7ef89a2 100644 --- a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts +++ b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts @@ -11,6 +11,7 @@ import { AggregateQuery, isOfAggregateQueryType, getAggregateQueryMode, + cleanupESQLQueryForLensSuggestions, Query, TimeRange, } from '@kbn/es-query'; @@ -85,7 +86,8 @@ export const useLensSuggestions = ({ const interval = computeInterval(timeRange, data); const language = getAggregateQueryMode(query); - const esqlQuery = `${query[language]} | EVAL timestamp=DATE_TRUNC(${interval}, ${dataView.timeFieldName}) | stats rows = count(*) by timestamp | rename timestamp as \`${dataView.timeFieldName} every ${interval}\``; + const safeQuery = cleanupESQLQueryForLensSuggestions(query[language]); + const esqlQuery = `${safeQuery} | EVAL timestamp=DATE_TRUNC(${interval}, ${dataView.timeFieldName}) | stats rows = count(*) by timestamp | rename timestamp as \`${dataView.timeFieldName} every ${interval}\``; const context = { dataViewSpec: dataView?.toSpec(), fieldName: '',