diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index d3497a63cc462..a1a22785d60be 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -102,7 +102,7 @@ pageLoadAssetSize: kibanaUtils: 54161 kql: 15428 kubernetesSecurity: 6807 - lens: 86000 + lens: 97240 licenseManagement: 8265 licensing: 10073 links: 8620 diff --git a/src/platform/packages/shared/kbn-esql-utils/index.ts b/src/platform/packages/shared/kbn-esql-utils/index.ts index 1ed1f4e079872..8a51b920d34d1 100644 --- a/src/platform/packages/shared/kbn-esql-utils/index.ts +++ b/src/platform/packages/shared/kbn-esql-utils/index.ts @@ -9,6 +9,7 @@ export { getESQLAdHocDataview, + getESQLTimeFieldFromQuery, getIndexPatternFromESQLQuery, getSourceCommandFromESQLQuery, hasTransformationalCommand, diff --git a/src/platform/packages/shared/kbn-esql-utils/src/index.ts b/src/platform/packages/shared/kbn-esql-utils/src/index.ts index ce5903ae9f9fc..795eaed78bc4f 100644 --- a/src/platform/packages/shared/kbn-esql-utils/src/index.ts +++ b/src/platform/packages/shared/kbn-esql-utils/src/index.ts @@ -8,6 +8,7 @@ */ export { getESQLAdHocDataview, getIndexForESQLQuery } from './utils/get_esql_adhoc_dataview'; +export { getESQLTimeFieldFromQuery } from './utils/get_esql_time_field_from_query'; export { getInitialESQLQuery } from './utils/get_initial_esql_query'; export { getESQLWithSafeLimit } from './utils/get_esql_with_safe_limit'; export { diff --git a/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_adhoc_dataview.ts b/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_adhoc_dataview.ts index 00eb2a41cd74f..7ed16575f376e 100644 --- a/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_adhoc_dataview.ts +++ b/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_adhoc_dataview.ts @@ -9,19 +9,9 @@ import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { HttpStart } from '@kbn/core/public'; import { ESQL_TYPE } from '@kbn/data-view-utils'; -import { LRUCache } from 'lru-cache'; -import { - type ESQLSourceResult, - SOURCES_AUTOCOMPLETE_ROUTE, - TIMEFIELD_ROUTE, -} from '@kbn/esql-types'; +import { type ESQLSourceResult, SOURCES_AUTOCOMPLETE_ROUTE } from '@kbn/esql-types'; import { getIndexPatternFromESQLQuery } from './get_index_pattern_from_query'; - -// Caches the in-flight or resolved TIMEFIELD_ROUTE promise by query. -// Storing the Promise (not the resolved value) deduplicates concurrent calls: -// if multiple callers request the same query before the first resolves, -// they all await the same promise instead of each firing a separate HTTP request. -const timeFieldCache = new LRUCache>({ max: 100 }); +import { getESQLTimeFieldFromQuery } from './get_esql_time_field_from_query'; // uses browser sha256 method with fallback if unavailable async function sha256(str: string) { @@ -83,23 +73,7 @@ export async function getESQLAdHocDataview({ // optional http service to use to fetch the time field, if needed http?: HttpStart; }) { - let timeFieldName: string | undefined; - if (timeFieldCache.has(query)) { - timeFieldName = await timeFieldCache.get(query); - } else if (http) { - const encodedQuery = encodeURIComponent(query); - const pendingRequest = http - .get(`${TIMEFIELD_ROUTE}${encodedQuery}`) - .then((response) => (response as { timeField?: string } | undefined)?.timeField) - .catch((error) => { - // eslint-disable-next-line no-console - console.error('Failed to fetch the timefield', error); - timeFieldCache.delete(query); - return undefined; - }); - timeFieldCache.set(query, pendingRequest); - timeFieldName = await pendingRequest; - } + const timeFieldName = await getESQLTimeFieldFromQuery({ query, http }); const indexPattern = getIndexPatternFromESQLQuery(query); const prefix = options?.idPrefix ?? 'esql'; diff --git a/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_time_field_from_query.ts b/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_time_field_from_query.ts new file mode 100644 index 0000000000000..f0053ef062f54 --- /dev/null +++ b/src/platform/packages/shared/kbn-esql-utils/src/utils/get_esql_time_field_from_query.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +import type { HttpStart } from '@kbn/core/public'; +import { TIMEFIELD_ROUTE } from '@kbn/esql-types'; +import { LRUCache } from 'lru-cache'; + +// Caches the in-flight or resolved TIMEFIELD_ROUTE promise by query. +// Storing the Promise (not the resolved value) deduplicates concurrent calls: +// if multiple callers request the same query before the first resolves, +// they all await the same promise instead of each firing a separate HTTP request. +const timeFieldCache = new LRUCache>({ max: 100 }); + +/** + * Resolves the default time field for an ES|QL query by calling the timefield API. + * + * When `http` is omitted, returns `undefined` (unless a prior successful request + * for the same query left a value in the in-memory cache). + * + * Concurrent requests for the same query share one HTTP request via an LRU-backed promise cache. + */ +export async function getESQLTimeFieldFromQuery({ + query, + http, +}: { + query: string; + http?: HttpStart; +}): Promise { + const cached = timeFieldCache.get(query); + if (cached !== undefined) { + return cached; + } + if (!http) { + return undefined; + } + const pendingRequest = http + // eslint-disable-next-line @kbn/eslint/no_unsafe_dynamic_http_path -- buildPath can't be used in common package + .get(`${TIMEFIELD_ROUTE}${encodeURIComponent(query)}`) + .then((response) => (response as { timeField?: string } | undefined)?.timeField) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('Failed to fetch the timefield', error); + timeFieldCache.delete(query); + return undefined; + }); + timeFieldCache.set(query, pendingRequest); + return pendingRequest; +} diff --git a/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.test.ts b/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.test.ts index 7b6fbf0eed6eb..c1cdc1a53e5ac 100644 --- a/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.test.ts +++ b/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.test.ts @@ -5,13 +5,9 @@ * 2.0. */ -import type { DataViewsContract, DataViewField, DataViewSpec } from '@kbn/data-views-plugin/public'; -import type { TextBasedPersistedState } from '@kbn/lens-common'; -import type { HttpStart } from '@kbn/core/public'; -import { getESQLAdHocDataview } from '@kbn/esql-utils'; +import type { DataViewsContract, DataViewField } from '@kbn/data-views-plugin/public'; import { ensureIndexPattern, - ensureESQLTimeFieldOnAdHocDataViews, loadIndexPatternRefs, loadIndexPatterns, buildIndexPatternField, @@ -23,10 +19,6 @@ jest.mock('@kbn/esql-utils', () => ({ getESQLAdHocDataview: jest.fn(), })); -const mockGetESQLAdHocDataview = getESQLAdHocDataview as jest.MockedFunction< - typeof getESQLAdHocDataview ->; - describe('loader', () => { describe('loadIndexPatternRefs', () => { it('should return a list of sorted indexpattern refs', async () => { @@ -358,205 +350,4 @@ describe('loader', () => { expect(field.meta).toEqual(true); }); }); - - describe('ensureESQLTimeFieldOnAdHocDataViews', () => { - const mockHttp = {} as HttpStart; - const mockDataViews = mockDataViewsService() as unknown as DataViewsContract; - - beforeEach(() => { - mockGetESQLAdHocDataview.mockReset(); - }); - - it('should return adHocDataViews unchanged when textBasedState is undefined', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*' }, - }; - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState: undefined, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result).toBe(adHocDataViews); - expect(mockGetESQLAdHocDataview).not.toHaveBeenCalled(); - }); - - it('should return adHocDataViews unchanged when layers is empty', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*' }, - }; - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState: { layers: {} } as TextBasedPersistedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result).toEqual(adHocDataViews); - expect(mockGetESQLAdHocDataview).not.toHaveBeenCalled(); - }); - - it('should skip layers without an ES|QL query', async () => { - const adHocDataViews: Record = {}; - const textBasedState = { - layers: { - layer1: { columns: [], index: 'dv1' }, - }, - } as unknown as TextBasedPersistedState; - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result).toEqual({}); - expect(mockGetESQLAdHocDataview).not.toHaveBeenCalled(); - }); - - it('should skip enrichment when the existing spec already has a timeFieldName', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*', timeFieldName: '@timestamp' }, - }; - const textBasedState = { - layers: { - layer1: { columns: [], index: 'dv1', query: { esql: 'FROM logs-*' } }, - }, - } as unknown as TextBasedPersistedState; - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result).toEqual(adHocDataViews); - expect(mockGetESQLAdHocDataview).not.toHaveBeenCalled(); - }); - - it('should call getESQLAdHocDataview when spec is missing timeFieldName', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*' }, - }; - const textBasedState = { - layers: { - layer1: { columns: [], index: 'dv1', query: { esql: 'FROM logs-*' } }, - }, - } as unknown as TextBasedPersistedState; - - mockGetESQLAdHocDataview.mockResolvedValue({ - id: 'dv1', - toSpec: () => ({ id: 'dv1', title: 'logs-*', timeFieldName: '@timestamp' }), - } as never); - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(mockGetESQLAdHocDataview).toHaveBeenCalledWith( - expect.objectContaining({ - query: 'FROM logs-*', - options: { - skipFetchFields: true, - createNewInstanceEvenIfCachedOneAvailable: true, - }, - http: mockHttp, - }) - ); - expect(result.dv1).toEqual({ id: 'dv1', title: 'logs-*', timeFieldName: '@timestamp' }); - }); - - it('should use freshDataView.id as the key when layer.index is falsy', async () => { - const adHocDataViews: Record = {}; - const textBasedState = { - layers: { - layer1: { columns: [], query: { esql: 'FROM logs-*' } }, - }, - } as unknown as TextBasedPersistedState; - - mockGetESQLAdHocDataview.mockResolvedValue({ - id: 'generated-id', - toSpec: () => ({ id: 'generated-id', title: 'logs-*', timeFieldName: '@timestamp' }), - } as never); - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result['generated-id']).toEqual({ - id: 'generated-id', - title: 'logs-*', - timeFieldName: '@timestamp', - }); - }); - - it('should handle mixed layers: only enrich specs missing timeFieldName', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*', timeFieldName: '@timestamp' }, - dv2: { id: 'dv2', title: 'metrics-*' }, - }; - const textBasedState = { - layers: { - layer1: { columns: [], index: 'dv1', query: { esql: 'FROM logs-*' } }, - layer2: { columns: [], index: 'dv2', query: { esql: 'FROM metrics-*' } }, - }, - } as unknown as TextBasedPersistedState; - - mockGetESQLAdHocDataview.mockResolvedValue({ - id: 'dv2', - toSpec: () => ({ id: 'dv2', title: 'metrics-*', timeFieldName: '@timestamp' }), - } as never); - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(mockGetESQLAdHocDataview).toHaveBeenCalledTimes(1); - expect(mockGetESQLAdHocDataview).toHaveBeenCalledWith( - expect.objectContaining({ query: 'FROM metrics-*' }) - ); - expect(result.dv1).toEqual(adHocDataViews.dv1); - expect(result.dv2.timeFieldName).toBe('@timestamp'); - }); - - it('should not mutate the original adHocDataViews object', async () => { - const adHocDataViews: Record = { - dv1: { id: 'dv1', title: 'logs-*' }, - }; - const textBasedState = { - layers: { - layer1: { columns: [], index: 'dv1', query: { esql: 'FROM logs-*' } }, - }, - } as unknown as TextBasedPersistedState; - - mockGetESQLAdHocDataview.mockResolvedValue({ - id: 'dv1', - toSpec: () => ({ id: 'dv1', title: 'logs-*', timeFieldName: '@timestamp' }), - } as never); - - const result = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService: mockDataViews, - http: mockHttp, - }); - - expect(result).not.toBe(adHocDataViews); - expect(adHocDataViews.dv1.timeFieldName).toBeUndefined(); - }); - }); }); diff --git a/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.ts b/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.ts index 6600e14de5eef..895701699ab98 100644 --- a/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.ts +++ b/x-pack/platform/plugins/shared/lens/public/data_views_service/loader.ts @@ -13,15 +13,11 @@ import type { DataViewField, } from '@kbn/data-views-plugin/public'; import { keyBy } from 'lodash'; -import type { HttpStart } from '@kbn/core/public'; -import { getESQLAdHocDataview } from '@kbn/esql-utils'; -import { isOfAggregateQueryType } from '@kbn/es-query'; import type { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef, - TextBasedPersistedState, } from '@kbn/lens-common'; import { documentField } from '../datasources/form_based/document_field'; import { sortDataViewRefs } from '../utils'; @@ -159,67 +155,6 @@ function onRestrictionMapping(agg: string): string { return agg in renameOperationsMapping ? renameOperationsMapping[agg] : agg; } -/** - * Ensures ESQL ad-hoc DataView specs have a valid `timeFieldName` if any. - * - * Persisted specs may be missing time field info. For each text-based layer with - * an ES|QL query, this function checks whether the corresponding ad-hoc DataView - * spec already has a `timeFieldName`. If it does, the spec is kept as-is. If not, - * `getESQLAdHocDataview` is called to detect the time field via the TIMEFIELD_ROUTE. - * - * After calling this function the DataViewService instance cache is also populated - * with the correct DataView, so downstream `dataViews.create(spec)` calls - * (in `loadIndexPatterns`, `getUsedDataViews`, etc.) will return the cached instance - * with the right time field — even if they receive a stale spec. - */ -export async function ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews, - textBasedState, - dataViewsService, - http, -}: { - adHocDataViews: Record; - textBasedState: TextBasedPersistedState | undefined; - dataViewsService: DataViewsContract; - http?: HttpStart; -}): Promise> { - if (!textBasedState?.layers) { - return adHocDataViews; - } - - const result = { ...adHocDataViews }; - - for (const layer of Object.values(textBasedState.layers)) { - if (!layer.query || !isOfAggregateQueryType(layer.query)) { - continue; - } - - const existingSpec = layer.index ? result[layer.index] : undefined; - - // Skip regeneration when the persisted spec already has a timeFieldName - if (existingSpec?.timeFieldName) { - continue; - } - - const freshDataView = await getESQLAdHocDataview({ - dataViewsService, - query: layer.query.esql, - options: { - skipFetchFields: true, - createNewInstanceEvenIfCachedOneAvailable: true, - }, - http, - }); - const spec = freshDataView.toSpec(false); - - if (freshDataView.id) { - result[freshDataView.id] = spec; - } - } - - return result; -} - export async function loadIndexPatterns({ dataViews, patterns, diff --git a/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.test.ts b/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.test.ts index 97161bf12da37..8f602cb8a761e 100644 --- a/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.test.ts +++ b/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.test.ts @@ -872,100 +872,6 @@ describe('Textbased Data Source', () => { }); }); - describe('#initialize', () => { - it('should hydrate timeField from indexPatterns when layer has no timeField', () => { - const state = { - layers: { - a: { - columns: [{ columnId: 'col1', fieldName: 'bytes', meta: { type: 'number' } }], - query: { esql: 'FROM foo' }, - index: '1', - }, - }, - } as unknown as TextBasedPersistedState; - - const result = TextBasedDatasource.initialize(state, [], undefined, undefined, indexPatterns); - expect(result.layers.a.timeField).toBe('timestamp'); - }); - - it('should not overwrite timeField when layer already has one', () => { - const state = { - layers: { - a: { - columns: [{ columnId: 'col1', fieldName: 'bytes', meta: { type: 'number' } }], - query: { esql: 'FROM foo' }, - index: '1', - timeField: 'custom_time', - }, - }, - } as unknown as TextBasedPersistedState; - - const result = TextBasedDatasource.initialize(state, [], undefined, undefined, indexPatterns); - expect(result.layers.a.timeField).toBe('custom_time'); - }); - - it('should not hydrate when indexPatterns is undefined', () => { - const state = { - layers: { - a: { - columns: [{ columnId: 'col1', fieldName: 'bytes', meta: { type: 'number' } }], - query: { esql: 'FROM foo' }, - index: '1', - }, - }, - } as unknown as TextBasedPersistedState; - - const result = TextBasedDatasource.initialize(state, [], undefined, undefined, undefined); - expect(result.layers.a.timeField).toBeUndefined(); - }); - - it('should leave layer unchanged when layer.index does not match any indexPattern', () => { - const state = { - layers: { - a: { - columns: [{ columnId: 'col1', fieldName: 'bytes', meta: { type: 'number' } }], - query: { esql: 'FROM unknown' }, - index: 'non-existent', - }, - }, - } as unknown as TextBasedPersistedState; - - const result = TextBasedDatasource.initialize(state, [], undefined, undefined, indexPatterns); - expect(result.layers.a.timeField).toBeUndefined(); - }); - - it('should hydrate each layer independently', () => { - const patternsWithNoTime = { - ...indexPatterns, - '2': { ...indexPatterns['1'], id: '2', timeFieldName: undefined }, - }; - const state = { - layers: { - a: { - columns: [{ columnId: 'col1', fieldName: 'bytes', meta: { type: 'number' } }], - query: { esql: 'FROM foo' }, - index: '1', - }, - b: { - columns: [{ columnId: 'col2', fieldName: 'src', meta: { type: 'string' } }], - query: { esql: 'FROM bar' }, - index: '2', - }, - }, - } as unknown as TextBasedPersistedState; - - const result = TextBasedDatasource.initialize( - state, - [], - undefined, - undefined, - patternsWithNoTime - ); - expect(result.layers.a.timeField).toBe('timestamp'); - expect(result.layers.b.timeField).toBeUndefined(); - }); - }); - describe('#toExpression', () => { it('should generate an empty expression when no columns are selected', async () => { const state = TextBasedDatasource.initialize(); diff --git a/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.tsx b/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.tsx index 79d60737bdacf..aee193b52623f 100644 --- a/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.tsx +++ b/x-pack/platform/plugins/shared/lens/public/datasources/text_based/text_based_languages.tsx @@ -349,28 +349,9 @@ export function getTextBasedDatasource({ }); const initState = state ?? { layers: {} }; - const hydratedLayers: typeof initState.layers = {}; - - // Validate the layers without a timeField configured at runtime. - // The ad-hoc DataView specs are regenerated at initialization (with time field - // detection via HTTP), but the persisted layer state may not have a timeField (if comes from the API) - // This ensures each layer picks up the correct timeFieldName so that - // time-based filtering works correctly for ES|QL visualizations. - for (const [layerId, layer] of Object.entries(initState.layers)) { - if (layer.timeField || !indexPatterns) { - hydratedLayers[layerId] = layer; - continue; - } - - const matchedIndexPattern = layer.index ? indexPatterns[layer.index] : undefined; - hydratedLayers[layerId] = matchedIndexPattern?.timeFieldName - ? { ...layer, timeField: matchedIndexPattern.timeFieldName } - : layer; - } return { ...initState, - layers: hydratedLayers, indexPatternRefs: refs, initialContext: context, }; diff --git a/x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/state_helpers.ts b/x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/state_helpers.ts index 1713871de9e1b..60467862c308b 100644 --- a/x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/state_helpers.ts +++ b/x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/state_helpers.ts @@ -34,18 +34,13 @@ import type { VisualizationState, DocumentToExpressionReturnType, LensDocument, - TextBasedPersistedState, } from '@kbn/lens-common'; import { COLOR_MAPPING_OFF_BY_DEFAULT } from '../../../common/constants'; import { buildExpression } from './expression_helpers'; import { getActiveDatasourceIdFromDoc, sortDataViewRefs } from '../../utils'; import { readFromStorage } from '../../settings_storage'; -import { - loadIndexPatternRefs, - loadIndexPatterns, - ensureESQLTimeFieldOnAdHocDataViews, -} from '../../data_views_service/loader'; +import { loadIndexPatternRefs, loadIndexPatterns } from '../../data_views_service/loader'; import { getDatasourceLayers } from '../../state_management/utils'; // there are 2 ways of coloring, the color mapping where the user can map specific colors to @@ -262,17 +257,6 @@ export async function initializeSources( references ); - // Regenerate ESQL ad-hoc DataViews once at editor initialization. - // This replaces potentially stale persisted specs with fresh ones derived - // from the actual ES|QL queries, including time field detection via http. - const textBasedState = datasourceStates.textBased?.state as TextBasedPersistedState | undefined; - const refreshedAdHocDataViews = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews: adHocDataViews ?? {}, - textBasedState, - dataViewsService: dataViews, - http, - }); - const { indexPatternRefs, indexPatterns } = await initializeDataViews( { datasourceMap, @@ -282,7 +266,7 @@ export async function initializeSources( storage, defaultIndexPatternId, references, - adHocDataViews: refreshedAdHocDataViews, + adHocDataViews, annotationGroups, }, options @@ -425,18 +409,6 @@ export async function persistedStateToExpression( ]) ); - // Ensure ESQL ad-hoc DataViews have the correct time field before - // initializing DataViews — same as in initializeSources for the editor path. - const textBasedState = datasourceStatesFromSO.textBased?.state as - | TextBasedPersistedState - | undefined; - const refreshedAdHocDataViews = await ensureESQLTimeFieldOnAdHocDataViews({ - adHocDataViews: adHocDataViews ?? {}, - textBasedState, - dataViewsService: services.dataViews, - http: services.http, - }); - const { indexPatterns, indexPatternRefs } = await initializeDataViews( { datasourceMap, @@ -445,7 +417,7 @@ export async function persistedStateToExpression( dataViews: services.dataViews, storage: services.storage, defaultIndexPatternId: services.uiSettings.get('defaultIndex'), - adHocDataViews: refreshedAdHocDataViews, + adHocDataViews, annotationGroups, }, { isFullEditor: false } diff --git a/x-pack/platform/plugins/shared/lens/public/react_embeddable/helper.ts b/x-pack/platform/plugins/shared/lens/public/react_embeddable/helper.ts index 07a1c044a644d..1b0f60c74b07b 100644 --- a/x-pack/platform/plugins/shared/lens/public/react_embeddable/helper.ts +++ b/x-pack/platform/plugins/shared/lens/public/react_embeddable/helper.ts @@ -39,12 +39,12 @@ import { BehaviorSubject } from 'rxjs'; import { LENS_ITEM_LATEST_VERSION } from '@kbn/lens-common/content_management/constants'; import { isLensAPIFormat } from '@kbn/lens-embeddable-utils/config_builder/utils'; - import type { StrippedLensState } from '../../common/transforms/helpers'; import { isFlattenedAPIConfig, unflattenAPIConfig } from '../../common/transforms/utils'; import { getLensBuilder } from '../lazy_builder'; import type { ESQLStartServices } from './esql'; import { loadESQLAttributes } from './esql'; +import { hydrateESQLTimeFields } from './hydrate_esql_time_fields'; import type { LensEmbeddableStartServices } from './types'; import type { FlattenedLensByValuePanelSchema } from '../../server/types'; @@ -93,10 +93,12 @@ export async function deserializeState( try { const { attributes, managed, sharingSavedObjectProps } = await attributeService.loadFromLibrary(refId); + // hydrate by ref API/ESQL attributes with the time field and date column types + const hydratedAttributes = await hydrateESQLTimeFields(attributes, services.coreStart.http); return { ...state, ref_id: refId, - attributes, + attributes: hydratedAttributes, managed, sharingSavedObjectProps, } satisfies LensRuntimeState; @@ -121,8 +123,16 @@ export async function deserializeState( return { ...newState, attributes: fallbackAttributes }; } } + // hydrate by value API/ESQL attributes with the time field and date column types + const hydratedAttributes = await hydrateESQLTimeFields( + newState.attributes ?? fallbackAttributes, + services.coreStart.http + ); - return newState; + return { + ...newState, + attributes: hydratedAttributes, + }; } export function isTextBasedLanguage(state: LensRuntimeState) { diff --git a/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.test.ts b/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.test.ts new file mode 100644 index 0000000000000..778ef8a7b24ea --- /dev/null +++ b/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.test.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock } from '@kbn/core/public/mocks'; +import * as esqlUtils from '@kbn/esql-utils'; +import type { LensRuntimeState, TextBasedPersistedState } from '@kbn/lens-common'; + +// Barrel re-exports are non-configurable; wrapping the module makes spyOn work. +jest.mock('@kbn/esql-utils', () => ({ + __esModule: true, + ...jest.requireActual('@kbn/esql-utils'), +})); +import { LENS_ITEM_LATEST_VERSION } from '@kbn/lens-common/content_management/constants'; +import expect from 'expect'; +import { hydrateESQLTimeFields } from './hydrate_esql_time_fields'; + +describe('hydrateESQLTimeFields', () => { + const http = coreMock.createStart().http; + const textBasedEsqlAttributes: LensRuntimeState['attributes'] = { + version: LENS_ITEM_LATEST_VERSION, + title: '', + description: '', + visualizationType: 'lnsXY', + references: [], + state: { + query: { query: '', language: 'kuery' }, + filters: [], + internalReferences: [], + datasourceStates: { + textBased: { + layers: { + layer1: { + query: { esql: 'FROM logs-* | LIMIT 10' }, + columns: [], + }, + }, + }, + }, + visualization: {}, + }, + }; + it('returns input attributes when time-field resolution throws', async () => { + const spy = jest + .spyOn(esqlUtils, 'getESQLTimeFieldFromQuery') + .mockRejectedValue(new Error('network')); + const result = await hydrateESQLTimeFields(textBasedEsqlAttributes, http); + expect(result).toBe(textBasedEsqlAttributes); + spy.mockRestore(); + }); + + it('does not mark TBUCKET on a non-time field as date type', async () => { + const attrs: LensRuntimeState['attributes'] = { + version: LENS_ITEM_LATEST_VERSION, + title: '', + description: '', + visualizationType: 'lnsXY', + references: [], + state: { + query: { query: '', language: 'kuery' }, + filters: [], + internalReferences: [], + datasourceStates: { + textBased: { + layers: { + layer1: { + query: { + esql: 'FROM logs-* | STATS count() BY ts = TBUCKET(other_date_field, 1 day)', + }, + timeField: '@timestamp', + columns: [ + { columnId: 'c1', fieldName: 'count()', meta: { type: 'number' } }, + { columnId: 'c2', fieldName: 'ts', meta: { type: 'string' } }, + ], + }, + }, + }, + }, + visualization: {}, + }, + }; + const result = await hydrateESQLTimeFields(attrs, http); + const { layers } = result.state.datasourceStates?.textBased as TextBasedPersistedState; + expect(layers.layer1.columns[1].meta?.type).toBe('string'); + }); + + it('marks TBUCKET on the time field as date type', async () => { + const attrs: LensRuntimeState['attributes'] = { + version: LENS_ITEM_LATEST_VERSION, + title: '', + description: '', + visualizationType: 'lnsXY', + references: [], + state: { + query: { query: '', language: 'kuery' }, + filters: [], + internalReferences: [], + datasourceStates: { + textBased: { + layers: { + layer1: { + query: { + esql: 'FROM logs-* | STATS count() BY ts = TBUCKET(@timestamp, 1 day)', + }, + timeField: '@timestamp', + columns: [ + { columnId: 'c1', fieldName: 'count()', meta: { type: 'number' } }, + { columnId: 'c2', fieldName: 'ts', meta: { type: 'string' } }, + ], + }, + }, + }, + }, + visualization: {}, + }, + }; + const result = await hydrateESQLTimeFields(attrs, http); + const { layers } = result.state.datasourceStates?.textBased as TextBasedPersistedState; + expect(layers.layer1.columns[1].meta?.type).toBe('date'); + }); + + it('marks renamed direct time-field references as date type', async () => { + const attrs: LensRuntimeState['attributes'] = { + version: LENS_ITEM_LATEST_VERSION, + title: '', + description: '', + visualizationType: 'lnsXY', + references: [], + state: { + query: { query: '', language: 'kuery' }, + filters: [], + internalReferences: [], + datasourceStates: { + textBased: { + layers: { + layer1: { + query: { esql: 'FROM logs-* | STATS count() BY ts = @timestamp' }, + timeField: '@timestamp', + columns: [ + { columnId: 'c1', fieldName: 'count()', meta: { type: 'number' } }, + { columnId: 'c2', fieldName: 'ts', meta: { type: 'string' } }, + ], + }, + }, + }, + }, + visualization: {}, + }, + }; + const result = await hydrateESQLTimeFields(attrs, http); + const { layers } = result.state.datasourceStates?.textBased as TextBasedPersistedState; + expect(layers.layer1.columns[1].meta?.type).toBe('date'); + expect(layers.layer1.columns[0].meta?.type).toBe('number'); + }); +}); diff --git a/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.ts b/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.ts new file mode 100644 index 0000000000000..fd9daf0d3838d --- /dev/null +++ b/x-pack/platform/plugins/shared/lens/public/react_embeddable/hydrate_esql_time_fields.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isOfAggregateQueryType } from '@kbn/es-query'; +import { getESQLTimeFieldFromQuery, getQuerySummary } from '@kbn/esql-utils'; +import { isAssignment, singleItems, Walker } from '@elastic/esql'; +import type { LensRuntimeState, TextBasedPersistedState } from '@kbn/lens-common'; +import type { ESQLStartServices } from './esql'; + +/** + * Hydrates ES|QL text-based layers and their ad-hoc DataView specs with time + * field information that is not part of the persisted state. + * + * The Lens-as-code / API surface does not include a `timeFieldName` definition + * — for ES|QL visualizations the time field must be derived at runtime from the + * query and the underlying data (via the TIMEFIELD_ROUTE HTTP endpoint). This + * function performs that detection and patches three things: + * + * 1. `layer.timeField` — so downstream expression building applies time-based + * filtering correctly. + * 2. `adHocDataViews[id].timeFieldName` — so every `dataViews.create(spec)` + * call produces a DataView instance with the correct time field, preventing + * the DataViewsService cache from being polluted with a stale, time-field-less + * instance. + * 3. Time-field-derived columns (`meta.type` → `'date'`) — columns that + * directly reference the time field (including renamed references like + * `ts = @timestamp`) or are produced by BUCKET / TBUCKET / DATE_TRUNC on + * the time field are tagged as date type. + * + * This runs once during `deserializeState` — the earliest async entry point for + * the embeddable — before any downstream consumer (`getUsedDataViews`, + * `persistedStateToExpression`, etc.) can touch the DataViewsService cache. + * + * If the time-field HTTP request fails (for example a transient network error), + * that layer is left unchanged rather than rejecting deserialization. + */ +export async function hydrateESQLTimeFields( + attributes: LensRuntimeState['attributes'], + http: ESQLStartServices['coreStart']['http'] +): Promise { + const textBasedState = attributes.state?.datasourceStates?.textBased as + | TextBasedPersistedState + | undefined; + if (!textBasedState?.layers) { + return attributes; + } + + const adHocDataViews = { ...(attributes.state.adHocDataViews ?? {}) }; + const layers = { ...textBasedState.layers }; + let changed = false; + + for (const [layerId, layer] of Object.entries(layers)) { + if (!layer.query || !isOfAggregateQueryType(layer.query)) { + continue; + } + + let timeFieldName = layer.timeField; + if (!timeFieldName) { + try { + timeFieldName = + (await getESQLTimeFieldFromQuery({ query: layer.query.esql, http })) ?? undefined; + } catch { + // Network or server errors during time-field detection should not fail + // deserialization; continue without runtime hydration for this layer. + } + } + + if (!timeFieldName) { + continue; + } + + const temporalColumns = getTimeFieldDerivedColumns(layer.query.esql, timeFieldName); + const columns = + temporalColumns.size > 0 + ? layer.columns.map((c) => + c.fieldName && temporalColumns.has(c.fieldName) && c.meta?.type !== 'date' + ? { ...c, meta: { ...c.meta, type: 'date' as const } } + : c + ) + : layer.columns; + + if (layer.timeField !== timeFieldName || columns !== layer.columns) { + layers[layerId] = { ...layer, timeField: timeFieldName, columns }; + changed = true; + } + + if (layer.index && adHocDataViews[layer.index] && !adHocDataViews[layer.index].timeFieldName) { + adHocDataViews[layer.index] = { ...adHocDataViews[layer.index], timeFieldName }; + changed = true; + } + } + + if (!changed) { + return attributes; + } + + return { + ...attributes, + state: { + ...attributes.state, + datasourceStates: { + ...attributes.state.datasourceStates, + textBased: { ...textBasedState, layers }, + }, + adHocDataViews, + }, + }; +} + +const TEMPORAL_GROUPING_FUNCTIONS = new Set(['bucket', 'tbucket', 'date_trunc']); + +/** + * Given an ES|QL query and its detected timeFieldName, returns the set of + * output column names that are time-field-derived. This includes: + * + * - Direct references to the time field (e.g. `... BY @timestamp`). + * - Renamed references (e.g. `... BY ts = @timestamp`). + * - Temporal grouping functions — BUCKET, TBUCKET, or DATE_TRUNC — applied + * to the time field (e.g. `... BY bucket(@timestamp, 1 day)`). + * + * Returns an empty set when the query contains no such columns. + */ +function getTimeFieldDerivedColumns(esqlQuery: string, timeFieldName: string): Set { + const result = new Set(); + try { + const { grouping } = getQuerySummary(esqlQuery); + if (!grouping) return result; + + for (const { field, arg } of grouping) { + if (field === timeFieldName) { + result.add(field); + continue; + } + + const def = isAssignment(arg) ? [...singleItems(arg.args)][1] : arg; + if (!def) continue; + + if (def.type === 'column' && def.name === timeFieldName) { + result.add(field); + continue; + } + + if (def.type !== 'function') continue; + + const funcName = def.name.toLowerCase(); + if (funcName === '=') continue; + + if (!TEMPORAL_GROUPING_FUNCTIONS.has(funcName)) continue; + + let referencesTimeField = false; + Walker.walk(def, { + visitColumn(col) { + if (col.name === timeFieldName) referencesTimeField = true; + }, + }); + if (referencesTimeField) result.add(field); + } + } catch { + // Don't block initialization on parse errors + } + return result; +} diff --git a/x-pack/platform/plugins/shared/lens/test/scout/ui/parallel_tests/esql_timeseries_dashboard_api.spec.ts b/x-pack/platform/plugins/shared/lens/test/scout/ui/parallel_tests/esql_timeseries_dashboard_api.spec.ts new file mode 100644 index 0000000000000..f2fdd75601dd5 --- /dev/null +++ b/x-pack/platform/plugins/shared/lens/test/scout/ui/parallel_tests/esql_timeseries_dashboard_api.spec.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DebugState } from '@elastic/charts'; +import { LENS_EMBEDDABLE_TYPE } from '@kbn/lens-common'; +import type { KbnClient } from '@kbn/scout'; +import { spaceTest } from '@kbn/scout'; +import { expect } from '@kbn/scout/ui'; + +import { testData } from '../fixtures'; + +const DASHBOARD_API_HEADERS = { + 'Content-Type': 'application/json', + 'elastic-api-version': '2023-10-31', +} as const; + +/** Matches logstash_functional data used by other Lens Scout tests. */ +const LOGSTASH_ABSOLUTE_RANGE = { + from: '2015-09-19T06:31:44.000Z', + to: '2015-09-23T18:31:44.000Z', +} as const; + +type EsqlTimeseriesCase = Readonly<{ + description: string; + query: string; + xColumn: string; + yColumn: string; +}>; + +const TIMESERIES_CASES: EsqlTimeseriesCase[] = [ + { + description: 'TBUCKET', + query: 'FROM logstash-* | STATS count = COUNT(*) BY ts = TBUCKET(50)', + xColumn: 'ts', + yColumn: 'count', + }, + { + description: 'BUCKET on timestamp', + query: 'FROM logstash-* | STATS count = COUNT(*) BY ts = BUCKET(@timestamp, 1 hour)', + xColumn: 'ts', + yColumn: 'count', + }, + { + description: 'DATE_TRUNC on timestamp', + query: 'FROM logstash-* | STATS count = COUNT(*) BY ts = DATE_TRUNC(1 hour, @timestamp)', + xColumn: 'ts', + yColumn: 'count', + }, +]; + +function buildLensLineTimeseriesPanel(esql: EsqlTimeseriesCase) { + return { + type: LENS_EMBEDDABLE_TYPE, + grid: { x: 0, y: 0, w: 36, h: 20 }, + config: { + type: 'xy' as const, + title: `ES|QL ${esql.description}`, + layers: [ + { + type: 'line' as const, + ignore_global_filters: false, + sampling: 1, + data_source: { + type: 'esql' as const, + query: esql.query, + }, + x: { column: esql.xColumn }, + y: [{ column: esql.yColumn }], + }, + ], + }, + }; +} + +async function createDashboardWithLensPanel( + kbnClient: KbnClient, + spaceId: string, + title: string, + esql: EsqlTimeseriesCase +): Promise { + const response = await kbnClient.request<{ id: string }>({ + method: 'POST', + path: `/s/${spaceId}/api/dashboards`, + headers: DASHBOARD_API_HEADERS, + body: { + title, + time_range: { + from: LOGSTASH_ABSOLUTE_RANGE.from, + to: LOGSTASH_ABSOLUTE_RANGE.to, + mode: 'absolute' as const, + }, + panels: [buildLensLineTimeseriesPanel(esql)], + }, + }); + + expect([200, 201]).toContain(response.status); + expect(response.data.id).toBeTruthy(); + return response.data.id; +} + +function assertTemporalXAxis(debug: DebugState) { + const xAxis = debug.axes?.x?.[0]; + expect(xAxis, 'Expected chart debug state to include an x-axis').toBeDefined(); + + const tickValues = xAxis!.values ?? []; + expect(tickValues.length, 'Expected at least one x-axis tick value').toBeGreaterThan(0); + + const eachTickIsEpochMs = tickValues.every((v) => typeof v === 'number' && v > 1_000_000_000_000); + expect( + eachTickIsEpochMs, + `Expected x-axis tick values to be epoch milliseconds (time scale); got ${JSON.stringify( + tickValues.slice(0, 5) + )}` + ).toBe(true); +} + +spaceTest.describe( + 'Lens ES|QL timeseries via dashboard API', + { tag: '@local-stateful-classic' }, + () => { + spaceTest.beforeAll(async ({ scoutSpace }) => { + await scoutSpace.uiSettings.set({ + defaultIndex: testData.DATA_VIEW_ID.LOGSTASH, + 'dateFormat:tz': 'UTC', + 'timepicker:timeDefaults': `{ "from": "${testData.LOGSTASH_IN_RANGE_DATES.from}", "to": "${testData.LOGSTASH_IN_RANGE_DATES.to}"}`, + }); + }); + + spaceTest.beforeEach(async ({ context }) => { + await context.addInitScript(() => { + (window as unknown as { _echDebugStateFlag?: boolean })._echDebugStateFlag = true; + }); + }); + + spaceTest.afterAll(async ({ scoutSpace }) => { + await scoutSpace.uiSettings.unset('defaultIndex', 'dateFormat:tz', 'timepicker:timeDefaults'); + await scoutSpace.savedObjects.cleanStandardList(); + }); + + for (const esqlCase of TIMESERIES_CASES) { + spaceTest( + `renders a time-scaled x-axis for ${esqlCase.description}`, + async ({ browserAuth, kbnClient, page, pageObjects, scoutSpace }) => { + const title = `Scout ES|QL timeseries API ${esqlCase.description} ${Date.now()}`; + const dashboardId = await createDashboardWithLensPanel( + kbnClient, + scoutSpace.id, + title, + esqlCase + ); + + await browserAuth.loginAsPrivilegedUser(); + + const { dashboard } = pageObjects; + + await dashboard.openDashboardWithId(dashboardId); + await dashboard.waitForPanelsToLoad(1); + + await expect(page.testSubj.locator('embeddable-lens-failure')).toBeHidden(); + await expect(page.testSubj.locator('xyVisChart')).toBeVisible(); + + const chart = page.testSubj.locator('xyVisChart'); + await expect( + chart.locator('.echChartStatus[data-ech-render-complete="true"]') + ).toBeAttached({ + timeout: 30_000, + }); + + const chartStatus = chart.locator('.echChartStatus'); + const debugJson = await chartStatus.getAttribute('data-ech-debug-state'); + await expect(chartStatus, 'Elastic Charts debug state attribute missing').toHaveAttribute( + 'data-ech-debug-state' + ); + const debug = JSON.parse(debugJson ?? '{}') as DebugState; + + assertTemporalXAxis(debug); + } + ); + } + } +);