diff --git a/src/plugins/data/common/query/filters/persistable_state.test.ts b/src/plugins/data/common/query/filters/persistable_state.test.ts index 9ce3e2536a70a..392b029d8c7c2 100644 --- a/src/plugins/data/common/query/filters/persistable_state.test.ts +++ b/src/plugins/data/common/query/filters/persistable_state.test.ts @@ -26,7 +26,7 @@ describe('filter manager persistable state tests', () => { const updatedFilters = inject(filters, [ { type: DATA_VIEW_SAVED_OBJECT_TYPE, name: 'test123', id: '123' }, ]); - expect(updatedFilters[0]).toHaveProperty('meta.index', undefined); + expect(updatedFilters[0]).toHaveProperty('meta.index', 'test'); }); }); diff --git a/src/plugins/data/common/query/filters/persistable_state.ts b/src/plugins/data/common/query/filters/persistable_state.ts index a309573fb9df2..a2696723fbab7 100644 --- a/src/plugins/data/common/query/filters/persistable_state.ts +++ b/src/plugins/data/common/query/filters/persistable_state.ts @@ -46,7 +46,8 @@ export const inject = (filters: Filter[], references: SavedObjectReference[]) => ...filter, meta: { ...filter.meta, - index: reference && reference.id, + // if no reference has been found, keep the current "index" property (used for adhoc data views) + index: reference ? reference.id : filter.meta.index, }, }; }); diff --git a/src/plugins/data/common/query/persistable_state.test.ts b/src/plugins/data/common/query/persistable_state.test.ts index 2fcfce910ebb8..f44e5276ca7fe 100644 --- a/src/plugins/data/common/query/persistable_state.test.ts +++ b/src/plugins/data/common/query/persistable_state.test.ts @@ -41,7 +41,7 @@ describe('query service persistable state tests', () => { const updatedQueryState = inject(queryState, [ { type: DATA_VIEW_SAVED_OBJECT_TYPE, name: 'test123', id: '123' }, ]); - expect(updatedQueryState.filters[0]).toHaveProperty('meta.index', undefined); + expect(updatedQueryState.filters[0]).toHaveProperty('meta.index', 'test'); }); }); diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index 39ccfb44e7def..301e0a8c347b3 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -69,6 +69,7 @@ export function ChangeDataView({ onSaveTextLanguageQuery, onTextLangQuerySubmit, textBasedLanguage, + adHocDataViews, }: DataViewPickerPropsExtended) { const { euiTheme } = useEuiTheme(); const [isPopoverOpen, setPopoverIsOpen] = useState(false); @@ -93,10 +94,21 @@ export function ChangeDataView({ useEffect(() => { const fetchDataViews = async () => { const dataViewsRefs = await data.dataViews.getIdsWithTitle(); + if (adHocDataViews?.length) { + adHocDataViews.forEach((adHocDataView) => { + if (adHocDataView.id) { + dataViewsRefs.push({ + title: adHocDataView.title, + name: adHocDataView.name, + id: adHocDataView.id, + }); + } + }); + } setDataViewsList(dataViewsRefs); }; fetchDataViews(); - }, [data, currentDataViewId]); + }, [data, currentDataViewId, adHocDataViews]); useEffect(() => { if (trigger.label) { diff --git a/src/plugins/unified_search/public/dataview_picker/index.tsx b/src/plugins/unified_search/public/dataview_picker/index.tsx index 82909714cffad..2b3e20801f501 100644 --- a/src/plugins/unified_search/public/dataview_picker/index.tsx +++ b/src/plugins/unified_search/public/dataview_picker/index.tsx @@ -7,6 +7,7 @@ */ import React from 'react'; +import type { DataView } from '@kbn/data-views-plugin/public'; import type { EuiButtonProps, EuiSelectableProps } from '@elastic/eui'; import type { AggregateQuery, Query } from '@kbn/es-query'; import { ChangeDataView } from './change_dataview'; @@ -44,6 +45,10 @@ export interface DataViewPickerProps { * The id of the selected dataview. */ currentDataViewId?: string; + /** + * The adHocDataview selected. + */ + adHocDataViews?: DataView[]; /** * EuiSelectable properties. */ @@ -84,6 +89,7 @@ export interface DataViewPickerPropsExtended extends DataViewPickerProps { export const DataViewPicker = ({ isMissingCurrent, currentDataViewId, + adHocDataViews, onChangeDataView, onAddField, onDataViewCreated, @@ -98,6 +104,7 @@ export const DataViewPicker = ({ ) => dispatch(setState(state)), [dispatch] ); + const [indexPatterns, setIndexPatterns] = useState([]); + const [adHocDataViews, setAdHocDataViews] = useState(); + const [currentIndexPattern, setCurrentIndexPattern] = useState(); + const [rejectedIndexPatterns, setRejectedIndexPatterns] = useState([]); + const dispatchChangeIndexPattern = React.useCallback( - async (indexPatternId) => { + async (dataViewOrId) => { + const indexPatternId = typeof dataViewOrId === 'string' ? dataViewOrId : dataViewOrId.id; const [newIndexPatternRefs, newIndexPatterns] = await Promise.all([ // Reload refs in case it's a new indexPattern created on the spot dataViews.indexPatternRefs[indexPatternId] @@ -265,9 +272,25 @@ export const LensTopNavMenu = ({ cache: dataViews.indexPatterns, }), ]); + + // enhance the references with the adHoc dataviews + if (adHocDataViews?.length) { + adHocDataViews.forEach((adHoc) => { + if (adHoc.id) { + newIndexPatternRefs.push({ + title: adHoc.title, + name: adHoc.name, + id: adHoc.id, + }); + } + }); + } dispatch( changeIndexPattern({ - dataViews: { indexPatterns: newIndexPatterns, indexPatternRefs: newIndexPatternRefs }, + dataViews: { + indexPatterns: newIndexPatterns, + indexPatternRefs: newIndexPatternRefs, + }, datasourceIds: Object.keys(datasourceStates), visualizationIds: visualization.activeId ? [visualization.activeId] : [], indexPatternId, @@ -281,12 +304,10 @@ export const LensTopNavMenu = ({ dispatch, indexPatternService, visualization.activeId, + adHocDataViews, ] ); - const [indexPatterns, setIndexPatterns] = useState([]); - const [currentIndexPattern, setCurrentIndexPattern] = useState(); - const [rejectedIndexPatterns, setRejectedIndexPatterns] = useState([]); const canEditDataView = Boolean(dataViewEditor?.userPermissions.editDataView()); const closeFieldEditor = useRef<() => void | undefined>(); const closeDataViewEditor = useRef<() => void | undefined>(); @@ -344,6 +365,21 @@ export const LensTopNavMenu = ({ } }, [indexPatterns]); + // add to the dataview picker list the adHoc dataviews + useEffect(() => { + const setAdHoc = async () => { + await asyncForEach(indexPatterns, async (indexPattern) => { + if (indexPattern.id && !adHocDataViews?.some((d) => d.id === indexPattern.id)) { + const dataViewInstance = await data.dataViews.get(indexPattern.id); + if (!dataViewInstance.isPersisted()) { + setAdHocDataViews([...(adHocDataViews ?? []), indexPattern]); + } + } + }); + }; + setAdHoc(); + }, [adHocDataViews, data.dataViews, indexPatterns]); + useEffect(() => { return () => { // Make sure to close the editors when unmounting @@ -703,14 +739,20 @@ export const LensTopNavMenu = ({ closeDataViewEditor.current = dataViewEditor.openEditor({ onSave: async (dataView) => { if (dataView.id) { - dispatchChangeIndexPattern(dataView.id); + dispatchChangeIndexPattern(dataView); + setCurrentIndexPattern(dataView); + if (!dataView.isPersisted()) { + // add the ad-hoc dataview on the indexPatterns list + setIndexPatterns([...indexPatterns, dataView]); + } refreshFieldList(); } }, + allowAdHocDataView: true, }); } : undefined, - [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, refreshFieldList] + [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, indexPatterns, refreshFieldList] ); const dataViewPickerProps = { @@ -720,6 +762,7 @@ export const LensTopNavMenu = ({ title: currentIndexPattern?.title || '', }, currentDataViewId: currentIndexPattern?.id, + adHocDataViews, onAddField: addField, onDataViewCreated: createNewDataView, onChangeDataView: (newIndexPatternId: string) => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts index cfb3e5d96dd9c..51ba441813e93 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts @@ -43,7 +43,8 @@ import { loadIndexPatternRefs, loadIndexPatterns } from '../../indexpattern_serv function getIndexPatterns( references?: SavedObjectReference[], initialContext?: VisualizeFieldContext | VisualizeEditorContext, - initialId?: string + initialId?: string, + adHocDataviews?: string[] ) { const indexPatternIds = []; if (initialContext) { @@ -67,6 +68,9 @@ function getIndexPatterns( } } } + if (adHocDataviews) { + indexPatternIds.push(...adHocDataviews); + } return [...new Set(indexPatternIds)]; } @@ -113,7 +117,27 @@ export async function initializeDataViews( ? fallbackId : undefined; - const usedIndexPatterns = getIndexPatterns(references, initialContext, initialId); + const adHocDataviewsIds: string[] = []; + let adHocDataviews; + Object.keys(datasourceMap).forEach((datasourceId) => { + const datasource = datasourceMap[datasourceId]; + const datasourceState = datasourceStates[datasourceId]?.state; + const adHocSpecs = datasource?.getAdHocIndexSpecs?.(datasourceState); + if (adHocSpecs) { + const dataViewsIds: string[] = Object.keys(adHocSpecs); + if (dataViewsIds.length) { + adHocDataviewsIds.push(...dataViewsIds); + adHocDataviews = Object.values(adHocSpecs); + } + } + }); + + const usedIndexPatterns = getIndexPatterns( + references, + initialContext, + initialId, + adHocDataviewsIds + ); // load them const availableIndexPatterns = new Set(indexPatternRefs.map(({ id }: IndexPatternRef) => id)); @@ -125,6 +149,7 @@ export async function initializeDataViews( patterns: usedIndexPatterns, notUsedPatterns, cache: {}, + adHocDataviews, }); return { indexPatternRefs, indexPatterns }; diff --git a/x-pack/plugins/lens/public/embeddable/embeddable.tsx b/x-pack/plugins/lens/public/embeddable/embeddable.tsx index 6aa2c19de6705..14476371c6f37 100644 --- a/x-pack/plugins/lens/public/embeddable/embeddable.tsx +++ b/x-pack/plugins/lens/public/embeddable/embeddable.tsx @@ -76,7 +76,12 @@ import { LensAttributeService } from '../lens_attribute_service'; import type { ErrorMessage, TableInspectorAdapter } from '../editor_frame_service/types'; import { getLensInspectorService, LensInspector } from '../lens_inspector_service'; import { SharingSavedObjectProps, VisualizationDisplayOptions } from '../types'; -import { getActiveDatasourceIdFromDoc, getIndexPatternsObjects, inferTimeField } from '../utils'; +import { + getActiveDatasourceIdFromDoc, + getIndexPatternsObjects, + inferTimeField, + getAdHocIndexSpecs, +} from '../utils'; import { getLayerMetaInfo, combineQueryAndFilters } from '../app_plugin/show_underlying_data'; import { convertDataViewIntoLensIndexPattern } from '../indexpattern_service/loader'; @@ -770,8 +775,23 @@ export class Embeddable this.activeDataInfo.activeDatasource = this.deps.datasourceMap[activeDatasourceId]; const docDatasourceState = this.savedVis?.state.datasourceStates[activeDatasourceId]; + const adHocIndexPatterns = + this.activeDataInfo.activeDatasource?.getAdHocIndexSpecs?.(docDatasourceState); + + const adHocDataviews: DataView[] = []; + + if (adHocIndexPatterns) { + const adHocSpecs = Object.values(adHocIndexPatterns); + if (adHocSpecs?.length) { + for (const addHocDataView of adHocSpecs) { + const d = await this.deps.dataViews.create(addHocDataView); + adHocDataviews.push(d); + } + } + } + const allIndexPatterns = [...this.indexPatterns, ...adHocDataviews]; - const indexPatternsCache = this.indexPatterns.reduce( + const indexPatternsCache = allIndexPatterns.reduce( (acc, indexPattern) => ({ [indexPattern.id!]: convertDataViewIntoLensIndexPattern(indexPattern), ...acc, @@ -831,6 +851,19 @@ export class Embeddable this.savedVis?.references.map(({ id }) => id) || [], this.deps.dataViews ); + const activeDatasourceId = getActiveDatasourceIdFromDoc(this.savedVis); + if (activeDatasourceId) { + const activeDatasource = this.deps.datasourceMap[activeDatasourceId]; + const dataSourceState = this.savedVis?.state.datasourceStates[activeDatasourceId]; + const { adHocDataviews } = await getAdHocIndexSpecs( + activeDatasource, + dataSourceState, + this.deps.dataViews + ); + if (adHocDataviews.length) { + indexPatterns.push(...adHocDataviews); + } + } this.indexPatterns = uniqBy(indexPatterns, 'id'); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/change_indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/change_indexpattern.tsx deleted file mode 100644 index 75ebbfdeb27a4..0000000000000 --- a/x-pack/plugins/lens/public/indexpattern_datasource/change_indexpattern.tsx +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; -import React, { useState } from 'react'; -import { EuiPopover, EuiPopoverTitle, EuiSelectableProps } from '@elastic/eui'; -import { ToolbarButton, ToolbarButtonProps } from '@kbn/kibana-react-plugin/public'; -import { DataViewsList } from '@kbn/unified-search-plugin/public'; -import type { IndexPatternRef } from '../types'; - -export type ChangeIndexPatternTriggerProps = ToolbarButtonProps & { - label: string; - title?: string; -}; - -export function ChangeIndexPattern({ - indexPatternRefs, - isMissingCurrent, - indexPatternId, - onChangeIndexPattern, - trigger, - selectableProps, -}: { - trigger: ChangeIndexPatternTriggerProps; - indexPatternRefs: IndexPatternRef[]; - isMissingCurrent?: boolean; - onChangeIndexPattern: (newId: string) => void; - indexPatternId?: string; - selectableProps?: EuiSelectableProps; -}) { - const [isPopoverOpen, setPopoverIsOpen] = useState(false); - - // be careful to only add color with a value, otherwise it will fallbacks to "primary" - const colorProp = isMissingCurrent - ? { - color: 'danger' as const, - } - : {}; - - const createTrigger = function () { - const { label, title, ...rest } = trigger; - return ( - setPopoverIsOpen(!isPopoverOpen)} - fullWidth - {...colorProp} - {...rest} - > - {label} - - ); - }; - - return ( - <> - setPopoverIsOpen(false)} - display="block" - panelPaddingSize="none" - ownFocus - > -
- - {i18n.translate('xpack.lens.indexPattern.changeDataViewTitle', { - defaultMessage: 'Data view', - })} - - - { - onChangeIndexPattern(newId); - setPopoverIsOpen(false); - }} - currentDataViewId={indexPatternId} - selectableProps={selectableProps} - /> -
- - - ); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index 02ca96e147605..33cbe250f0f12 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -165,6 +165,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { fromDate: dateRange.fromDate, toDate: dateRange.toDate, fieldName: field.name, + spec: indexPattern.spec, }), }) .then((results) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index e91aa1b286bec..0f031351dc963 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -282,6 +282,7 @@ describe('IndexPattern Data Source', () => { }, }, }, + adHocIndexPatterns: {}, }, savedObjectReferences: [ { name: 'indexpattern-datasource-layer-first', type: 'index-pattern', id: '1' }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index a2ab9fd238215..2855b1976c537 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -202,6 +202,10 @@ export function getIndexPatternDatasource({ return Object.keys(state.layers); }, + getAdHocIndexSpecs(state: IndexPatternPersistedState) { + return state?.adHocIndexPatterns; + }, + removeColumn({ prevState, layerId, columnId, indexPatterns }) { const indexPattern = indexPatterns[prevState.layers[layerId]?.indexPatternId]; return mergeLayer({ @@ -540,15 +544,18 @@ export function getIndexPatternDatasource({ } return null; }, - getSourceId: () => layer.indexPatternId, - getFilters: (activeData: FramePublicAPI['activeData'], timeRange?: TimeRange) => - getFiltersInLayer( + getSourceId: () => { + return layer.adHocSpec?.id || layer.indexPatternId; + }, + getFilters: (activeData: FramePublicAPI['activeData'], timeRange?: TimeRange) => { + return getFiltersInLayer( layer, visibleColumnIds, activeData?.[layerId], indexPatterns[layer.indexPatternId], timeRange - ), + ); + }, getVisualDefaults: () => getVisualDefaultsForLayer(layer), getMaxPossibleNumValues: (columnId) => { if (layer && layer.columns[columnId]) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index f7a6912390027..ec3e47cc8993a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -49,16 +49,25 @@ function getLayerReferenceName(layerId: string) { export function extractReferences({ layers }: IndexPatternPrivateState) { const savedObjectReferences: SavedObjectReference[] = []; - const persistableLayers: Record> = {}; + const persistableState: IndexPatternPersistedState = { + layers: {}, + adHocIndexPatterns: {}, + }; Object.entries(layers).forEach(([layerId, { indexPatternId, ...persistableLayer }]) => { - savedObjectReferences.push({ - type: 'index-pattern', - id: indexPatternId, - name: getLayerReferenceName(layerId), - }); - persistableLayers[layerId] = persistableLayer; + persistableState.layers[layerId] = persistableLayer; + if (persistableLayer.adHocSpec) { + if (!persistableState.adHocIndexPatterns![indexPatternId]) { + persistableState.adHocIndexPatterns![indexPatternId] = persistableLayer.adHocSpec!; + } + } else { + savedObjectReferences.push({ + type: 'index-pattern', + id: indexPatternId, + name: getLayerReferenceName(layerId), + }); + } }); - return { savedObjectReferences, state: { layers: persistableLayers } }; + return { savedObjectReferences, state: persistableState }; } export function injectReferences( @@ -69,11 +78,14 @@ export function injectReferences( Object.entries(state.layers).forEach(([layerId, persistedLayer]) => { layers[layerId] = { ...persistedLayer, - indexPatternId: references.find(({ name }) => name === getLayerReferenceName(layerId))!.id, + indexPatternId: + persistedLayer.adHocSpec?.id || + references.find(({ name }) => name === getLayerReferenceName(layerId))!.id, }; }); return { layers, + adHocIndexPatterns: state.adHocIndexPatterns, }; } @@ -117,7 +129,11 @@ function getUsedIndexPatterns({ const usedPatterns = ( initialContext ? indexPatternIds - : uniq(state ? Object.values(state.layers).map((l) => l.indexPatternId) : [fallbackId]) + : uniq( + state + ? Object.values(state.layers).map((l) => l.adHocSpec?.id ?? l.indexPatternId) + : [fallbackId] + ) ) // take out the undefined from the list .filter(Boolean); @@ -154,6 +170,11 @@ export function loadInitialState({ indexPatternRefs, }); + if (persistedState?.adHocIndexPatterns) { + Object.entries(persistedState?.adHocIndexPatterns).forEach(([id, { name, title }]) => { + indexPatternRefs.push({ id, name, title: title || '' }); + }); + } const availableIndexPatterns = new Set(indexPatternRefs.map(({ id }: IndexPatternRef) => id)); const notUsedPatterns: string[] = difference([...availableIndexPatterns], usedPatterns); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts index d76e6723c1be9..2a1f93f5b31ca 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts @@ -101,6 +101,7 @@ export const createMockedIndexPattern = (): IndexPattern => { hasRestrictions: false, fields, getFieldByName: getFieldByNameFactory(fields), + spec: undefined, }; }; @@ -140,6 +141,7 @@ export const createMockedRestrictedIndexPattern = () => { fieldFormatMap: { bytes: { id: 'bytes', params: { pattern: '0.0' } } }, fields, getFieldByName: getFieldByNameFactory(fields), + spec: undefined, typeMeta: { params: { rollup_index: 'my-fake-index-pattern', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts index a290c2d239a1e..629a9aba257d2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts @@ -147,6 +147,7 @@ export function getDisallowedTermsMessage( fromDate: frame.dateRange.fromDate, toDate: frame.dateRange.toDate, size: currentColumn.params.size, + spec: indexPattern.spec, }), } ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 3d079584e32f9..fc6d521fa8a45 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -200,6 +200,12 @@ export function insertNewColumn({ if (layer.columns[columnId]) { throw new Error(`Can't insert a column with an ID that is already in use`); } + if (indexPattern.spec) { + layer = { + ...layer, + adHocSpec: indexPattern.spec, + }; + } const baseOptions = { indexPattern, @@ -1356,6 +1362,7 @@ export function updateLayerIndexPattern( return { ...layer, indexPatternId: newIndexPattern.id, + adHocSpec: newIndexPattern?.spec, columns: newColumns, columnOrder: newColumnOrder, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 46a9ffaafb566..68facca974f5b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import type { DataViewSpec } from '@kbn/data-views-plugin/common'; import type { DragDropIdentifier } from '../drag_drop/providers'; import type { IncompleteColumn, GenericIndexPatternColumn } from './operations'; import type { DragDropOperation } from '../types'; @@ -55,10 +55,12 @@ export interface IndexPatternLayer { indexPatternId: string; // Partial columns represent the temporary invalid states incompleteColumns?: Record; + adHocSpec?: DataViewSpec; } export interface IndexPatternPersistedState { layers: Record>; + adHocIndexPatterns?: Record; } export type PersistedIndexPatternLayer = Omit; diff --git a/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts b/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts index 912028fba81a2..dc9edff3c31b7 100644 --- a/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts @@ -64,6 +64,7 @@ describe('loader', () => { id: 'foo', title: 'Foo index', metaFields: [], + isPersisted: () => true, typeMeta: { aggs: { date_histogram: { @@ -100,7 +101,8 @@ describe('loader', () => { id: 'foo', title: 'Foo index', })), - } as unknown as Pick, + create: jest.fn(), + } as unknown as Pick, }); expect(cache.foo.getFieldByName('bytes')!.aggregationRestrictions).toEqual({ @@ -120,6 +122,7 @@ describe('loader', () => { id: 'foo', title: 'Foo index', metaFields: ['timestamp'], + isPersisted: () => true, typeMeta: { aggs: { date_histogram: { @@ -156,7 +159,8 @@ describe('loader', () => { id: 'foo', title: 'Foo index', })), - } as unknown as Pick, + create: jest.fn(), + } as unknown as Pick, }); expect(cache.foo.getFieldByName('timestamp')!.meta).toEqual(true); @@ -198,12 +202,13 @@ describe('loader', () => { timeFieldName: 'timestamp', hasRestrictions: false, fields: [], + isPersisted: () => true, }; } return Promise.reject(); }), getIdsWithTitle: jest.fn(), - } as unknown as Pick; + } as unknown as Pick; const cache = await loadIndexPatterns({ cache: {}, patterns: ['1', '2'], @@ -234,7 +239,7 @@ describe('loader', () => { throw err; }), getIdsWithTitle: jest.fn(), - } as unknown as Pick, + } as unknown as Pick, onError, }); diff --git a/x-pack/plugins/lens/public/indexpattern_service/loader.ts b/x-pack/plugins/lens/public/indexpattern_service/loader.ts index 45dd7eb9db8f3..52e025dcf616f 100644 --- a/x-pack/plugins/lens/public/indexpattern_service/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_service/loader.ts @@ -6,7 +6,7 @@ */ import { isNestedField } from '@kbn/data-views-plugin/common'; -import type { DataViewsContract, DataView } from '@kbn/data-views-plugin/public'; +import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public'; import { keyBy } from 'lodash'; import { HttpSetup } from '@kbn/core/public'; import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types'; @@ -15,7 +15,7 @@ import { BASE_API_URL, DateRange, ExistingFields } from '../../common'; import { DataViewsState } from '../state_management'; type ErrorHandler = (err: Error) => void; -type MinimalDataViewsContract = Pick; +type MinimalDataViewsContract = Pick; /** * All these functions will be used by the Embeddable instance too, @@ -92,6 +92,7 @@ export function convertDataViewIntoLensIndexPattern( fields: newFields, getFieldByName: getFieldByNameFactory(newFields), hasRestrictions: !!typeMeta?.aggs, + spec: dataView.isPersisted() ? undefined : dataView.toSpec(false), }; } @@ -122,12 +123,14 @@ export async function loadIndexPatterns({ patterns, notUsedPatterns, cache, + adHocDataviews, onIndexPatternRefresh, }: { dataViews: MinimalDataViewsContract; patterns: string[]; notUsedPatterns?: string[]; cache: Record; + adHocDataviews?: DataViewSpec[]; onIndexPatternRefresh?: () => void; }) { const missingIds = patterns.filter((id) => !cache[id]); @@ -157,6 +160,12 @@ export async function loadIndexPatterns({ } } } + if (adHocDataviews?.length) { + for (const addHocDataView of adHocDataviews) { + const d = await dataViews.create(addHocDataView); + indexPatterns.push(d); + } + } const indexPatternsObject = indexPatterns.reduce( (acc, indexPattern) => ({ @@ -228,6 +237,10 @@ async function refreshExistingFields({ body.timeFieldName = pattern.timeFieldName; } + if (pattern.spec) { + body.spec = pattern.spec; + } + return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, { body: JSON.stringify(body), }) as Promise; diff --git a/x-pack/plugins/lens/public/indexpattern_service/mocks.ts b/x-pack/plugins/lens/public/indexpattern_service/mocks.ts index 6eb7156a91cff..048333786fdd6 100644 --- a/x-pack/plugins/lens/public/indexpattern_service/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_service/mocks.ts @@ -42,6 +42,7 @@ const indexPattern1 = { title: 'my-fake-index-pattern', timeFieldName: 'timestamp', hasRestrictions: false, + isPersisted: () => true, fields: [ { name: 'timestamp', @@ -127,6 +128,7 @@ const indexPattern2 = { title: 'my-fake-restricted-pattern', timeFieldName: 'timestamp', hasRestrictions: true, + isPersisted: () => true, fieldFormatMap: { bytes: { id: 'bytes', params: { pattern: '0.0' } } }, fields: [ { @@ -198,7 +200,11 @@ export const sampleIndexPatterns = { export function mockDataViewsService() { return { get: jest.fn(async (id: '1' | '2') => { - const result = { ...sampleIndexPatternsFromService[id], metaFields: [] }; + const result = { + ...sampleIndexPatternsFromService[id], + metaFields: [], + isPersisted: () => true, + }; if (!result.fields) { result.fields = []; } @@ -216,5 +222,6 @@ export function mockDataViewsService() { }, ]; }), - } as unknown as Pick; + create: jest.fn(), + } as unknown as Pick; } diff --git a/x-pack/plugins/lens/public/mocks/data_plugin_mock.ts b/x-pack/plugins/lens/public/mocks/data_plugin_mock.ts index 157c910440fb6..8260c71b04981 100644 --- a/x-pack/plugins/lens/public/mocks/data_plugin_mock.ts +++ b/x-pack/plugins/lens/public/mocks/data_plugin_mock.ts @@ -102,7 +102,7 @@ export function mockDataPlugin( extract: (filtersIn: Filter[]) => { const state = filtersIn.map((filter) => ({ ...filter, - meta: { ...filter.meta, index: 'extracted!' }, + meta: { ...filter.meta }, })); return { state, references: [] }; }, @@ -127,6 +127,13 @@ export function mockDataPlugin( indexPatterns: { get: jest.fn().mockImplementation((id) => Promise.resolve({ id, isTimeBased: () => true })), }, + dataViews: { + get: jest + .fn() + .mockImplementation((id) => + Promise.resolve({ id, isTimeBased: () => true, isPersisted: () => true }) + ), + }, search: createMockSearchService(), nowProvider: { get: jest.fn(), diff --git a/x-pack/plugins/lens/public/state_management/selectors.ts b/x-pack/plugins/lens/public/state_management/selectors.ts index f1f53197978fa..3354d4f8e7f61 100644 --- a/x-pack/plugins/lens/public/state_management/selectors.ts +++ b/x-pack/plugins/lens/public/state_management/selectors.ts @@ -113,8 +113,16 @@ export const selectSavedObjectFormat = createSelector( references.push(...savedObjectReferences); }); + const adHocFilters = filters + .filter((f) => !references.some((r) => r.type === 'index-pattern' && r.id === f.meta.index)) + .map((f) => ({ ...f, meta: { ...f.meta, value: undefined } })); + + const referencedFilters = filters.filter((f) => + references.some((r) => r.type === 'index-pattern' && r.id === f.meta.index) + ); + const { state: persistableFilters, references: filterReferences } = - extractFilterReferences(filters); + extractFilterReferences(referencedFilters); references.push(...filterReferences); @@ -128,7 +136,7 @@ export const selectSavedObjectFormat = createSelector( state: { visualization: visualization.state, query, - filters: persistableFilters, + filters: [...persistableFilters, ...adHocFilters], datasourceStates: persistibleDatasourceStates, }, }; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 0a76ceec5d315..f7009af897739 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -26,7 +26,7 @@ import type { } from '@kbn/ui-actions-plugin/public'; import type { ClickTriggerEvent, BrushTriggerEvent } from '@kbn/charts-plugin/public'; import type { IndexPatternAggRestrictions } from '@kbn/data-plugin/public'; -import type { FieldSpec } from '@kbn/data-views-plugin/common'; +import type { FieldSpec, DataViewSpec } from '@kbn/data-views-plugin/common'; import type { FieldFormatParams } from '@kbn/field-formats-plugin/common'; import type { DraggingIdentifier, DragDropIdentifier, DragContextState } from './drag_drop'; import type { DateRange, LayerType, SortingHint } from '../common'; @@ -68,6 +68,7 @@ export interface IndexPattern { } >; hasRestrictions: boolean; + spec?: DataViewSpec; } export type IndexPatternField = FieldSpec & { @@ -273,6 +274,7 @@ export interface Datasource { removeLayer: (state: T, layerId: string) => T; clearLayer: (state: T, layerId: string) => T; getLayers: (state: T) => string[]; + getAdHocIndexSpecs?: (state: P) => Record | undefined; removeColumn: (props: { prevState: T; layerId: string; @@ -900,7 +902,7 @@ export interface Visualization { /** Optional, if the visualization supports multiple layers */ removeLayer?: (state: T, layerId: string) => T; /** Track added layers in internal state */ - appendLayer?: (state: T, layerId: string, type: LayerType) => T; + appendLayer?: (state: T, layerId: string, type: LayerType, indexPatternId?: string) => T; /** Retrieve a list of supported layer types with initialization data */ getSupportedLayers: ( diff --git a/x-pack/plugins/lens/public/utils.ts b/x-pack/plugins/lens/public/utils.ts index e5995762b5505..5bae9bce1c663 100644 --- a/x-pack/plugins/lens/public/utils.ts +++ b/x-pack/plugins/lens/public/utils.ts @@ -153,6 +153,27 @@ export async function getIndexPatternsObjects( return { indexPatterns: fullfilled.map((response) => response.value), rejectedIds }; } +export async function getAdHocIndexSpecs( + dataSource: Datasource, + dataSourceState: unknown, + dataViews: DataViewsContract +): Promise<{ adHocDataviews: DataView[] }> { + const adHocIndexPatterns = dataSource?.getAdHocIndexSpecs?.(dataSourceState); + + const adHocDataviews: DataView[] = []; + + if (adHocIndexPatterns) { + const adHocSpecs = Object.values(adHocIndexPatterns); + if (adHocSpecs?.length) { + for (const addHocDataView of adHocSpecs) { + const d = await dataViews.create(addHocDataView); + adHocDataviews.push(d); + } + } + } + return { adHocDataviews }; +} + export function getRemoveOperation( activeVisualization: Visualization, visualizationState: VisualizationState['state'], diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index 8de7aa009b54a..a5ae772241999 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -12,7 +12,7 @@ import { schema } from '@kbn/config-schema'; import { RequestHandlerContext, ElasticsearchClient } from '@kbn/core/server'; import { CoreSetup, Logger } from '@kbn/core/server'; import { RuntimeField } from '@kbn/data-views-plugin/common'; -import { DataViewsService, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; +import { DataViewsService, DataView, FieldSpec, DataViewSpec } from '@kbn/data-views-plugin/common'; import { UI_SETTINGS } from '@kbn/data-plugin/server'; import { BASE_API_URL } from '../../common'; import { FIELD_EXISTENCE_SETTING } from '../ui_settings'; @@ -51,6 +51,7 @@ export async function existingFieldsRoute(setup: CoreSetup, fromDate: schema.maybe(schema.string()), toDate: schema.maybe(schema.string()), timeFieldName: schema.maybe(schema.string()), + spec: schema.object({}, { unknowns: 'allow' }), }), }, }, @@ -110,6 +111,7 @@ async function fetchFieldExistence({ fromDate, toDate, timeFieldName, + spec, includeFrozen, useSampling, }: { @@ -120,6 +122,7 @@ async function fetchFieldExistence({ fromDate?: string; toDate?: string; timeFieldName?: string; + spec?: DataViewSpec; includeFrozen: boolean; useSampling: boolean; }) { @@ -132,13 +135,17 @@ async function fetchFieldExistence({ fromDate, toDate, timeFieldName, + spec, includeFrozen, }); } const uiSettingsClient = (await context.core).uiSettings.client; const metaFields: string[] = await uiSettingsClient.get(UI_SETTINGS.META_FIELDS); - const dataView = await dataViewsService.get(indexPatternId); + const dataView = + spec && Object.keys(spec).length !== 0 + ? await dataViewsService.create(spec) + : await dataViewsService.get(indexPatternId); const allFields = buildFieldList(dataView, metaFields); const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, { // filled in by data views service @@ -159,6 +166,7 @@ async function legacyFetchFieldExistenceSampling({ fromDate, toDate, timeFieldName, + spec, includeFrozen, }: { indexPatternId: string; @@ -168,11 +176,14 @@ async function legacyFetchFieldExistenceSampling({ fromDate?: string; toDate?: string; timeFieldName?: string; + spec?: DataViewSpec; includeFrozen: boolean; }) { const coreContext = await context.core; const metaFields: string[] = await coreContext.uiSettings.client.get(UI_SETTINGS.META_FIELDS); - const indexPattern = await dataViewsService.get(indexPatternId); + const indexPattern = spec + ? await dataViewsService.create(spec) + : await dataViewsService.get(indexPatternId); const fields = buildFieldList(indexPattern, metaFields); const runtimeMappings = indexPattern.getRuntimeMappings(); diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 35a15ea44be67..a844801473878 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -33,6 +33,7 @@ export async function initFieldsRoute(setup: CoreSetup) { toDate: schema.string(), fieldName: schema.string(), size: schema.maybe(schema.number()), + spec: schema.object({}, { unknowns: 'allow' }), }, { unknowns: 'allow' } ), @@ -40,7 +41,7 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = (await context.core).elasticsearch.client.asCurrentUser; - const { fromDate, toDate, fieldName, dslQuery, size } = req.body; + const { fromDate, toDate, fieldName, dslQuery, size, spec } = req.body; const [{ savedObjects, elasticsearch }, { dataViews }] = await setup.getStartServices(); const savedObjectsClient = savedObjects.getScopedClient(req); @@ -51,7 +52,10 @@ export async function initFieldsRoute(setup: CoreSetup) { ); try { - const indexPattern = await indexPatternsService.get(req.params.indexPatternId); + const indexPattern = + spec && Object.keys(spec).length !== 0 + ? await indexPatternsService.create(spec) + : await indexPatternsService.get(req.params.indexPatternId); const timeFieldName = indexPattern.timeFieldName; const field = indexPattern.fields.find((f) => f.name === fieldName);