diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index f786d01232227..79308c43bf0aa 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -100,7 +100,7 @@ pageLoadAssetSize: bfetch: 22837 kibanaUtils: 79713 data: 491273 - dataViews: 41532 + dataViews: 42532 expressions: 140958 fieldFormats: 65209 kibanaReact: 74422 diff --git a/src/plugins/data_view_field_editor/public/open_editor.tsx b/src/plugins/data_view_field_editor/public/open_editor.tsx index c66e8183b9ab6..d3a91739dc94a 100644 --- a/src/plugins/data_view_field_editor/public/open_editor.tsx +++ b/src/plugins/data_view_field_editor/public/open_editor.tsx @@ -19,6 +19,7 @@ import { UsageCollectionStart, DataViewsPublicPluginStart, FieldFormatsStart, + RuntimeType, } from './shared_imports'; import type { PluginStart, InternalFieldType, CloseEditor } from './types'; @@ -104,7 +105,12 @@ export const getFieldEditorOpener = } const isNewRuntimeField = !fieldName; - const isExistingRuntimeField = field && field.runtimeField && !field.isMapped; + const isExistingRuntimeField = + field && + field.runtimeField && + !field.isMapped && + // treat composite field instances as mapped fields for field editing purposes + field.runtimeField.type !== ('composite' as RuntimeType); const fieldTypeToProcess: InternalFieldType = isNewRuntimeField || isExistingRuntimeField ? 'runtime' : 'concrete'; diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap index abed44135e6ea..843c90d4f617b 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/__snapshots__/indexed_fields_table.test.tsx.snap @@ -143,6 +143,9 @@ exports[`IndexedFieldsTable IndexedFieldsTable with rollup index pattern should }, Object { "displayName": "runtime", + "esTypes": Array [ + "long", + ], "excluded": false, "format": "", "hasRuntime": true, @@ -219,6 +222,9 @@ exports[`IndexedFieldsTable should filter based on the schema filter 1`] = ` Array [ Object { "displayName": "runtime", + "esTypes": Array [ + "long", + ], "excluded": false, "format": "", "hasRuntime": true, @@ -356,6 +362,9 @@ exports[`IndexedFieldsTable should render normally 1`] = ` }, Object { "displayName": "runtime", + "esTypes": Array [ + "long", + ], "excluded": false, "format": "", "hasRuntime": true, diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx index b2197a6dcb203..44385c7b6e41d 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.test.tsx @@ -10,7 +10,7 @@ import React from 'react'; import { shallow } from 'enzyme'; import { DataView } from 'src/plugins/data_views/public'; import { IndexedFieldItem } from '../../types'; -import { Table, renderFieldName, getConflictModalContent } from './table'; +import { Table, renderFieldName, getConflictModalContent, showDelete } from './table'; import { overlayServiceMock, themeServiceMock } from 'src/core/public/mocks'; const theme = themeServiceMock.createStartContract(); @@ -198,4 +198,42 @@ describe('Table', () => { }) ).toMatchSnapshot(); }); + + test('showDelete', () => { + const runtimeFields = [ + { + name: 'customer', + info: [], + excluded: false, + kbnType: 'string', + type: 'keyword', + isMapped: false, + isUserEditable: true, + hasRuntime: true, + runtimeField: { + type: 'keyword', + }, + }, + { + name: 'thing', + info: [], + excluded: false, + kbnType: 'string', + type: 'keyword', + isMapped: false, + isUserEditable: true, + hasRuntime: true, + runtimeField: { + type: 'composite', + }, + }, + ] as IndexedFieldItem[]; + + // indexed field + expect(showDelete(items[0])).toBe(false); + // runtime field - primitive type + expect(showDelete(runtimeFields[0])).toBe(true); + // runtime field - composite type + expect(showDelete(runtimeFields[1])).toBe(false); + }); }); diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.tsx index f860d9fafb1c0..f8ce07b11d3e8 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/components/table/table.tsx @@ -33,6 +33,9 @@ import { toMountPoint } from '../../../../../../../kibana_react/public'; import { DataView } from '../../../../../../../data_views/public'; import { IndexedFieldItem } from '../../types'; +export const showDelete = (field: IndexedFieldItem) => + !field.isMapped && field.isUserEditable && field.runtimeField?.type !== 'composite'; + // localized labels const additionalInfoAriaLabel = i18n.translate( 'indexPatternManagement.editIndexPattern.fields.table.additionalInfoAriaLabel', @@ -454,7 +457,7 @@ export class Table extends PureComponent { onClick: (field) => deleteField(field.name), type: 'icon', 'data-test-subj': 'deleteField', - available: (field) => !field.isMapped && field.isUserEditable, + available: showDelete, }, ], width: '40px', diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx index c7b92c227a5d9..ff03bd404b6c4 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/indexed_fields_table/indexed_fields_table.test.tsx @@ -94,6 +94,8 @@ const fields = [ name: 'runtime', displayName: 'runtime', runtimeField, + esTypes: ['long'], + type: 'number', }, ].map(mockFieldToIndexPatternField); diff --git a/src/plugins/data_views/common/constants.ts b/src/plugins/data_views/common/constants.ts index d656a044e1080..b5a73eee8b3d3 100644 --- a/src/plugins/data_views/common/constants.ts +++ b/src/plugins/data_views/common/constants.ts @@ -14,6 +14,7 @@ export const RUNTIME_FIELD_TYPES = [ 'ip', 'boolean', 'geo_point', + 'composite', ] as const; /** diff --git a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap index 731b5ba6e260c..0a9109c282922 100644 --- a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap +++ b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap @@ -1,5 +1,31 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`IndexPattern addRuntimeField and removeRuntimeField add and remove composite runtime field as new fields 1`] = ` +Object { + "fields": Object { + "a": Object { + "type": "keyword", + }, + "b": Object { + "type": "long", + }, + }, + "script": Object { + "source": "emit('hello world');", + }, + "type": "composite", +} +`; + +exports[`IndexPattern addRuntimeField and removeRuntimeField add and remove runtime field as new field 1`] = ` +Object { + "script": Object { + "source": "emit('hello world');", + }, + "type": "keyword", +} +`; + exports[`IndexPattern toSpec should match snapshot 1`] = ` Object { "allowNoIndex": false, diff --git a/src/plugins/data_views/common/data_views/data_view.test.ts b/src/plugins/data_views/common/data_views/data_view.test.ts index 40a3a06c869ab..514aba80bdb1e 100644 --- a/src/plugins/data_views/common/data_views/data_view.test.ts +++ b/src/plugins/data_views/common/data_views/data_view.test.ts @@ -16,7 +16,7 @@ import { IndexPatternField } from '../fields'; import { fieldFormatsMock } from '../../../field_formats/common/mocks'; import { FieldFormat } from '../../../field_formats/common'; -import { RuntimeField } from '../types'; +import { RuntimeField, RuntimeTypeExceptComposite } from '../types'; import { stubLogstashFields } from '../field.stub'; import { stubbedSavedObjectIndexPattern } from '../data_view.stub'; @@ -37,6 +37,8 @@ const runtimeField = { name: 'runtime_field', runtimeField: runtimeFieldScript, scripted: false, + esTypes: ['keyword'], + type: 'string', }; fieldFormatsMock.getInstance = jest.fn().mockImplementation(() => new MockFieldFormatter()) as any; @@ -223,20 +225,80 @@ describe('IndexPattern', () => { }, }; + const runtimeWithAttrs = { + ...runtime, + popularity: 5, + customLabel: 'custom name', + format: { + id: 'bytes', + }, + }; + + const runtimeComposite = { + type: 'composite' as RuntimeField['type'], + script: { + source: "emit('hello world');", + }, + fields: { + a: { + type: 'keyword' as RuntimeTypeExceptComposite, + }, + b: { + type: 'long' as RuntimeTypeExceptComposite, + }, + }, + }; + + const runtimeCompositeWithAttrs = { + type: runtimeComposite.type, + script: runtimeComposite.script, + fields: { + a: { + ...runtimeComposite.fields.a, + popularity: 3, + customLabel: 'custom name a', + format: { + id: 'bytes', + }, + }, + b: { + ...runtimeComposite.fields.b, + popularity: 4, + customLabel: 'custom name b', + format: { + id: 'bytes', + }, + }, + }, + }; + beforeEach(() => { const formatter = { toJSON: () => ({ id: 'bytes' }), } as FieldFormat; indexPattern.getFormatterForField = () => formatter; + indexPattern.getFormatterForFieldNoDefault = () => undefined; }); test('add and remove runtime field to existing field', () => { - indexPattern.addRuntimeField('@tags', runtime); + indexPattern.addRuntimeField('@tags', runtimeWithAttrs); expect(indexPattern.toSpec().runtimeFieldMap).toEqual({ '@tags': runtime, runtime_field: runtimeField.runtimeField, }); - expect(indexPattern.toSpec()!.fields!['@tags'].runtimeField).toEqual(runtime); + const field = indexPattern.toSpec()!.fields!['@tags']; + expect(field.runtimeField).toEqual(runtime); + expect(field.count).toEqual(5); + expect(field.format).toEqual({ + id: 'bytes', + }); + expect(field.customLabel).toEqual('custom name'); + expect(indexPattern.toSpec().fieldAttrs).toEqual({ + '@tags': { + customLabel: 'custom name', + count: 5, + }, + }); indexPattern.removeRuntimeField('@tags'); expect(indexPattern.toSpec().runtimeFieldMap).toEqual({ @@ -246,11 +308,12 @@ describe('IndexPattern', () => { }); test('add and remove runtime field as new field', () => { - indexPattern.addRuntimeField('new_field', runtime); + indexPattern.addRuntimeField('new_field', runtimeWithAttrs); expect(indexPattern.toSpec().runtimeFieldMap).toEqual({ runtime_field: runtimeField.runtimeField, new_field: runtime, }); + expect(indexPattern.getRuntimeField('new_field')).toMatchSnapshot(); expect(indexPattern.toSpec()!.fields!.new_field.runtimeField).toEqual(runtime); indexPattern.removeRuntimeField('new_field'); @@ -260,6 +323,35 @@ describe('IndexPattern', () => { expect(indexPattern.toSpec()!.fields!.new_field).toBeUndefined(); }); + test('add and remove composite runtime field as new fields', () => { + const fieldCount = indexPattern.fields.length; + indexPattern.addRuntimeField('new_field', runtimeCompositeWithAttrs); + expect(indexPattern.toSpec().runtimeFieldMap).toEqual({ + runtime_field: runtimeField.runtimeField, + new_field: runtimeComposite, + }); + expect(indexPattern.fields.length - fieldCount).toEqual(2); + expect(indexPattern.getRuntimeField('new_field')).toMatchSnapshot(); + expect(indexPattern.toSpec()!.fields!['new_field.a']).toBeDefined(); + expect(indexPattern.toSpec()!.fields!['new_field.b']).toBeDefined(); + expect(indexPattern.toSpec()!.fieldAttrs).toEqual({ + 'new_field.a': { + count: 3, + customLabel: 'custom name a', + }, + 'new_field.b': { + count: 4, + customLabel: 'custom name b', + }, + }); + + indexPattern.removeRuntimeField('new_field'); + expect(indexPattern.toSpec().runtimeFieldMap).toEqual({ + runtime_field: runtimeField.runtimeField, + }); + expect(indexPattern.toSpec()!.fields!.new_field).toBeUndefined(); + }); + test('should not allow runtime field with * in name', async () => { try { await indexPattern.addRuntimeField('test*123', runtime); diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index 67a127407f94a..5f3ee24109027 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -12,7 +12,7 @@ import _, { each, reject } from 'lodash'; import { castEsToKbnFieldTypeName, ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { FieldAttrs, FieldAttrSet, DataViewAttributes } from '..'; -import type { RuntimeField } from '../types'; +import type { RuntimeField, RuntimeFieldSpec, RuntimeType, FieldConfiguration } from '../types'; import { CharacterNotAllowedInField } from '../../../kibana_utils/common'; import { IIndexPattern, IFieldType } from '../../common'; @@ -24,6 +24,7 @@ import { SerializedFieldFormat, } from '../../../field_formats/common'; import { DataViewSpec, TypeMeta, SourceFilter, DataViewFieldMap } from '../types'; +import { removeFieldAttrs } from './utils'; interface DataViewDeps { spec?: DataViewSpec; @@ -79,7 +80,7 @@ export class DataView implements IIndexPattern { private shortDotsEnable: boolean = false; private fieldFormats: FieldFormatsStartCommon; private fieldAttrs: FieldAttrs; - private runtimeFieldMap: Record; + private runtimeFieldMap: Record; /** * prevents errors when index pattern exists before indices @@ -184,11 +185,13 @@ export class DataView implements IIndexPattern { }; }); + const runtimeFields = this.getRuntimeMappings(); + return { storedFields: ['*'], scriptFields, docvalueFields, - runtimeFields: this.runtimeFieldMap, + runtimeFields, }; } @@ -316,31 +319,34 @@ export class DataView implements IIndexPattern { /** * Add a runtime field - Appended to existing mapped field or a new field is - * created as appropriate + * created as appropriate. * @param name Field name * @param runtimeField Runtime field definition */ - addRuntimeField(name: string, runtimeField: RuntimeField) { - const existingField = this.getFieldByName(name); - + addRuntimeField(name: string, runtimeField: RuntimeField): DataViewField[] { if (name.includes('*')) { throw new CharacterNotAllowedInField('*', name); } - if (existingField) { - existingField.runtimeField = runtimeField; - } else { - this.fields.add({ - name, - runtimeField, - type: castEsToKbnFieldTypeName(runtimeField.type), - aggregatable: true, - searchable: true, - count: 0, - readFromDocValues: false, - }); + const { type, script, customLabel, format, popularity } = runtimeField; + + if (type === 'composite') { + return this.addCompositeRuntimeField(name, runtimeField); } - this.runtimeFieldMap[name] = runtimeField; + + this.runtimeFieldMap[name] = removeFieldAttrs(runtimeField); + const field = this.updateOrAddRuntimeField( + name, + type, + { type, script }, + { + customLabel, + format, + popularity, + } + ); + + return [field]; } /** @@ -356,7 +362,60 @@ export class DataView implements IIndexPattern { * @param name */ getRuntimeField(name: string): RuntimeField | null { - return this.runtimeFieldMap[name] ?? null; + if (!this.runtimeFieldMap[name]) { + return null; + } + + const { type, script, fields } = { ...this.runtimeFieldMap[name] }; + const runtimeField: RuntimeField = { + type, + script, + }; + + if (type === 'composite') { + runtimeField.fields = fields; + } + + return runtimeField; + } + + getAllRuntimeFields(): Record { + return Object.keys(this.runtimeFieldMap).reduce>( + (acc, fieldName) => ({ + ...acc, + [fieldName]: this.getRuntimeField(fieldName)!, + }), + {} + ); + } + + getFieldsByRuntimeFieldName(name: string): Record | undefined { + const runtimeField = this.getRuntimeField(name); + if (!runtimeField) { + return; + } + + if (runtimeField.type === 'composite') { + return Object.entries(runtimeField.fields!).reduce>( + (acc, [subFieldName, subField]) => { + const fieldFullName = `${name}.${subFieldName}`; + const dataViewField = this.getFieldByName(fieldFullName); + + if (!dataViewField) { + // We should never enter here as all composite runtime subfield + // are converted to data view fields. + return acc; + } + acc[subFieldName] = dataViewField; + return acc; + }, + {} + ); + } + + const primitveRuntimeField = this.getFieldByName(name); + + return primitveRuntimeField && { [name]: primitveRuntimeField }; } /** @@ -381,17 +440,26 @@ export class DataView implements IIndexPattern { */ removeRuntimeField(name: string) { const existingField = this.getFieldByName(name); - if (existingField) { - if (existingField.isMapped) { - // mapped field, remove runtimeField def - existingField.runtimeField = undefined; - } else { - this.fields.remove(existingField); - } + + if (existingField && existingField.isMapped) { + // mapped field, remove runtimeField def + existingField.runtimeField = undefined; + } else { + Object.values(this.getFieldsByRuntimeFieldName(name) || {}).forEach((field) => { + this.fields.remove(field); + }); } delete this.runtimeFieldMap[name]; } + /** + * Return the "runtime_mappings" section of the ES search query + */ + getRuntimeMappings(): estypes.MappingRuntimeFields { + // @ts-expect-error The ES client does not yet include the "composite" runtime type + return _.cloneDeep(this.runtimeFieldMap); + } + /** * Get formatter for a given field name. Return undefined if none exists * @param field @@ -432,9 +500,7 @@ export class DataView implements IIndexPattern { if (fieldObject) { if (!newCount) fieldObject.deleteCount(); else fieldObject.count = newCount; - return; } - this.setFieldAttrs(fieldName, 'count', newCount); } @@ -445,6 +511,92 @@ export class DataView implements IIndexPattern { public readonly deleteFieldFormat = (fieldName: string) => { delete this.fieldFormatMap[fieldName]; }; + + private addCompositeRuntimeField(name: string, runtimeField: RuntimeField): DataViewField[] { + const { fields } = runtimeField; + + // Make sure subFields are provided + if (fields === undefined || Object.keys(fields).length === 0) { + throw new Error(`Can't add composite runtime field [name = ${name}] without subfields.`); + } + + // Make sure no field with the same name already exist + if (this.getFieldByName(name) !== undefined) { + throw new Error( + `Can't create composite runtime field ["${name}"] as there is already a field with this name` + ); + } + + // We first remove the runtime composite field with the same name which will remove all of its subFields. + // This guarantees that we don't leave behind orphan data view fields + this.removeRuntimeField(name); + + const runtimeFieldSpec = removeFieldAttrs(runtimeField); + + // We don't add composite runtime fields to the field list as + // they are not fields but **holder** of fields. + // What we do add to the field list are all their subFields. + const dataViewFields = Object.entries(fields).map(([subFieldName, subField]) => + // Every child field gets the complete runtime field script for consumption by searchSource + this.updateOrAddRuntimeField(`${name}.${subFieldName}`, subField.type, runtimeFieldSpec, { + customLabel: subField.customLabel, + format: subField.format, + popularity: subField.popularity, + }) + ); + + this.runtimeFieldMap[name] = removeFieldAttrs(runtimeField); + return dataViewFields; + } + + private updateOrAddRuntimeField( + fieldName: string, + fieldType: RuntimeType, + runtimeFieldSpec: RuntimeFieldSpec, + config: FieldConfiguration + ): DataViewField { + if (fieldType === 'composite') { + throw new Error( + `Trying to add composite field as primmitive field, this shouldn't happen! [name = ${fieldName}]` + ); + } + + // Create the field if it does not exist or update an existing one + let createdField: DataViewField | undefined; + const existingField = this.getFieldByName(fieldName); + + if (existingField) { + existingField.runtimeField = runtimeFieldSpec; + } else { + createdField = this.fields.add({ + name: fieldName, + runtimeField: runtimeFieldSpec, + type: castEsToKbnFieldTypeName(fieldType), + esTypes: [fieldType], + aggregatable: true, + searchable: true, + count: config.popularity ?? 0, + readFromDocValues: false, + }); + } + + // Apply configuration to the field + if (config.customLabel || config.customLabel === null) { + this.setFieldCustomLabel(fieldName, config.customLabel); + } + + if (config.popularity || config.popularity === null) { + this.setFieldCount(fieldName, config.popularity); + } + + if (config.format) { + this.setFieldFormat(fieldName, config.format); + } else if (config.format === null) { + this.deleteFieldFormat(fieldName); + } + + return createdField ?? existingField!; + } } /** diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 8ce6b00d131bb..e93673bfa3ec5 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -14,7 +14,7 @@ import { castEsToKbnFieldTypeName } from '@kbn/field-types'; import { DATA_VIEW_SAVED_OBJECT_TYPE, FLEET_ASSETS_TO_IGNORE, SavedObjectsClientCommon } from '..'; import { createDataViewCache } from '.'; -import type { RuntimeField } from '../types'; +import type { RuntimeField, RuntimeFieldSpec, RuntimeType } from '../types'; import { DataView } from './data_view'; import { createEnsureDefaultDataView, EnsureDefaultDataView } from './ensure_default_data_view'; import { @@ -450,20 +450,36 @@ export class DataViewsService { spec.fieldAttrs ); + const addRuntimeFieldToSpecFields = ( + name: string, + fieldType: RuntimeType, + runtimeField: RuntimeFieldSpec + ) => { + spec.fields![name] = { + name, + type: castEsToKbnFieldTypeName(fieldType), + esTypes: [fieldType], + runtimeField, + aggregatable: true, + searchable: true, + readFromDocValues: false, + customLabel: spec.fieldAttrs?.[name]?.customLabel, + count: spec.fieldAttrs?.[name]?.count, + }; + }; + // CREATE RUNTIME FIELDS - for (const [key, value] of Object.entries(runtimeFieldMap || {})) { + for (const [name, runtimeField] of Object.entries(runtimeFieldMap || {})) { // do not create runtime field if mapped field exists - if (!spec.fields[key]) { - spec.fields[key] = { - name: key, - type: castEsToKbnFieldTypeName(value.type), - runtimeField: value, - aggregatable: true, - searchable: true, - readFromDocValues: false, - customLabel: spec.fieldAttrs?.[key]?.customLabel, - count: spec.fieldAttrs?.[key]?.count, - }; + if (!spec.fields[name]) { + // For composite runtime field we add the subFields, **not** the composite + if (runtimeField.type === 'composite') { + Object.entries(runtimeField.fields!).forEach(([subFieldName, subField]) => { + addRuntimeFieldToSpecFields(`${name}.${subFieldName}`, subField.type, runtimeField); + }); + } else { + addRuntimeFieldToSpecFields(name, runtimeField.type, runtimeField); + } } } } catch (err) { diff --git a/src/plugins/data_views/common/data_views/utils.ts b/src/plugins/data_views/common/data_views/utils.ts new file mode 100644 index 0000000000000..31a6b573ff2b3 --- /dev/null +++ b/src/plugins/data_views/common/data_views/utils.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { RuntimeField, RuntimeFieldSpec, RuntimeTypeExceptComposite } from '../types'; + +export const removeFieldAttrs = (runtimeField: RuntimeField): RuntimeFieldSpec => { + const { type, script, fields } = runtimeField; + const fieldsTypeOnly = fields && { + fields: Object.entries(fields).reduce((col, [fieldName, field]) => { + col[fieldName] = { type: field.type }; + return col; + }, {} as Record), + }; + + return { + type, + script, + ...fieldsTypeOnly, + }; +}; diff --git a/src/plugins/data_views/common/fields/data_view_field.test.ts b/src/plugins/data_views/common/fields/data_view_field.test.ts index 9c611354683c2..72fbd96d40b65 100644 --- a/src/plugins/data_views/common/fields/data_view_field.test.ts +++ b/src/plugins/data_views/common/fields/data_view_field.test.ts @@ -27,7 +27,7 @@ describe('Field', function () { script: 'script', lang: 'java' as const, count: 1, - esTypes: ['text'], // note, this will get replaced by the runtime field type + esTypes: ['keyword'], aggregatable: true, filterable: true, searchable: true, diff --git a/src/plugins/data_views/common/fields/data_view_field.ts b/src/plugins/data_views/common/fields/data_view_field.ts index ca74f0c52d253..a10c4268888db 100644 --- a/src/plugins/data_views/common/fields/data_view_field.ts +++ b/src/plugins/data_views/common/fields/data_view_field.ts @@ -8,9 +8,9 @@ /* eslint-disable max-classes-per-file */ -import { KbnFieldType, getKbnFieldType, castEsToKbnFieldTypeName } from '@kbn/field-types'; +import { KbnFieldType, getKbnFieldType } from '@kbn/field-types'; import { KBN_FIELD_TYPES } from '@kbn/field-types'; -import type { RuntimeField } from '../types'; +import type { RuntimeFieldSpec } from '../types'; import type { IFieldType } from './types'; import { FieldSpec, DataView } from '..'; import { @@ -49,7 +49,7 @@ export class DataViewField implements IFieldType { return this.spec.runtimeField; } - public set runtimeField(runtimeField: RuntimeField | undefined) { + public set runtimeField(runtimeField: RuntimeFieldSpec | undefined) { this.spec.runtimeField = runtimeField; } @@ -108,13 +108,11 @@ export class DataViewField implements IFieldType { } public get type() { - return this.runtimeField?.type - ? castEsToKbnFieldTypeName(this.runtimeField?.type) - : this.spec.type; + return this.spec.type; } public get esTypes() { - return this.runtimeField?.type ? [this.runtimeField?.type] : this.spec.esTypes; + return this.spec.esTypes; } public get scripted() { @@ -144,6 +142,10 @@ export class DataViewField implements IFieldType { return this.spec.isMapped; } + public get isRuntimeField() { + return !this.isMapped && this.runtimeField !== undefined; + } + // not writable, not serialized public get sortable() { return ( @@ -228,6 +230,10 @@ export class DataViewField implements IFieldType { isMapped: this.isMapped, }; } + + public isRuntimeCompositeSubField() { + return this.runtimeField?.type === 'composite'; + } } /** diff --git a/src/plugins/data_views/common/fields/field_list.ts b/src/plugins/data_views/common/fields/field_list.ts index e2c850c0c4dd0..c7ef23735d7bd 100644 --- a/src/plugins/data_views/common/fields/field_list.ts +++ b/src/plugins/data_views/common/fields/field_list.ts @@ -15,7 +15,7 @@ import { DataView } from '../data_views'; type FieldMap = Map; export interface IIndexPatternFieldList extends Array { - add(field: FieldSpec): void; + add(field: FieldSpec): DataViewField; getAll(): DataViewField[]; getByName(name: DataViewField['name']): DataViewField | undefined; getByType(type: DataViewField['type']): DataViewField[]; @@ -55,11 +55,12 @@ export const fieldList = ( public readonly getByType = (type: DataViewField['type']) => [ ...(this.groups.get(type) || new Map()).values(), ]; - public readonly add = (field: FieldSpec) => { + public readonly add = (field: FieldSpec): DataViewField => { const newField = new DataViewField({ ...field, shortDotsEnable }); this.push(newField); this.setByName(newField); this.setByGroup(newField); + return newField; }; public readonly remove = (field: IFieldType) => { diff --git a/src/plugins/data_views/common/index.ts b/src/plugins/data_views/common/index.ts index ece482ff79eaa..ab60fbf34071a 100644 --- a/src/plugins/data_views/common/index.ts +++ b/src/plugins/data_views/common/index.ts @@ -28,6 +28,8 @@ export type { FieldFormatMap, RuntimeType, RuntimeField, + RuntimeFieldSpec, + RuntimeFieldSubField, IIndexPattern, DataViewAttributes, IndexPatternAttributes, diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index f991b61bfad5f..cd0fad414813e 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -22,13 +22,49 @@ export type { QueryDslQueryContainer }; export type FieldFormatMap = Record; export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number]; -export interface RuntimeField { + +export type RuntimeTypeExceptComposite = Exclude; + +export interface RuntimeFieldBase { type: RuntimeType; script?: { source: string; }; } +/** + * The RuntimeField that will be sent in the ES Query "runtime_mappings" object + */ +export interface RuntimeFieldSpec extends RuntimeFieldBase { + fields?: Record< + string, + { + // It is not recursive, we can't create a composite inside a composite. + type: RuntimeTypeExceptComposite; + } + >; +} + +export interface FieldConfiguration { + format?: SerializedFieldFormat | null; + customLabel?: string; + popularity?: number; +} + +/** + * This is the RuntimeField interface enhanced with Data view field + * configuration: field format definition, customLabel or popularity. + * + * @see {@link RuntimeField} + */ +export interface RuntimeField extends RuntimeFieldBase, FieldConfiguration { + fields?: Record; +} + +export interface RuntimeFieldSubField extends FieldConfiguration { + type: RuntimeTypeExceptComposite; +} + /** * @deprecated * IIndexPattern allows for an IndexPattern OR an index pattern saved object @@ -216,7 +252,7 @@ export interface FieldSpec extends DataViewFieldBase { readFromDocValues?: boolean; indexed?: boolean; customLabel?: string; - runtimeField?: RuntimeField; + runtimeField?: RuntimeFieldSpec; // not persisted shortDotsEnable?: boolean; isMapped?: boolean; @@ -244,7 +280,7 @@ export interface DataViewSpec { typeMeta?: TypeMeta; type?: string; fieldFormats?: Record; - runtimeFieldMap?: Record; + runtimeFieldMap?: Record; fieldAttrs?: FieldAttrs; allowNoIndex?: boolean; } diff --git a/src/plugins/data_views/public/index.ts b/src/plugins/data_views/public/index.ts index 11b3edaa09628..fb926fe5f6316 100644 --- a/src/plugins/data_views/public/index.ts +++ b/src/plugins/data_views/public/index.ts @@ -15,7 +15,7 @@ export { } from '../common/lib'; export { onRedirectNoIndexPattern } from './data_views'; -export type { IIndexPatternFieldList, TypeMeta } from '../common'; +export type { IIndexPatternFieldList, TypeMeta, RuntimeType } from '../common'; export type { DataViewSpec } from '../common'; export { IndexPatternField, DataViewField, DataViewType, META_FIELDS } from '../common'; diff --git a/src/plugins/data_views/server/rest_api_routes/create_data_view.ts b/src/plugins/data_views/server/rest_api_routes/create_data_view.ts index 4b103ba87662c..00e6e94887f50 100644 --- a/src/plugins/data_views/server/rest_api_routes/create_data_view.ts +++ b/src/plugins/data_views/server/rest_api_routes/create_data_view.ts @@ -10,11 +10,7 @@ import { UsageCounter } from 'src/plugins/usage_collection/server'; import { schema } from '@kbn/config-schema'; import { DataViewSpec, DataViewsService } from 'src/plugins/data_views/common'; import { handleErrors } from './util/handle_errors'; -import { - fieldSpecSchema, - runtimeFieldSpecSchema, - serializedFieldFormatSchema, -} from './util/schemas'; +import { fieldSpecSchema, runtimeFieldSchema, serializedFieldFormatSchema } from './util/schemas'; import { IRouter, StartServicesAccessor } from '../../../../core/server'; import type { DataViewsServerPluginStartDependencies, DataViewsServerPluginStart } from '../types'; import { @@ -71,7 +67,7 @@ const dataViewSpecSchema = schema.object({ ) ), allowNoIndex: schema.maybe(schema.boolean()), - runtimeFieldMap: schema.maybe(schema.recordOf(schema.string(), runtimeFieldSpecSchema)), + runtimeFieldMap: schema.maybe(schema.recordOf(schema.string(), runtimeFieldSchema)), }); const registerCreateDataViewRouteFactory = diff --git a/src/plugins/data_views/server/rest_api_routes/fields/update_fields.test.ts b/src/plugins/data_views/server/rest_api_routes/fields/update_fields.test.ts index caf673ebbf3d9..453d6c90e5cc6 100644 --- a/src/plugins/data_views/server/rest_api_routes/fields/update_fields.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/fields/update_fields.test.ts @@ -23,6 +23,7 @@ describe('create runtime field', () => { fields: { getByName: jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce({}), }, + getFieldsByRuntimeFieldName: jest.fn().mockReturnValueOnce({}), } as unknown as DataView) ); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.test.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.test.ts index 555cbaad32252..ac2e6bc54fe15 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.test.ts @@ -20,8 +20,17 @@ describe('create runtime field', () => { ({ addRuntimeField: jest.fn(), fields: { - getByName: jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce({}), + getByName: jest + .fn() + .mockReturnValueOnce(undefined) + .mockReturnValueOnce(undefined) + .mockReturnValueOnce({}), }, + getRuntimeField: jest + .fn() + .mockReturnValueOnce(undefined) + .mockReturnValueOnce(undefined) + .mockReturnValueOnce({}), } as unknown as DataView) ); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.ts index d8f20c9ea6daf..8753e58c8e240 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/create_runtime_field.ts @@ -10,7 +10,7 @@ import { UsageCounter } from 'src/plugins/usage_collection/server'; import { DataViewsService, RuntimeField } from 'src/plugins/data_views/common'; import { schema } from '@kbn/config-schema'; import { handleErrors } from '../util/handle_errors'; -import { runtimeFieldSpecSchema } from '../util/schemas'; +import { runtimeFieldSchema } from '../util/schemas'; import { IRouter, StartServicesAccessor } from '../../../../../core/server'; import type { DataViewsServerPluginStart, @@ -45,18 +45,21 @@ export const createRuntimeField = async ({ usageCollection?.incrementCounter({ counterName }); const dataView = await dataViewsService.get(id); - if (dataView.fields.getByName(name)) { + if (dataView.fields.getByName(name) || dataView.getRuntimeField(name)) { throw new Error(`Field [name = ${name}] already exists.`); } - dataView.addRuntimeField(name, runtimeField); + const firstNameSegment = name.split('.')[0]; - const field = dataView.fields.getByName(name); - if (!field) throw new Error(`Could not create a field [name = ${name}].`); + if (dataView.fields.getByName(firstNameSegment) || dataView.getRuntimeField(firstNameSegment)) { + throw new Error(`Field [name = ${firstNameSegment}] already exists.`); + } + + const createdRuntimeFields = dataView.addRuntimeField(name, runtimeField); await dataViewsService.updateSavedObject(dataView); - return { dataView, field }; + return { dataView, fields: createdRuntimeFields }; }; const runtimeCreateFieldRouteFactory = @@ -84,7 +87,7 @@ const runtimeCreateFieldRouteFactory = minLength: 1, maxLength: 1_000, }), - runtimeField: runtimeFieldSpecSchema, + runtimeField: runtimeFieldSchema, }), }, }, @@ -100,16 +103,16 @@ const runtimeCreateFieldRouteFactory = const id = req.params.id; const { name, runtimeField } = req.body; - const { dataView, field } = await createRuntimeField({ + const { dataView, fields } = await createRuntimeField({ dataViewsService, usageCollection, counterName: `${req.route.method} ${path}`, id, name, - runtimeField, + runtimeField: runtimeField as RuntimeField, }); - return res.ok(responseFormatter({ serviceKey, dataView, field })); + return res.ok(responseFormatter({ serviceKey, dataView, fields })); }) ); }; diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.test.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.test.ts index 10fa4d63f3441..8262916e9ef4e 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.test.ts @@ -19,11 +19,7 @@ describe('delete runtime field', () => { async (id: string) => ({ removeRuntimeField: jest.fn(), - fields: { - getByName: jest.fn().mockReturnValueOnce({ - runtimeField: {}, - }), - }, + getRuntimeField: jest.fn().mockReturnValueOnce({}), } as unknown as DataView) ); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.ts index 706fe49582b5a..b846eb74771d9 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/delete_runtime_field.ts @@ -35,16 +35,12 @@ export const deleteRuntimeField = async ({ }: DeleteRuntimeFieldArgs) => { usageCollection?.incrementCounter({ counterName }); const dataView = await dataViewsService.get(id); - const field = dataView.fields.getByName(name); + const field = dataView.getRuntimeField(name); if (!field) { throw new ErrorIndexPatternFieldNotFound(id, name); } - if (!field.runtimeField) { - throw new Error('Only runtime fields can be deleted.'); - } - dataView.removeRuntimeField(name); await dataViewsService.updateSavedObject(dataView); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.test.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.test.ts index 4e75e7b0d6f2c..2552e88d5725a 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.test.ts @@ -23,6 +23,8 @@ describe('get runtime field', () => { runtimeField: {}, }), }, + getRuntimeField: jest.fn().mockReturnValueOnce({}), + getFieldsByRuntimeFieldName: jest.fn().mockReturnValueOnce({}), } as unknown as DataView) ); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.ts index bcb9e93152626..b8f87a21b819f 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/get_runtime_field.ts @@ -43,17 +43,13 @@ export const getRuntimeField = async ({ usageCollection?.incrementCounter({ counterName }); const dataView = await dataViewsService.get(id); - const field = dataView.fields.getByName(name); + const field = dataView.getRuntimeField(name); if (!field) { throw new ErrorIndexPatternFieldNotFound(id, name); } - if (!field.runtimeField) { - throw new Error('Only runtime fields can be retrieved.'); - } - - return { dataView, field }; + return { dataView, fields: Object.values(dataView.getFieldsByRuntimeFieldName(name) || {}) }; }; const getRuntimeFieldRouteFactory = @@ -95,7 +91,7 @@ const getRuntimeFieldRouteFactory = const id = req.params.id; const name = req.params.name; - const { dataView, field } = await getRuntimeField({ + const { dataView, fields } = await getRuntimeField({ dataViewsService, usageCollection, counterName: `${req.route.method} ${path}`, @@ -103,7 +99,7 @@ const getRuntimeFieldRouteFactory = name, }); - return res.ok(responseFormatter({ serviceKey, dataView, field })); + return res.ok(responseFormatter({ serviceKey, dataView, fields: fields || [] })); }) ); }; diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/put_runtime_field.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/put_runtime_field.ts index cf6b05c378e14..a50fe84133159 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/put_runtime_field.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/put_runtime_field.ts @@ -10,7 +10,7 @@ import { UsageCounter } from 'src/plugins/usage_collection/server'; import { DataViewsService, RuntimeField } from 'src/plugins/data_views/common'; import { schema } from '@kbn/config-schema'; import { handleErrors } from '../util/handle_errors'; -import { runtimeFieldSpecSchema } from '../util/schemas'; +import { runtimeFieldSchema } from '../util/schemas'; import { IRouter, StartServicesAccessor } from '../../../../../core/server'; import type { DataViewsServerPluginStart, @@ -55,14 +55,11 @@ export const putRuntimeField = async ({ dataView.removeRuntimeField(name); } - dataView.addRuntimeField(name, runtimeField); + const fields = dataView.addRuntimeField(name, runtimeField); await dataViewsService.updateSavedObject(dataView); - const field = dataView.fields.getByName(name); - if (!field) throw new Error(`Could not create a field [name = ${name}].`); - - return { dataView, field }; + return { dataView, fields }; }; const putRuntimeFieldRouteFactory = @@ -90,7 +87,7 @@ const putRuntimeFieldRouteFactory = minLength: 1, maxLength: 1_000, }), - runtimeField: runtimeFieldSpecSchema, + runtimeField: runtimeFieldSchema, }), }, }, @@ -104,9 +101,12 @@ const putRuntimeFieldRouteFactory = req ); const id = req.params.id; - const { name, runtimeField } = req.body; + const { name, runtimeField } = req.body as { + name: string; + runtimeField: RuntimeField; + }; - const { dataView, field } = await putRuntimeField({ + const { dataView, fields } = await putRuntimeField({ dataViewsService, id, name, @@ -115,7 +115,7 @@ const putRuntimeFieldRouteFactory = counterName: `${req.route.method} ${path}`, }); - return res.ok(responseFormatter({ serviceKey, dataView, field })); + return res.ok(responseFormatter({ serviceKey, dataView, fields })); }) ); }; diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.test.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.test.ts index a2db82466868b..3152ac1be91c2 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.test.ts @@ -18,20 +18,22 @@ const dataView = { }, } as DataView; -const field = { - toSpec: () => { - return { - name: 'field', - }; +const fields = [ + { + toSpec: () => { + return { + name: 'field', + }; + }, }, -} as DataViewField; +] as DataViewField[]; describe('responseFormatter', () => { it('returns correct format', () => { const response = responseFormatter({ serviceKey: SERVICE_KEY, dataView, - field, + fields, }); expect(response).toMatchSnapshot(); }); @@ -40,7 +42,7 @@ describe('responseFormatter', () => { const response = responseFormatter({ serviceKey: SERVICE_KEY_LEGACY, dataView, - field, + fields, }); expect(response).toMatchSnapshot(); }); diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.ts index 77f431cef48f8..7764f5bf0974b 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/response_formatter.ts @@ -11,14 +11,14 @@ import { SERVICE_KEY_LEGACY, SERVICE_KEY_TYPE, SERVICE_KEY } from '../../constan interface ResponseFormatterArgs { serviceKey: SERVICE_KEY_TYPE; - field: DataViewField; + fields: DataViewField[]; dataView: DataView; } -export const responseFormatter = ({ serviceKey, field, dataView }: ResponseFormatterArgs) => { +export const responseFormatter = ({ serviceKey, fields, dataView }: ResponseFormatterArgs) => { const response = { body: { - fields: [field.toSpec()], + fields: fields.map((field) => field.toSpec()), [SERVICE_KEY]: dataView.toSpec(), }, }; @@ -26,7 +26,7 @@ export const responseFormatter = ({ serviceKey, field, dataView }: ResponseForma const legacyResponse = { body: { [SERVICE_KEY_LEGACY]: dataView.toSpec(), - field: field.toSpec(), + field: fields[0].toSpec(), }, }; diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.test.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.test.ts index e21c0775f79b2..f3ca214b41d0f 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.test.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.test.ts @@ -22,7 +22,7 @@ describe('update runtime field', () => { addRuntimeField: jest.fn(), getRuntimeField: jest.fn().mockReturnValue({}), fields: { - getByName: jest.fn().mockReturnValue({ + getByName: jest.fn().mockReturnValueOnce({ runtimeField: {}, }), }, diff --git a/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.ts b/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.ts index d097408ec211d..27893ff1b7c05 100644 --- a/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.ts +++ b/src/plugins/data_views/server/rest_api_routes/runtime_fields/update_runtime_field.ts @@ -11,7 +11,7 @@ import { schema } from '@kbn/config-schema'; import { DataViewsService, RuntimeField } from 'src/plugins/data_views/common'; import { ErrorIndexPatternFieldNotFound } from '../../error'; import { handleErrors } from '../util/handle_errors'; -import { runtimeFieldSpec, runtimeFieldSpecTypeSchema } from '../util/schemas'; +import { runtimeFieldSchema } from '../util/schemas'; import { IRouter, StartServicesAccessor } from '../../../../../core/server'; import type { DataViewsServerPluginStart, @@ -52,16 +52,14 @@ export const updateRuntimeField = async ({ } dataView.removeRuntimeField(name); - dataView.addRuntimeField(name, { + const fields = dataView.addRuntimeField(name, { ...existingRuntimeField, ...runtimeField, }); await dataViewsService.updateSavedObject(dataView); - const field = dataView.fields.getByName(name); - if (!field) throw new Error(`Could not create a field [name = ${name}].`); - return { dataView, field }; + return { dataView, fields }; }; const updateRuntimeFieldRouteFactory = @@ -90,12 +88,7 @@ const updateRuntimeFieldRouteFactory = }), body: schema.object({ name: schema.never(), - runtimeField: schema.object({ - ...runtimeFieldSpec, - // We need to overwrite the below fields on top of `runtimeFieldSpec`, - // because some fields would be optional - type: schema.maybe(runtimeFieldSpecTypeSchema), - }), + runtimeField: runtimeFieldSchema, }), }, }, @@ -112,7 +105,7 @@ const updateRuntimeFieldRouteFactory = const name = req.params.name; const runtimeField = req.body.runtimeField as Partial; - const { dataView, field } = await updateRuntimeField({ + const { dataView, fields } = await updateRuntimeField({ dataViewsService, usageCollection, counterName: `${req.route.method} ${path}`, @@ -121,7 +114,7 @@ const updateRuntimeFieldRouteFactory = runtimeField, }); - return res.ok(responseFormatter({ serviceKey, dataView, field })); + return res.ok(responseFormatter({ serviceKey, dataView, fields })); }) ); }; diff --git a/src/plugins/data_views/server/rest_api_routes/update_data_view.ts b/src/plugins/data_views/server/rest_api_routes/update_data_view.ts index ed3f75599b90a..e71ae8c8ec88e 100644 --- a/src/plugins/data_views/server/rest_api_routes/update_data_view.ts +++ b/src/plugins/data_views/server/rest_api_routes/update_data_view.ts @@ -10,11 +10,7 @@ import { schema } from '@kbn/config-schema'; import { DataViewSpec, DataViewsService } from 'src/plugins/data_views/common'; import { UsageCounter } from 'src/plugins/usage_collection/server'; import { handleErrors } from './util/handle_errors'; -import { - fieldSpecSchema, - runtimeFieldSpecSchema, - serializedFieldFormatSchema, -} from './util/schemas'; +import { fieldSpecSchema, runtimeFieldSchema, serializedFieldFormatSchema } from './util/schemas'; import { IRouter, StartServicesAccessor } from '../../../../core/server'; import type { DataViewsServerPluginStartDependencies, DataViewsServerPluginStart } from '../types'; import { @@ -39,7 +35,7 @@ const indexPatternUpdateSchema = schema.object({ fieldFormats: schema.maybe(schema.recordOf(schema.string(), serializedFieldFormatSchema)), fields: schema.maybe(schema.recordOf(schema.string(), fieldSpecSchema)), allowNoIndex: schema.maybe(schema.boolean()), - runtimeFieldMap: schema.maybe(schema.recordOf(schema.string(), runtimeFieldSpecSchema)), + runtimeFieldMap: schema.maybe(schema.recordOf(schema.string(), runtimeFieldSchema)), }); interface UpdateDataViewArgs { diff --git a/src/plugins/data_views/server/rest_api_routes/util/schemas.ts b/src/plugins/data_views/server/rest_api_routes/util/schemas.ts index 79f493f303801..e31a4314c76f3 100644 --- a/src/plugins/data_views/server/rest_api_routes/util/schemas.ts +++ b/src/plugins/data_views/server/rest_api_routes/util/schemas.ts @@ -7,7 +7,7 @@ */ import { schema, Type } from '@kbn/config-schema'; -import { RUNTIME_FIELD_TYPES, RuntimeType } from '../../../common'; +import { /* RUNTIME_FIELD_TYPES,*/ RuntimeType } from '../../../common'; export const serializedFieldFormatSchema = schema.object({ id: schema.maybe(schema.string()), @@ -60,17 +60,63 @@ export const fieldSpecSchema = schema.object(fieldSpecSchemaFields, { unknowns: 'ignore', }); -export const runtimeFieldSpecTypeSchema = schema.oneOf( - RUNTIME_FIELD_TYPES.map((runtimeFieldType) => schema.literal(runtimeFieldType)) as [ +export const RUNTIME_FIELD_TYPES2 = [ + 'keyword', + 'long', + 'double', + 'date', + 'ip', + 'boolean', + 'geo_point', +] as const; + +export const runtimeFieldNonCompositeFieldsSpecTypeSchema = schema.oneOf( + RUNTIME_FIELD_TYPES2.map((runtimeFieldType) => schema.literal(runtimeFieldType)) as [ Type ] ); -export const runtimeFieldSpec = { - type: runtimeFieldSpecTypeSchema, + +export const primitiveRuntimeFieldSchema = schema.object({ + type: runtimeFieldNonCompositeFieldsSpecTypeSchema, script: schema.maybe( schema.object({ source: schema.string(), }) ), -}; -export const runtimeFieldSpecSchema = schema.object(runtimeFieldSpec); + format: schema.maybe(serializedFieldFormatSchema), + customLabel: schema.maybe(schema.string()), + popularity: schema.maybe( + schema.number({ + min: 0, + }) + ), +}); + +export const compositeRuntimeFieldSchema = schema.object({ + type: schema.literal('composite') as Type, + script: schema.maybe( + schema.object({ + source: schema.string(), + }) + ), + fields: schema.maybe( + schema.recordOf( + schema.string(), + schema.object({ + type: runtimeFieldNonCompositeFieldsSpecTypeSchema, + format: schema.maybe(serializedFieldFormatSchema), + customLabel: schema.maybe(schema.string()), + popularity: schema.maybe( + schema.number({ + min: 0, + }) + ), + }) + ) + ), +}); + +export const runtimeFieldSchema = schema.oneOf([ + primitiveRuntimeFieldSchema, + compositeRuntimeFieldSchema, +]); diff --git a/src/plugins/vis_types/vega/public/data_model/search_api.test.ts b/src/plugins/vis_types/vega/public/data_model/search_api.test.ts index 9ecc160246e28..979f7f05cdf1d 100644 --- a/src/plugins/vis_types/vega/public/data_model/search_api.test.ts +++ b/src/plugins/vis_types/vega/public/data_model/search_api.test.ts @@ -22,6 +22,7 @@ const mockComputedFields = ( getComputedFields: () => ({ runtimeFields, }), + getRuntimeMappings: () => runtimeFields, }, ]); }; diff --git a/src/plugins/vis_types/vega/public/data_model/search_api.ts b/src/plugins/vis_types/vega/public/data_model/search_api.ts index 11302ad65d56b..6a7ee55b299d0 100644 --- a/src/plugins/vis_types/vega/public/data_model/search_api.ts +++ b/src/plugins/vis_types/vega/public/data_model/search_api.ts @@ -32,7 +32,7 @@ export const extendSearchParamsWithRuntimeFields = async ( const indexPattern = (await indexPatterns.find(indexPatternString)).find( (index) => index.title === indexPatternString ); - runtimeMappings = indexPattern?.getComputedFields().runtimeFields; + runtimeMappings = indexPattern?.getRuntimeMappings(); } return { diff --git a/test/api_integration/apis/index_patterns/runtime_fields_crud/create_runtime_field/main.ts b/test/api_integration/apis/index_patterns/runtime_fields_crud/create_runtime_field/main.ts index bc0773df056ed..af1bdcf8b8aa2 100644 --- a/test/api_integration/apis/index_patterns/runtime_fields_crud/create_runtime_field/main.ts +++ b/test/api_integration/apis/index_patterns/runtime_fields_crud/create_runtime_field/main.ts @@ -92,8 +92,105 @@ export default function ({ getService }: FtrProviderContext) { expect(field.runtimeField.type).to.be('long'); expect(field.runtimeField.script.source).to.be("emit(doc['field_name'].value)"); expect(field.scripted).to.be(false); + + const response3 = await supertest + .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field`) + .send({ + name: 'runtimeBar', + runtimeField: { + type: 'long', + script: { + source: "emit(doc['field_name'].value)", + }, + }, + }); + + expect(response3.status).to.be(400); + await supertest.delete(`${config.path}/${response1.body[config.serviceKey].id}`); }); + + it('prevents field name collisions', async () => { + const title = `basic*`; + const response1 = await supertest.post(config.path).send({ + override: true, + [config.serviceKey]: { + title, + }, + }); + + const response2 = await supertest + .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field`) + .send({ + name: 'runtimeBar', + runtimeField: { + type: 'long', + script: { + source: "emit(doc['field_name'].value)", + }, + }, + }); + + expect(response2.status).to.be(200); + + const response3 = await supertest + .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field`) + .send({ + name: 'runtimeBar', + runtimeField: { + type: 'long', + script: { + source: "emit(doc['field_name'].value)", + }, + }, + }); + + expect(response3.status).to.be(400); + + const response4 = await supertest + .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field`) + .send({ + name: 'runtimeComposite', + runtimeField: { + type: 'composite', + script: { + source: 'emit("a","a"); emit("b","b")', + }, + fields: { + a: { + type: 'keyword', + }, + b: { + type: 'keyword', + }, + }, + }, + }); + + expect(response4.status).to.be(200); + + const response5 = await supertest + .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field`) + .send({ + name: 'runtimeComposite', + runtimeField: { + type: 'composite', + script: { + source: 'emit("a","a"); emit("b","b")', + }, + fields: { + a: { + type: 'keyword', + }, + b: { + type: 'keyword', + }, + }, + }, + }); + + expect(response5.status).to.be(400); + }); }); }); }); diff --git a/test/api_integration/apis/index_patterns/runtime_fields_crud/delete_runtime_field/errors.ts b/test/api_integration/apis/index_patterns/runtime_fields_crud/delete_runtime_field/errors.ts index 54c982ec7f325..22099425df394 100644 --- a/test/api_integration/apis/index_patterns/runtime_fields_crud/delete_runtime_field/errors.ts +++ b/test/api_integration/apis/index_patterns/runtime_fields_crud/delete_runtime_field/errors.ts @@ -59,16 +59,6 @@ export default function ({ getService }: FtrProviderContext) { expect(response1.status).to.be(404); }); - it('returns error when attempting to delete a field which is not a runtime field', async () => { - const response2 = await supertest.delete( - `${config.path}/${indexPattern.id}/runtime_field/foo` - ); - - expect(response2.status).to.be(400); - expect(response2.body.statusCode).to.be(400); - expect(response2.body.message).to.be('Only runtime fields can be deleted.'); - }); - it('returns error when ID is too long', async () => { const id = `xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx-xxxxxxxxxx`; const response = await supertest.delete(`${config.path}/${id}/runtime_field/foo`); diff --git a/test/api_integration/apis/index_patterns/runtime_fields_crud/get_runtime_field/errors.ts b/test/api_integration/apis/index_patterns/runtime_fields_crud/get_runtime_field/errors.ts index b6bebb224b33f..f7a751387902c 100644 --- a/test/api_integration/apis/index_patterns/runtime_fields_crud/get_runtime_field/errors.ts +++ b/test/api_integration/apis/index_patterns/runtime_fields_crud/get_runtime_field/errors.ts @@ -68,16 +68,6 @@ export default function ({ getService }: FtrProviderContext) { '[request params.id]: value has length [1759] but it must have a maximum length of [1000].' ); }); - - it('returns error when attempting to fetch a field which is not a runtime field', async () => { - const response2 = await supertest.get( - `${config.path}/${indexPattern.id}/runtime_field/foo` - ); - - expect(response2.status).to.be(400); - expect(response2.body.statusCode).to.be(400); - expect(response2.body.message).to.be('Only runtime fields can be retrieved.'); - }); }); }); }); diff --git a/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/errors.ts b/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/errors.ts index 09e781d70bb8d..87687230897dc 100644 --- a/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/errors.ts +++ b/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/errors.ts @@ -20,6 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const id = `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx-${Date.now()}`; const response = await supertest.post(`${config.path}/${id}/runtime_field/foo`).send({ runtimeField: { + type: 'keyword', script: { source: "doc['something_new'].value", }, @@ -34,6 +35,7 @@ export default function ({ getService }: FtrProviderContext) { const response = await supertest.post(`${config.path}/${id}/runtime_field/foo`).send({ name: 'foo', runtimeField: { + type: 'keyword', script: { source: "doc['something_new'].value", }, diff --git a/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/main.ts b/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/main.ts index 83e1b4d5716ea..49388bb55166a 100644 --- a/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/main.ts +++ b/test/api_integration/apis/index_patterns/runtime_fields_crud/update_runtime_field/main.ts @@ -54,6 +54,7 @@ export default function ({ getService }: FtrProviderContext) { .post(`${config.path}/${response1.body[config.serviceKey].id}/runtime_field/runtimeFoo`) .send({ runtimeField: { + type: 'keyword', script: { source: "doc['something_new'].value", }, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector_service.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector_service.ts index 303d54c9d45cc..e7ad239e0c980 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector_service.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/full_time_range_selector/full_time_range_selector_service.ts @@ -6,7 +6,6 @@ */ import moment from 'moment'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { TimefilterContract } from 'src/plugins/data/public'; import dateMath from '@elastic/datemath'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; @@ -27,8 +26,7 @@ export async function setFullTimeRange( query?: QueryDslQueryContainer, excludeFrozenData?: boolean ): Promise { - const runtimeMappings = indexPattern.getComputedFields() - .runtimeFields as estypes.MappingRuntimeFields; + const runtimeMappings = indexPattern.getRuntimeMappings(); const resp = await getTimeFieldRange({ index: indexPattern.title, timeFieldName: indexPattern.timeFieldName, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index a6176b8e3c1ce..c4e9e898454d5 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -198,7 +198,7 @@ export const useDataVisualizerGridData = ( sessionId: searchSessionId, index: currentIndexPattern.title, timeFieldName: currentIndexPattern.timeFieldName, - runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields, + runtimeFieldMap: currentIndexPattern.getRuntimeMappings(), aggregatableFields, nonAggregatableFields, fieldsToFetch, diff --git a/x-pack/plugins/infra/common/dependency_mocks/index_patterns.ts b/x-pack/plugins/infra/common/dependency_mocks/index_patterns.ts index 03d3ec757bf55..46f961d6de472 100644 --- a/x-pack/plugins/infra/common/dependency_mocks/index_patterns.ts +++ b/x-pack/plugins/infra/common/dependency_mocks/index_patterns.ts @@ -5,15 +5,17 @@ * 2.0. */ +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { from, of } from 'rxjs'; import { delay } from 'rxjs/operators'; import { DataView, DataViewsContract } from '../../../../../src/plugins/data_views/common'; -import { fieldList, FieldSpec, RuntimeField } from '../../../../../src/plugins/data/common'; +import { fieldList, FieldSpec } from '../../../../../src/plugins/data/common'; type IndexPatternMock = Pick< DataView, | 'fields' | 'getComputedFields' + | 'getRuntimeMappings' | 'getFieldByName' | 'getTimeField' | 'id' @@ -23,6 +25,7 @@ type IndexPatternMock = Pick< >; type IndexPatternMockSpec = Pick & { fields: FieldSpec[]; + runtimeFields?: estypes.MappingRuntimeFields; }; export const createIndexPatternMock = ({ @@ -30,6 +33,7 @@ export const createIndexPatternMock = ({ title, type = undefined, fields, + runtimeFields, timeFieldName, }: IndexPatternMockSpec): IndexPatternMock => { const indexPatternFieldList = fieldList(fields); @@ -43,21 +47,12 @@ export const createIndexPatternMock = ({ isTimeBased: () => timeFieldName != null, getFieldByName: (fieldName) => indexPatternFieldList.find(({ name }) => name === fieldName), getComputedFields: () => ({ - runtimeFields: indexPatternFieldList.reduce>( - (accumulatedFields, { name, runtimeField }) => ({ - ...accumulatedFields, - ...(runtimeField != null - ? { - [name]: runtimeField, - } - : {}), - }), - {} - ), + runtimeFields: runtimeFields ?? {}, scriptFields: {}, storedFields: [], docvalueFields: [], }), + getRuntimeMappings: () => runtimeFields ?? {}, }; }; diff --git a/x-pack/plugins/infra/common/log_sources/resolved_log_source_configuration.ts b/x-pack/plugins/infra/common/log_sources/resolved_log_source_configuration.ts index 70f8e1cebabfe..914c55824373a 100644 --- a/x-pack/plugins/infra/common/log_sources/resolved_log_source_configuration.ts +++ b/x-pack/plugins/infra/common/log_sources/resolved_log_source_configuration.ts @@ -7,7 +7,6 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { DataView, DataViewsContract } from '../../../../../src/plugins/data_views/common'; -import { ObjectEntries } from '../utility_types'; import { TIMESTAMP_FIELD, TIEBREAKER_FIELD } from '../constants'; import { ResolveLogSourceConfigurationError } from './errors'; import { @@ -106,27 +105,5 @@ const resolveKibanaIndexPatternReference = async ( // this might take other sources of runtime fields into account in the future const resolveRuntimeMappings = (indexPattern: DataView): estypes.MappingRuntimeFields => { - const { runtimeFields } = indexPattern.getComputedFields(); - - const runtimeMappingsFromIndexPattern = ( - Object.entries(runtimeFields) as ObjectEntries - ).reduce( - (accumulatedMappings, [runtimeFieldName, runtimeFieldSpec]) => ({ - ...accumulatedMappings, - [runtimeFieldName]: { - type: runtimeFieldSpec.type, - ...(runtimeFieldSpec.script != null - ? { - script: { - lang: 'painless', // required in the es types - source: runtimeFieldSpec.script.source, - }, - } - : {}), - }, - }), - {} - ); - - return runtimeMappingsFromIndexPattern; + return indexPattern.getRuntimeMappings(); }; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts index 73f832e834c61..cfa9e84fb3651 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts @@ -80,7 +80,6 @@ describe('LogEntries search strategy', () => { runtime_field: { type: 'keyword', script: { - lang: 'painless', source: 'emit("runtime value")', }, }, @@ -359,6 +358,14 @@ const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({ searchable: true, }, ], + runtimeFields: { + runtime_field: { + type: 'keyword', + script: { + source: 'emit("runtime value")', + }, + }, + }, }), ]), }); diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts index 20f3e41cef159..c2f3c70580040 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts @@ -79,7 +79,6 @@ describe('LogEntry search strategy', () => { runtime_field: { type: 'keyword', script: { - lang: 'painless', source: 'emit("runtime value")', }, }, @@ -314,6 +313,14 @@ const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({ searchable: true, }, ], + runtimeFields: { + runtime_field: { + type: 'keyword', + script: { + source: 'emit("runtime value")', + }, + }, + }, }), ]), }); diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index c5f7cfab9c245..bb235774da3bf 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -173,6 +173,8 @@ async function legacyFetchFieldExistenceSampling({ const indexPattern = await dataViewsService.get(indexPatternId); const fields = buildFieldList(indexPattern, metaFields); + const runtimeMappings = indexPattern.getRuntimeMappings(); + const docs = await fetchIndexPatternStats({ fromDate, toDate, @@ -181,6 +183,7 @@ async function legacyFetchFieldExistenceSampling({ index: indexPattern.title, timeFieldName: timeFieldName || indexPattern.timeFieldName, fields, + runtimeMappings, includeFrozen, }); @@ -216,6 +219,7 @@ async function fetchIndexPatternStats({ fromDate, toDate, fields, + runtimeMappings, includeFrozen, }: { client: ElasticsearchClient; @@ -225,12 +229,12 @@ async function fetchIndexPatternStats({ fromDate?: string; toDate?: string; fields: Field[]; + runtimeMappings: estypes.MappingRuntimeFields; includeFrozen: boolean; }) { const query = toQuery(timeFieldName, fromDate, toDate, dslQuery); const scriptedFields = fields.filter((f) => f.isScript); - const runtimeFields = fields.filter((f) => f.runtimeField); const result = await client.search( { index, @@ -242,11 +246,7 @@ async function fetchIndexPatternStats({ sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [], fields: ['*'], _source: false, - runtime_mappings: runtimeFields.reduce((acc, field) => { - if (!field.runtimeField) return acc; - acc[field.name] = field.runtimeField; - return acc; - }, {} as Record), + runtime_mappings: runtimeMappings, script_fields: scriptedFields.reduce((acc, field) => { acc[field.name] = { script: { diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 6c1a93759030a..127d798cb76c2 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -84,6 +84,7 @@ export async function initFieldsRoute(setup: CoreSetup) { .filter((f) => f.runtimeField) .reduce((acc, f) => { if (!f.runtimeField) return acc; + // @ts-expect-error The MappingRuntimeField from @elastic/elasticsearch does not expose the "composite" runtime type yet acc[f.name] = f.runtimeField; return acc; }, {} as Record); diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index 7a07a3b0ae6ab..1d73413b3e386 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -117,6 +117,7 @@ export const useIndexData = ( if (combinedRuntimeMappings !== undefined) { result = Object.keys(combinedRuntimeMappings).map((fieldName) => { const field = combinedRuntimeMappings[fieldName]; + // @ts-expect-error @elastic/elasticsearch does not support yet "composite" type for runtime fields const schema = getDataGridSchemaFromESFieldType(field.type); return { id: fieldName, schema }; });