diff --git a/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts new file mode 100644 index 0000000000000..87f92e4f978dc --- /dev/null +++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { omit } from 'lodash'; +import { FieldDefinitionConfig } from '../models'; + +// Parameters that we consider first class and provide a curated experience for +const FIRST_CLASS_PARAMETERS = ['type', 'format']; + +// Advanced parameters that we provide a generic experience (JSON blob) for +export const getAdvancedParameters = (fieldName: string, fieldConfig: FieldDefinitionConfig) => { + // @timestamp can't ignore malformed dates as it's used for sorting in logsdb + const additionalOmissions = fieldName === '@timestamp' ? ['ignore_malformed'] : []; + return omit(fieldConfig, FIRST_CLASS_PARAMETERS.concat(additionalOmissions)); +}; diff --git a/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts index f8afc7a941b9f..e4a8a35a9cdf9 100644 --- a/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts +++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts @@ -10,3 +10,4 @@ export * from './hierarchy'; export * from './lifecycle'; export * from './condition_fields'; export * from './condition_to_query_dsl'; +export * from './field_definition'; diff --git a/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts b/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts index 303e8ffd7014e..7f9f7121d52f5 100644 --- a/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts +++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import { z } from '@kbn/zod'; import { NonEmptyString } from '@kbn/zod-helpers'; @@ -20,15 +21,25 @@ export const FIELD_DEFINITION_TYPES = [ export type FieldDefinitionType = (typeof FIELD_DEFINITION_TYPES)[number]; -export interface FieldDefinitionConfig { +// We redefine "first class" parameters +export type FieldDefinitionConfig = MappingProperty & { type: FieldDefinitionType; format?: string; -} +}; + +// Parameters that we provide a generic (JSON blob) experience for +export type FieldDefinitionConfigAdvancedParameters = Omit< + FieldDefinitionConfig, + 'type' | 'format' +>; -export const fieldDefinitionConfigSchema: z.Schema = z.object({ - type: z.enum(FIELD_DEFINITION_TYPES), - format: z.optional(NonEmptyString), -}); +export const fieldDefinitionConfigSchema: z.Schema = z.intersection( + z.record(z.string(), z.unknown()), + z.object({ + type: z.enum(FIELD_DEFINITION_TYPES), + format: z.optional(NonEmptyString), + }) +); export interface FieldDefinition { [x: string]: FieldDefinitionConfig; @@ -39,9 +50,9 @@ export const fieldDefinitionSchema: z.Schema = z.record( fieldDefinitionConfigSchema ); -export interface InheritedFieldDefinitionConfig extends FieldDefinitionConfig { +export type InheritedFieldDefinitionConfig = FieldDefinitionConfig & { from: string; -} +}; export interface InheritedFieldDefinition { [x: string]: InheritedFieldDefinitionConfig; @@ -52,9 +63,9 @@ export const inheritedFieldDefinitionSchema: z.Schema z.intersection(fieldDefinitionConfigSchema, z.object({ from: NonEmptyString })) ); -export interface NamedFieldDefinitionConfig extends FieldDefinitionConfig { +export type NamedFieldDefinitionConfig = FieldDefinitionConfig & { name: string; -} +}; export const namedFieldDefinitionConfigSchema: z.Schema = z.intersection( diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index d0089692c419f..0636ec7e39c39 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -10,7 +10,13 @@ import { MappingDateProperty, MappingProperty, } from '@elastic/elasticsearch/lib/api/types'; -import { WiredStreamDefinition, isDslLifecycle, isIlmLifecycle, isRoot } from '@kbn/streams-schema'; +import { + WiredStreamDefinition, + getAdvancedParameters, + isDslLifecycle, + isIlmLifecycle, + isRoot, +} from '@kbn/streams-schema'; import { ASSET_VERSION } from '../../../../common/constants'; import { logsSettings } from './logs_layer'; import { getComponentTemplateName } from './name'; @@ -25,6 +31,13 @@ export function generateLayer( const property: MappingProperty = { type: props.type, }; + + const advancedParameters = getAdvancedParameters(field, props); + + if (Object.keys(advancedParameters).length > 0) { + Object.assign(property, advancedParameters); + } + if (field === '@timestamp') { // @timestamp can't ignore malformed dates as it's used for sorting in logsdb (property as MappingDateProperty).ignore_malformed = false; @@ -32,6 +45,7 @@ export function generateLayer( if (props.type === 'date' && props.format) { (property as MappingDateProperty).format = props.format; } + properties[field] = property; }); diff --git a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts index f8fbc194f4add..7c672886ac17b 100644 --- a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts +++ b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts @@ -165,7 +165,10 @@ export const schemaFieldsSimulationRoute = createServerRoute({ const propertiesForSimulation = Object.fromEntries( params.body.field_definitions.map((field) => [ field.name, - { type: field.type, ...(field.format ? { format: field.format } : {}) }, + { + type: field.type, + ...(field.format ? { format: field.format } : {}), + }, ]) ); diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx new file mode 100644 index 0000000000000..6ff83de1958e6 --- /dev/null +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiAccordion, + EuiCodeBlock, + EuiLink, + EuiPanel, + EuiSpacer, + EuiText, + useGeneratedHtmlId, +} from '@elastic/eui'; +import { CodeEditor } from '@kbn/code-editor'; +import { i18n } from '@kbn/i18n'; +import React, { useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { getAdvancedParameters } from '@kbn/streams-schema'; +import { SchemaField } from '../types'; +import { useKibana } from '../../../hooks/use_kibana'; + +const label = i18n.translate('xpack.streams.advancedFieldMappingOptions.label', { + defaultMessage: 'Advanced field mapping parameters', +}); + +export const AdvancedFieldMappingOptions = ({ + field, + onChange, + isEditing, +}: { + field: SchemaField; + onChange: (field: Partial) => void; + isEditing: boolean; +}) => { + const { core } = useKibana(); + + const accordionId = useGeneratedHtmlId({ prefix: 'accordionID' }); + + const jsonOptions = useMemo( + () => (field.additionalParameters ? JSON.stringify(field.additionalParameters, null, 2) : ''), + [field.additionalParameters] + ); + + return ( + + + + + + + ), + }} + /> + + + {isEditing ? ( + { + try { + onChange({ + additionalParameters: + value === '' ? undefined : getAdvancedParameters(field.name, JSON.parse(value)), + }); + } catch (error: unknown) { + // do nothing + } + }} + /> + ) : ( + + {jsonOptions ?? ''} + + )} + + + ); +}; diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx index adb98b2946dfb..88fa9f380f424 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx @@ -16,7 +16,6 @@ import { } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; -import useToggle from 'react-use/lib/useToggle'; import { WiredStreamDefinition } from '@kbn/streams-schema'; import { useStreamsAppRouter } from '../../../hooks/use_streams_app_router'; import { FieldParent } from '../field_parent'; @@ -56,18 +55,17 @@ const FIELD_SUMMARIES = { interface FieldSummaryProps { field: SchemaField; - isEditingByDefault: boolean; + isEditing: boolean; + toggleEditMode: () => void; stream: WiredStreamDefinition; onChange: (field: Partial) => void; } export const FieldSummary = (props: FieldSummaryProps) => { - const { field, isEditingByDefault, onChange, stream } = props; + const { field, isEditing, toggleEditMode, onChange, stream } = props; const router = useStreamsAppRouter(); - const [isEditing, toggleEditMode] = useToggle(isEditingByDefault); - return ( <> diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx index 8124672853553..06ecd95856725 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx @@ -19,9 +19,11 @@ import React, { useReducer } from 'react'; import { i18n } from '@kbn/i18n'; import { WiredStreamDefinition } from '@kbn/streams-schema'; import useAsyncFn from 'react-use/lib/useAsyncFn'; +import useToggle from 'react-use/lib/useToggle'; import { SamplePreviewTable } from './sample_preview_table'; import { FieldSummary } from './field_summary'; import { SchemaField } from '../types'; +import { AdvancedFieldMappingOptions } from './advanced_field_mapping_options'; export interface SchemaEditorFlyoutProps { field: SchemaField; @@ -40,6 +42,8 @@ export const SchemaEditorFlyout = ({ isEditingByDefault = false, withFieldSimulation = false, }: SchemaEditorFlyoutProps) => { + const [isEditing, toggleEditMode] = useToggle(isEditingByDefault); + const [nextField, setNextField] = useReducer( (prev: SchemaField, updated: Partial) => ({ @@ -66,10 +70,16 @@ export const SchemaEditorFlyout = ({ + {withFieldSimulation && ( diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts index 5a1c905fc79c8..8c12517b047d6 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts @@ -7,7 +7,11 @@ import { i18n } from '@kbn/i18n'; import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; -import { NamedFieldDefinitionConfig, WiredStreamGetResponse } from '@kbn/streams-schema'; +import { + NamedFieldDefinitionConfig, + WiredStreamGetResponse, + getAdvancedParameters, +} from '@kbn/streams-schema'; import { isEqual, omit } from 'lodash'; import { useMemo, useCallback } from 'react'; import { useStreamsAppFetch } from '../../../hooks/use_streams_app_fetch'; @@ -59,6 +63,7 @@ export const useSchemaFields = ({ name, type: field.type, format: field.format, + additionalParameters: getAdvancedParameters(name, field), parent: field.from, status: 'inherited', }) @@ -69,6 +74,7 @@ export const useSchemaFields = ({ name, type: field.type, format: field.format, + additionalParameters: getAdvancedParameters(name, field), parent: definition.stream.name, status: 'mapped', }) @@ -132,12 +138,12 @@ export const useSchemaFields = ({ refreshFields(); } catch (error) { - toasts.addError(error, { + toasts.addError(new Error(error.body.message), { title: i18n.translate('xpack.streams.streamDetailSchemaEditorEditErrorToast', { defaultMessage: 'Something went wrong editing the {field} field', values: { field: field.name }, }), - toastMessage: error.message, + toastMessage: error.body.message, toastLifeTimeMs: 5000, }); } diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx index 9a086c49a9355..51e082b8e38df 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx @@ -107,7 +107,7 @@ const createCellRenderer = return ; } - return field[columnId as keyof SchemaField] || EMPTY_CONTENT; + return <>{field[columnId as keyof SchemaField] || EMPTY_CONTENT}; }; const createFieldActionsCellRenderer = (fields: SchemaField[]): EuiDataGridControlColumn => ({ diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts index 36cbc11b4a545..04eef3f5385a2 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { FieldDefinitionConfig, WiredStreamDefinition } from '@kbn/streams-schema'; +import { + FieldDefinitionConfig, + FieldDefinitionConfigAdvancedParameters, + WiredStreamDefinition, +} from '@kbn/streams-schema'; export type SchemaFieldStatus = 'inherited' | 'mapped' | 'unmapped'; export type SchemaFieldType = FieldDefinitionConfig['type']; @@ -13,16 +17,19 @@ export type SchemaFieldType = FieldDefinitionConfig['type']; export interface BaseSchemaField extends Omit { name: string; parent: string; + format?: string; } export interface MappedSchemaField extends BaseSchemaField { status: 'inherited' | 'mapped'; type: SchemaFieldType; + additionalParameters?: FieldDefinitionConfigAdvancedParameters; } export interface UnmappedSchemaField extends BaseSchemaField { status: 'unmapped'; - type?: SchemaFieldType | undefined; + type?: SchemaFieldType; + additionalParameters?: FieldDefinitionConfigAdvancedParameters; } export type SchemaField = MappedSchemaField | UnmappedSchemaField; diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts index 66ed70c9d3d9d..7669e7515b072 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts @@ -12,5 +12,8 @@ export const convertToFieldDefinitionConfig = ( field: MappedSchemaField ): FieldDefinitionConfig => ({ type: field.type, - ...(field.format && field.type === 'date' ? { format: field.format } : {}), + ...(field.format && field.type === 'date' ? { format: field.format as string } : {}), + ...(field.additionalParameters && Object.keys(field.additionalParameters).length > 0 + ? field.additionalParameters + : {}), });