diff --git a/x-pack/platform/packages/shared/kbn-streams-schema/index.ts b/x-pack/platform/packages/shared/kbn-streams-schema/index.ts index 16a9d56847043..a01587fd88368 100644 --- a/x-pack/platform/packages/shared/kbn-streams-schema/index.ts +++ b/x-pack/platform/packages/shared/kbn-streams-schema/index.ts @@ -39,7 +39,9 @@ export { export { keepFields, namespacePrefixes, + otelReservedFields, isNamespacedEcsField, + isOtelReservedField, getRegularEcsField, } from './src/helpers/namespaced_ecs'; export { getAdvancedParameters } from './src/helpers/get_advanced_parameters'; diff --git a/x-pack/platform/packages/shared/kbn-streams-schema/src/helpers/namespaced_ecs.ts b/x-pack/platform/packages/shared/kbn-streams-schema/src/helpers/namespaced_ecs.ts index 45745c745eec5..2118aaba68a63 100644 --- a/x-pack/platform/packages/shared/kbn-streams-schema/src/helpers/namespaced_ecs.ts +++ b/x-pack/platform/packages/shared/kbn-streams-schema/src/helpers/namespaced_ecs.ts @@ -11,6 +11,23 @@ import { KEEP_FIELDS, NAMESPACE_PREFIXES } from '@kbn/streamlang'; export const keepFields: readonly string[] = KEEP_FIELDS; export const namespacePrefixes: readonly string[] = NAMESPACE_PREFIXES; +/** + * Field names that are reserved for OTel compatibility mode. + * These are either passthrough objects or alias fields that cannot be used as custom field names. + * IMPORTANT: This list must match the keys of baseMappings in logs_layer.ts. + * A test in logs_layer.test.ts ensures these stay in sync. + */ +export const otelReservedFields = [ + 'body', + 'attributes', + 'scope', + 'resource', + 'span.id', + 'message', + 'trace.id', + 'log.level', +] as const; + export const aliases: Record = { trace_id: 'trace.id', span_id: 'span.id', @@ -38,3 +55,11 @@ export function isNamespacedEcsField(field: string): boolean { KEEP_FIELDS.includes(field as any) ); } + +/** + * Checks if a field name is reserved for OTel compatibility mode. + * Reserved fields are either passthrough objects or alias fields that cannot be redefined. + */ +export function isOtelReservedField(field: string): boolean { + return otelReservedFields.includes(field as (typeof otelReservedFields)[number]); +} diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/logs_layer.test.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/logs_layer.test.ts index 20a56ebe5f4a6..06523162bd154 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/logs_layer.test.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/logs_layer.test.ts @@ -6,9 +6,18 @@ */ import type { InheritedFieldDefinition, Streams } from '@kbn/streams-schema'; +import { otelReservedFields } from '@kbn/streams-schema'; import { addAliasesForNamespacedFields, baseMappings, baseFields } from './logs_layer'; describe('logs_layer', () => { + describe('baseMappings and otelReservedFields sync', () => { + it('should have baseMappings keys match otelReservedFields', () => { + const baseMappingsKeys = Object.keys(baseMappings).sort(); + const reservedFieldsSorted = [...otelReservedFields].sort(); + + expect(baseMappingsKeys).toEqual(reservedFieldsSorted); + }); + }); describe('addAliasesForNamespacedFields', () => { let mockStreamDefinition: Streams.WiredStream.Definition; let mockInheritedFields: InheritedFieldDefinition; diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.test.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.test.tsx new file mode 100644 index 0000000000000..4fa3d73690c4a --- /dev/null +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.test.tsx @@ -0,0 +1,340 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { I18nProvider } from '@kbn/i18n-react'; +import { AddFieldFlyout } from './add_field_flyout'; +import { + createMockClassicStreamDefinition, + createMockWiredStreamDefinition, +} from '../../shared/mocks'; +import { SchemaEditorContextProvider } from '../schema_editor_context'; + +jest.mock('../../../../hooks/use_kibana', () => ({ + useKibana: () => ({ + core: { + docLinks: { + links: { + elasticsearch: { + mappingParameters: 'https://elastic.co/docs/mapping-parameters', + }, + }, + }, + }, + dependencies: { + start: { + streams: { + streamsRepositoryClient: { + fetch: jest.fn(), + }, + }, + fieldsMetadata: { + useFieldsMetadata: () => ({ + fieldsMetadata: {}, + loading: false, + }), + }, + }, + }, + }), +})); + +jest.mock('../../../../hooks/use_streams_app_router', () => ({ + useStreamsAppRouter: () => ({ + link: jest.fn(() => '/mock-link'), + }), +})); + +jest.mock('@kbn/code-editor', () => ({ + CodeEditor: () =>
CodeEditor
, +})); + +const renderAddFieldFlyout = ( + streamType: 'wired' | 'classic', + existingFieldNames: string[] = [] +) => { + const definition = + streamType === 'wired' + ? createMockWiredStreamDefinition() + : createMockClassicStreamDefinition(); + + const fields = existingFieldNames.map((name) => ({ + name, + type: 'keyword' as const, + parent: definition.stream.name, + status: 'mapped' as const, + })); + + const onClose = jest.fn(); + const onAddField = jest.fn(); + + return { + onClose, + onAddField, + ...render( + + + + + + ), + }; +}; + +const typeFieldName = async (user: ReturnType, fieldName: string) => { + const comboBox = screen.getByTestId('streamsAppSchemaEditorAddFieldFlyoutFieldName'); + const input = comboBox.querySelector('input'); + if (!input) throw new Error('Could not find input element'); + await user.clear(input); + await user.type(input, fieldName); + await user.keyboard('{Enter}'); +}; + +const getFieldNameError = () => { + const formRow = screen + .getByTestId('streamsAppSchemaEditorAddFieldFlyoutFieldName') + .closest('.euiFormRow'); + return formRow?.querySelector('.euiFormErrorText')?.textContent ?? null; +}; + +describe('AddFieldFlyout', () => { + describe('Field name validation for wired streams', () => { + it('shows error for non-namespaced field names', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'invalid_field'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain( + "Field invalid_field is not allowed to be defined as it doesn't match the namespaced ECS or OTel schema" + ); + }); + }); + + it('shows error for OTel reserved field names that are also not namespaced (message)', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + // `message` is not in keepFields and doesn't start with namespace prefix, + // so it fails the namespacing check first (matching server-side behavior) + await typeFieldName(user, 'message'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain("doesn't match the namespaced ECS or OTel schema"); + }); + }); + + it('shows error for OTel reserved fields that are also not namespaced (trace.id)', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + // `trace.id` is not in keepFields and doesn't start with namespace prefix, + // so it fails the namespacing check first + await typeFieldName(user, 'trace.id'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain("doesn't match the namespaced ECS or OTel schema"); + }); + }); + + it('shows error for body passthrough object (which is in keepFields)', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + // `body` IS in keepFields, so it passes the namespacing check, + // but then fails the OTel reserved check + await typeFieldName(user, 'body'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain('automatic alias'); + }); + }); + + it('shows error for attributes passthrough object (not in keepFields)', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + // `attributes` is not in keepFields and doesn't start with namespace prefix, + // so it fails the namespacing check first + await typeFieldName(user, 'attributes'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain("doesn't match the namespaced ECS or OTel schema"); + }); + }); + + it('allows namespaced field names with attributes prefix', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'attributes.custom_field'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + + it('allows namespaced field names with body.structured prefix', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'body.structured.data'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + + it('allows namespaced field names with resource.attributes prefix', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'resource.attributes.host.name'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + + it('allows keepFields like @timestamp', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, '@timestamp'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + + it('allows keepFields like trace_id', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'trace_id'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + }); + + describe('Field name validation for classic streams', () => { + it('allows non-namespaced field names', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('classic'); + + await typeFieldName(user, 'custom_field'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + + it('allows OTel reserved field names', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('classic'); + + await typeFieldName(user, 'message'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toBeNull(); + }); + }); + }); + + describe('Common validation for all stream types', () => { + it('shows error for duplicate field names in wired streams', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired', ['attributes.existing_field']); + + await typeFieldName(user, 'attributes.existing_field'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain('A field with this name already exists'); + }); + }); + + it('shows error for duplicate field names in classic streams', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('classic', ['existing_field']); + + await typeFieldName(user, 'existing_field'); + + await waitFor(() => { + const error = getFieldNameError(); + expect(error).toContain('A field with this name already exists'); + }); + }); + }); + + describe('Add field button state', () => { + it('disables the Add field button when field name validation fails', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'invalid_field'); + + await waitFor(() => { + const addFieldButton = screen.getByTestId('streamsAppSchemaEditorAddFieldButton'); + expect(addFieldButton).toBeDisabled(); + }); + }); + + it('disables the Add field button when field name is valid but type is not selected', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'attributes.custom_field'); + + await waitFor(() => { + const addFieldButton = screen.getByTestId('streamsAppSchemaEditorAddFieldButton'); + expect(addFieldButton).toBeDisabled(); + }); + }); + + it('enables the Add field button when all required fields are valid', async () => { + const user = userEvent.setup(); + renderAddFieldFlyout('wired'); + + await typeFieldName(user, 'attributes.custom_field'); + const typeSelect = screen.getByTestId('streamsAppFieldFormTypeSelect'); + await user.click(typeSelect); + const keywordOption = screen.getByTestId('option-type-keyword'); + await user.click(keywordOption); + + await waitFor(() => { + const addFieldButton = screen.getByTestId('streamsAppSchemaEditorAddFieldButton'); + expect(addFieldButton).not.toBeDisabled(); + }); + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.tsx index 158bfe4cb97fd..5ad0de3f1e8a3 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/add_field_flyout.tsx @@ -23,7 +23,13 @@ import { } from '@elastic/eui'; import React, { useMemo } from 'react'; import { i18n } from '@kbn/i18n'; -import { isSchema, recursiveRecord, Streams } from '@kbn/streams-schema'; +import { + isSchema, + recursiveRecord, + Streams, + isNamespacedEcsField, + isOtelReservedField, +} from '@kbn/streams-schema'; import type { SubmitHandler } from 'react-hook-form'; import { FormProvider, useController, useForm, useFormContext, useWatch } from 'react-hook-form'; import { CodeEditor } from '@kbn/code-editor'; @@ -125,7 +131,7 @@ export const AddFieldFlyout = ({ onAddField, onClose }: AddFieldFlyoutProps) => {i18n.translate('xpack.streams.schemaEditor.addFieldFlyout.addButtonLabel', { defaultMessage: 'Add field', @@ -149,6 +155,8 @@ export const FieldNameSelector = () => { source: ['ecs', 'otel'], }); + const isWiredStream = Streams.WiredStream.Definition.is(stream); + const { field, fieldState } = useController({ name: 'name', rules: { @@ -162,6 +170,28 @@ export const FieldNameSelector = () => { { defaultMessage: 'A field with this name already exists.' } ); } + if (isWiredStream) { + if (!isNamespacedEcsField(name)) { + return i18n.translate( + 'xpack.streams.schemaEditor.addFieldFlyout.fieldNameNotNamespacedError', + { + defaultMessage: + "Field {fieldName} is not allowed to be defined as it doesn't match the namespaced ECS or OTel schema.", + values: { fieldName: name }, + } + ); + } + if (isOtelReservedField(name)) { + return i18n.translate( + 'xpack.streams.schemaEditor.addFieldFlyout.fieldNameOtelReservedError', + { + defaultMessage: + 'Field {fieldName} is an automatic alias of another field because of OTel compatibility mode.', + values: { fieldName: name }, + } + ); + } + } return true; }, }, diff --git a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_mapping/wired_streams_schema.spec.ts b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_mapping/wired_streams_schema.spec.ts index 1bf22a4a5c491..41f5cae6d4220 100644 --- a/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_mapping/wired_streams_schema.spec.ts +++ b/x-pack/platform/plugins/shared/streams_app/test/scout/ui/tests/data_management/data_mapping/wired_streams_schema.spec.ts @@ -190,5 +190,55 @@ test.describe( value: `Alias for ${ecsFieldName}`, }); }); + + test('validates field names for wired streams', async ({ page, pageObjects }) => { + await pageObjects.streams.expectSchemaEditorTableVisible(); + + // Open the "Add field" flyout + await page.getByTestId('streamsAppContentAddFieldButton').click(); + await expect( + page.getByTestId('streamsAppSchemaEditorAddFieldFlyoutCloseButton') + ).toBeVisible(); + + // Try to add a non-namespaced field - should show error + const invalidFieldName = 'invalid_field'; + await pageObjects.streams.typeFieldName(invalidFieldName); + + // Check that an error is displayed + const formRow = page + .getByTestId('streamsAppSchemaEditorAddFieldFlyoutFieldName') + .locator('..'); + await expect(formRow.locator('.euiFormErrorText')).toContainText( + "doesn't match the namespaced ECS or OTel schema" + ); + + // Check that the Add button is disabled when there's a validation error + const addButton = page.getByTestId('streamsAppSchemaEditorAddFieldButton'); + await expect(addButton).toBeDisabled(); + + // Clear and try with a valid namespaced field + const clearButton = page.getByTestId('comboBoxClearButton'); + await clearButton.click(); + + const validFieldName = 'attributes.valid_field'; + await pageObjects.streams.typeFieldName(validFieldName); + + // Error should be gone + await expect(formRow.locator('.euiFormErrorText')).toBeHidden(); + + // Set field type and verify Add button works + await pageObjects.streams.setFieldMappingType('keyword'); + await addButton.click(); + await expect( + page.getByTestId('streamsAppSchemaEditorAddFieldFlyoutCloseButton') + ).toBeHidden(); + + // Verify the field was staged (visible in review) + await pageObjects.streams.reviewStagedFieldMappingChanges(); + await expect(page.getByText(validFieldName)).toBeVisible(); + + // Close the modal to clean up + await pageObjects.streams.closeSchemaReviewModal(); + }); } );