From c89039612ec137cf6757fe8296b65c88bfeabce8 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 08:40:44 +0200 Subject: [PATCH 01/50] feat(fields-metadata): add server and client services --- x-pack/plugins/fields_metadata/README.md | 3 + .../common/fields_metadata/common.ts | 8 ++ .../common/fields_metadata/index.ts | 9 ++ .../common/fields_metadata/types.ts | 82 +++++++++++++++++++ .../v1/find_fields_metadata.ts | 25 ++++++ .../common/fields_metadata/v1/index.ts | 8 ++ .../plugins/fields_metadata/common/index.ts | 17 ++++ .../plugins/fields_metadata/common/latest.ts | 8 ++ .../fields_metadata/common/runtime_types.ts | 29 +++++++ x-pack/plugins/fields_metadata/jest.config.js | 17 ++++ x-pack/plugins/fields_metadata/kibana.jsonc | 14 ++++ .../public/hooks/use_fields_metadata/index.ts | 0 .../use_fields_metadata.test.ts | 0 .../use_fields_metadata.ts | 36 ++++++++ .../plugins/fields_metadata/public/index.ts | 21 +++++ .../plugins/fields_metadata/public/mocks.tsx | 17 ++++ .../plugins/fields_metadata/public/plugin.ts | 45 ++++++++++ .../fields_metadata_client.mock.ts | 12 +++ .../fields_metadata/fields_metadata_client.ts | 41 ++++++++++ .../fields_metadata_service.mock.ts | 16 ++++ .../fields_metadata_service.ts | 27 ++++++ .../public/services/fields_metadata/index.ts | 10 +++ .../public/services/fields_metadata/types.ts | 27 ++++++ .../plugins/fields_metadata/public/types.ts | 40 +++++++++ .../server/fields_metadata_server.ts | 13 +++ .../plugins/fields_metadata/server/index.ts | 15 ++++ .../server/lib/shared_types.ts | 21 +++++ .../plugins/fields_metadata/server/mocks.ts | 32 ++++++++ .../plugins/fields_metadata/server/plugin.ts | 65 +++++++++++++++ .../fields_metadata/find_fields_metadata.ts | 53 ++++++++++++ .../server/routes/fields_metadata/index.ts | 12 +++ .../fields_metadata_client.mock.ts | 13 +++ .../fields_metadata_client.test.ts | 37 +++++++++ .../fields_metadata/fields_metadata_client.ts | 66 +++++++++++++++ .../fields_metadata_service.mock.ts | 17 ++++ .../fields_metadata_service.ts | 44 ++++++++++ .../server/services/fields_metadata/index.ts | 14 ++++ .../ecs_fields_source_client.ts | 49 +++++++++++ .../integration_fields_source_client.ts | 38 +++++++++ .../fields_metadata/source_clients/types.ts | 15 ++++ .../server/services/fields_metadata/types.ts | 25 ++++++ .../plugins/fields_metadata/server/types.ts | 34 ++++++++ x-pack/plugins/fields_metadata/tsconfig.json | 15 ++++ 43 files changed, 1090 insertions(+) create mode 100755 x-pack/plugins/fields_metadata/README.md create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/common.ts create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/index.ts create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/types.ts create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/v1/index.ts create mode 100644 x-pack/plugins/fields_metadata/common/index.ts create mode 100644 x-pack/plugins/fields_metadata/common/latest.ts create mode 100644 x-pack/plugins/fields_metadata/common/runtime_types.ts create mode 100644 x-pack/plugins/fields_metadata/jest.config.js create mode 100644 x-pack/plugins/fields_metadata/kibana.jsonc create mode 100644 x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts create mode 100644 x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts create mode 100644 x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts create mode 100644 x-pack/plugins/fields_metadata/public/index.ts create mode 100644 x-pack/plugins/fields_metadata/public/mocks.tsx create mode 100644 x-pack/plugins/fields_metadata/public/plugin.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.mock.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts create mode 100644 x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts create mode 100644 x-pack/plugins/fields_metadata/public/types.ts create mode 100644 x-pack/plugins/fields_metadata/server/fields_metadata_server.ts create mode 100644 x-pack/plugins/fields_metadata/server/index.ts create mode 100644 x-pack/plugins/fields_metadata/server/lib/shared_types.ts create mode 100644 x-pack/plugins/fields_metadata/server/mocks.ts create mode 100644 x-pack/plugins/fields_metadata/server/plugin.ts create mode 100644 x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts create mode 100644 x-pack/plugins/fields_metadata/server/routes/fields_metadata/index.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.mock.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/index.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts create mode 100644 x-pack/plugins/fields_metadata/server/types.ts create mode 100644 x-pack/plugins/fields_metadata/tsconfig.json diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md new file mode 100755 index 0000000000000..4940151d84281 --- /dev/null +++ b/x-pack/plugins/fields_metadata/README.md @@ -0,0 +1,3 @@ +# Fields metadata plugin + +Exposes services for async usage and search of field metadata. diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts new file mode 100644 index 0000000000000..fd1769dda63e6 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const FIND_FIELDS_METADATA_URL = '/api/fields_metadata'; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts new file mode 100644 index 0000000000000..4328ef2e38aa6 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './common'; +export * from './types'; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts new file mode 100644 index 0000000000000..e81ec94416d36 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EcsFlat } from '@elastic/ecs'; +import * as rt from 'io-ts'; + +export const allowedValueRT = rt.intersection([ + rt.type({ + description: rt.string, + name: rt.string, + }), + rt.partial({ + expected_event_types: rt.array(rt.string), + beta: rt.string, + }), +]); + +export const multiFieldRT = rt.type({ + flat_name: rt.string, + name: rt.string, + type: rt.string, +}); + +export const ecsFieldMetadataRT = rt.intersection([ + rt.type({ + dashed_name: rt.string, + description: rt.string, + flat_name: rt.string, + level: rt.string, + name: rt.string, + normalize: rt.array(rt.string), + short: rt.string, + type: rt.string, + }), + rt.partial({ + allowed_values: allowedValueRT, + beta: rt.string, + doc_values: rt.boolean, + example: rt.unknown, + expected_values: rt.array(rt.string), + format: rt.string, + ignore_above: rt.number, + index: rt.boolean, + input_format: rt.string, + multi_fields: rt.array(multiFieldRT), + object_type: rt.string, + original_fieldset: rt.string, + output_format: rt.string, + output_precision: rt.number, + pattern: rt.string, + required: rt.boolean, + scaling_factor: rt.number, + }), +]); + +export const integrationFieldMetadataRT = rt.intersection([ + rt.type({ + name: rt.string, + description: rt.string, + type: rt.string, + flat_name: rt.string, + }), + rt.partial({ + example: rt.unknown, + }), +]); + +export const fieldMetadataRT = rt.union([ecsFieldMetadataRT, integrationFieldMetadataRT]); + +export type TEcsFields = typeof EcsFlat; +export type EcsFieldName = keyof TEcsFields; +export type IntegrationFieldName = string; + +export type EcsFieldMetadata = TEcsFields[EcsFieldName]; +export type IntegrationFieldMetadata = rt.TypeOf; + +export type FieldName = EcsFieldName | (IntegrationFieldName & {}); +export type FieldMetadata = EcsFieldMetadata | IntegrationFieldMetadata; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts new file mode 100644 index 0000000000000..e7d400c036b07 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { arrayToStringRt } from '@kbn/io-ts-utils'; +import * as rt from 'io-ts'; +import { fieldMetadataRT } from '../types'; + +export const findFieldsMetadataRequestQueryRT = rt.exact( + rt.partial({ + fieldNames: arrayToStringRt.pipe(rt.array(rt.string)), + }) +); + +export const findFieldsMetadataResponsePayloadRT = rt.type({ + fields: rt.record(rt.string, fieldMetadataRT), +}); + +export type FindFieldsMetadataRequestQuery = rt.TypeOf; +export type FindFieldsMetadataResponsePayload = rt.TypeOf< + typeof findFieldsMetadataResponsePayloadRT +>; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/index.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/index.ts new file mode 100644 index 0000000000000..d54098d5d36fc --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './find_fields_metadata'; diff --git a/x-pack/plugins/fields_metadata/common/index.ts b/x-pack/plugins/fields_metadata/common/index.ts new file mode 100644 index 0000000000000..70582b4f99541 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { fieldMetadataRT } from './fields_metadata/types'; +export type { + EcsFieldMetadata, + EcsFieldName, + FieldMetadata, + FieldName, + IntegrationFieldMetadata, + IntegrationFieldName, + TEcsFields, +} from './fields_metadata/types'; diff --git a/x-pack/plugins/fields_metadata/common/latest.ts b/x-pack/plugins/fields_metadata/common/latest.ts new file mode 100644 index 0000000000000..b3ee1e9932f95 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './fields_metadata/v1'; diff --git a/x-pack/plugins/fields_metadata/common/runtime_types.ts b/x-pack/plugins/fields_metadata/common/runtime_types.ts new file mode 100644 index 0000000000000..19df92e3af99f --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/runtime_types.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RouteValidationFunction } from '@kbn/core/server'; +import { createPlainError, decodeOrThrow, formatErrors, throwErrors } from '@kbn/io-ts-utils'; +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { Errors, Type } from 'io-ts'; + +export { createPlainError, decodeOrThrow, formatErrors, throwErrors }; + +type ValdidationResult = ReturnType>; + +export const createValidationFunction = + ( + runtimeType: Type + ): RouteValidationFunction => + (inputValue, { badRequest, ok }) => + pipe( + runtimeType.decode(inputValue), + fold>( + (errors: Errors) => badRequest(formatErrors(errors)), + (result: DecodedValue) => ok(result) + ) + ); diff --git a/x-pack/plugins/fields_metadata/jest.config.js b/x-pack/plugins/fields_metadata/jest.config.js new file mode 100644 index 0000000000000..47ba6f1a113bd --- /dev/null +++ b/x-pack/plugins/fields_metadata/jest.config.js @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/plugins/fields_metadata'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/fields_metadata', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/plugins/fields_metadata/{common,public,server}/**/*.{ts,tsx}', + ], +}; diff --git a/x-pack/plugins/fields_metadata/kibana.jsonc b/x-pack/plugins/fields_metadata/kibana.jsonc new file mode 100644 index 0000000000000..2befc0c7be07b --- /dev/null +++ b/x-pack/plugins/fields_metadata/kibana.jsonc @@ -0,0 +1,14 @@ +{ + "type": "plugin", + "id": "@kbn/fields-metadata-plugin", + "owner": "@elastic/obs-ux-logs-team", + "description": "Exposes services for async usage and search of fields metadata.", + "plugin": { + "id": "fieldsMetadata", + "server": true, + "browser": true, + "configPath": ["xpack", "fields_metadata"], + "requiredPlugins": [], + "requiredBundles": [], + } +} diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts new file mode 100644 index 0000000000000..fed8b2e42dea3 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect } from 'react'; +import useAsyncFn from 'react-use/lib/useAsyncFn'; +import { FieldName } from '../../../common'; +import { IFieldsMetadataClient } from '../../services/fields_metadata'; + +interface UseFieldsMetadataFactoryDeps { + fieldsMetadataClient: IFieldsMetadataClient; +} + +interface Params { + fieldNames: FieldName[]; +} + +export const createUseFieldsMetadataHook = ({ + fieldsMetadataClient, +}: UseFieldsMetadataFactoryDeps) => { + return ({ fieldNames }: Params) => { + const [{ error, loading, value: fieldsMetadata }, load] = useAsyncFn( + () => fieldsMetadataClient.find({ fieldNames }), + [fieldNames] + ); + + useEffect(() => { + load(); + }, [load]); + + return { fieldsMetadata, loading, error }; + }; +}; diff --git a/x-pack/plugins/fields_metadata/public/index.ts b/x-pack/plugins/fields_metadata/public/index.ts new file mode 100644 index 0000000000000..e05935884742f --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldsMetadataPlugin } from './plugin'; + +export type { + FieldsMetadataClientSetupExports, + FieldsMetadataClientStartExports, + FieldsMetadataClientSetupDeps, + FieldsMetadataClientStartDeps, +} from './types'; + +// This exports static code and TypeScript types, +// as well as, Kibana Platform `plugin()` initializer. +export function plugin() { + return new FieldsMetadataPlugin(); +} diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx new file mode 100644 index 0000000000000..151e36eb7d178 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; +import { FieldsMetadataClientStartExports } from './types'; + +export const createFieldsMetadataPluginStartMock = + (): jest.Mocked => ({ + fieldsMetadata: createFieldsMetadataServiceStartMock(), + }); + +export const _ensureTypeCompatibility = (): FieldsMetadataClientStartExports => + createFieldsMetadataPluginStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts new file mode 100644 index 0000000000000..ade8954335100 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart } from '@kbn/core/public'; +import { createUseFieldsMetadataHook } from './hooks/use_fields_metadata/use_fields_metadata'; +import { FieldsMetadataService } from './services/fields_metadata'; +import { + FieldsMetadataClientCoreSetup, + FieldsMetadataClientPluginClass, + FieldsMetadataClientSetupDeps, + FieldsMetadataClientStartDeps, +} from './types'; + +export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { + private fieldsMetadata: FieldsMetadataService; + + constructor() { + this.fieldsMetadata = new FieldsMetadataService(); + } + + public setup(_: FieldsMetadataClientCoreSetup, pluginsSetup: FieldsMetadataClientSetupDeps) { + this.fieldsMetadata.setup(); + + return {}; + } + + public start(core: CoreStart, plugins: FieldsMetadataClientStartDeps) { + const { http } = core; + + const { client } = this.fieldsMetadata.start({ http }); + + const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataClient: client }); + + return { + client, + useFieldsMetadata, + }; + } + + public stop() {} +} diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.mock.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.mock.ts new file mode 100644 index 0000000000000..6264dcc6dc080 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.mock.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IFieldsMetadataClient } from './types'; + +export const createFieldsMetadataClientMock = (): jest.Mocked => ({ + find: jest.fn(), +}); diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts new file mode 100644 index 0000000000000..a8384c23a0ece --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core/public'; +import { + FindFieldsMetadataRequestQuery, + findFieldsMetadataRequestQueryRT, + FindFieldsMetadataResponsePayload, + findFieldsMetadataResponsePayloadRT, +} from '../../../common/latest'; +import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; +import { decodeOrThrow } from '../../../common/runtime_types'; +import { IFieldsMetadataClient } from './types'; + +export class FieldsMetadataClient implements IFieldsMetadataClient { + constructor(private readonly http: HttpStart) {} + + public async find({ + fieldNames, + }: FindFieldsMetadataRequestQuery): Promise { + const query = findFieldsMetadataRequestQueryRT.encode({ fieldNames }); + + const response = await this.http + .get(FIND_FIELDS_METADATA_URL, { query, version: '1' }) + .catch((error) => { + throw new Error(`Failed to fetch ecs fields ${fieldNames?.join() ?? ''}: ${error}`); + }); + + const data = decodeOrThrow( + findFieldsMetadataResponsePayloadRT, + (message: string) => + new Error(`Failed to decode ecs fields ${fieldNames?.join() ?? ''}: ${message}"`) + )(response); + + return data; + } +} diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts new file mode 100644 index 0000000000000..bfef30e155fd1 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createFieldsMetadataClientMock } from './fields_metadata_client.mock'; +import { FieldsMetadataServiceStart } from './types'; + +export const createFieldsMetadataServiceStartMock = () => ({ + client: createFieldsMetadataClientMock(), +}); + +export const _ensureTypeCompatibility = (): FieldsMetadataServiceStart => + createFieldsMetadataServiceStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts new file mode 100644 index 0000000000000..96b5e9464eac5 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldsMetadataClient } from './fields_metadata_client'; +import { + FieldsMetadataServiceStartDeps, + FieldsMetadataServiceSetup, + FieldsMetadataServiceStart, +} from './types'; + +export class FieldsMetadataService { + public setup(): FieldsMetadataServiceSetup { + return {}; + } + + public start({ http }: FieldsMetadataServiceStartDeps): FieldsMetadataServiceStart { + const client = new FieldsMetadataClient(http); + + return { + client, + }; + } +} diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts new file mode 100644 index 0000000000000..b0253f9d2881e --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './fields_metadata_client'; +export * from './fields_metadata_service'; +export * from './types'; diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts new file mode 100644 index 0000000000000..0f103f3a88fb4 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpStart } from '@kbn/core/public'; +import { + FindFieldsMetadataRequestQuery, + FindFieldsMetadataResponsePayload, +} from '../../../common/latest'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataServiceSetup {} + +export interface FieldsMetadataServiceStart { + client: IFieldsMetadataClient; +} + +export interface FieldsMetadataServiceStartDeps { + http: HttpStart; +} + +export interface IFieldsMetadataClient { + find(params: FindFieldsMetadataRequestQuery): Promise; +} diff --git a/x-pack/plugins/fields_metadata/public/types.ts b/x-pack/plugins/fields_metadata/public/types.ts new file mode 100644 index 0000000000000..a3f0639690764 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/types.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin as PluginClass } from '@kbn/core/public'; + +import { IFieldsMetadataClient } from './services/fields_metadata'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataClientSetupExports {} + +export interface FieldsMetadataClientStartExports { + client: IFieldsMetadataClient; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataClientSetupDeps {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataClientStartDeps {} + +export type FieldsMetadataClientCoreSetup = CoreSetup< + FieldsMetadataClientStartDeps, + FieldsMetadataClientStartExports +>; +export type FieldsMetadataClientCoreStart = CoreStart; +export type FieldsMetadataClientPluginClass = PluginClass< + FieldsMetadataClientSetupExports, + FieldsMetadataClientStartExports, + FieldsMetadataClientSetupDeps, + FieldsMetadataClientStartDeps +>; + +export type FieldsMetadataClientStartServicesAccessor = + FieldsMetadataClientCoreSetup['getStartServices']; +export type FieldsMetadataClientStartServices = + ReturnType; diff --git a/x-pack/plugins/fields_metadata/server/fields_metadata_server.ts b/x-pack/plugins/fields_metadata/server/fields_metadata_server.ts new file mode 100644 index 0000000000000..96c57c9847091 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/fields_metadata_server.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldsMetadataBackendLibs } from './lib/shared_types'; +import { initFieldsMetadataRoutes } from './routes/fields_metadata'; + +export const initFieldsMetadataServer = (libs: FieldsMetadataBackendLibs) => { + initFieldsMetadataRoutes(libs); +}; diff --git a/x-pack/plugins/fields_metadata/server/index.ts b/x-pack/plugins/fields_metadata/server/index.ts new file mode 100644 index 0000000000000..212fcc95b646b --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '@kbn/core/server'; + +export type { FieldsMetadataPluginSetup, FieldsMetadataPluginStart } from './types'; + +export async function plugin(context: PluginInitializerContext) { + const { FieldsMetadataPlugin } = await import('./plugin'); + return new FieldsMetadataPlugin(context); +} diff --git a/x-pack/plugins/fields_metadata/server/lib/shared_types.ts b/x-pack/plugins/fields_metadata/server/lib/shared_types.ts new file mode 100644 index 0000000000000..ed6f15cf9c485 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/lib/shared_types.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import { RequestHandlerContext } from '@kbn/core-http-request-handler-context-server'; +import { IRouter } from '@kbn/core-http-server'; +import { + FieldsMetadataPluginStartServicesAccessor, + FieldsMetadataServerPluginSetupDeps, +} from '../types'; + +export interface FieldsMetadataBackendLibs { + getStartServices: FieldsMetadataPluginStartServicesAccessor; + logger: Logger; + plugins: FieldsMetadataServerPluginSetupDeps; + router: IRouter; +} diff --git a/x-pack/plugins/fields_metadata/server/mocks.ts b/x-pack/plugins/fields_metadata/server/mocks.ts new file mode 100644 index 0000000000000..ed65ff2f88b65 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/mocks.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createFieldsMetadataServiceSetupMock, + createFieldsMetadataServiceStartMock, +} from './services/fields_metadata/fields_metadata_service.mock'; +import { FieldsMetadataPluginSetup, FieldsMetadataPluginStart } from './types'; + +const createFieldsMetadataSetupMock = () => { + const fieldsMetadataSetupMock: jest.Mocked = { + fieldsMetadata: createFieldsMetadataServiceSetupMock(), + }; + + return fieldsMetadataSetupMock; +}; + +const createFieldsMetadataStartMock = () => { + const fieldsMetadataStartMock: jest.Mocked = { + fieldsMetadata: createFieldsMetadataServiceStartMock(), + }; + return fieldsMetadataStartMock; +}; + +export const fieldsMetadataPluginMock = { + createSetupContract: createFieldsMetadataSetupMock, + createStartContract: createFieldsMetadataStartMock, +}; diff --git a/x-pack/plugins/fields_metadata/server/plugin.ts b/x-pack/plugins/fields_metadata/server/plugin.ts new file mode 100644 index 0000000000000..b28ec952de016 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/plugin.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext, CoreStart, Plugin, Logger } from '@kbn/core/server'; + +import { + FieldsMetadataPluginCoreSetup, + FieldsMetadataPluginSetup, + FieldsMetadataPluginStart, + FieldsMetadataServerPluginSetupDeps, + FieldsMetadataServerPluginStartDeps, +} from './types'; +import { initFieldsMetadataServer } from './fields_metadata_server'; +import { FieldsMetadataService } from './services/fields_metadata'; +import { FieldsMetadataBackendLibs } from './lib/shared_types'; + +export class FieldsMetadataPlugin + implements + Plugin< + FieldsMetadataPluginSetup, + FieldsMetadataPluginStart, + FieldsMetadataServerPluginSetupDeps, + FieldsMetadataServerPluginStartDeps + > +{ + private readonly logger: Logger; + private libs!: FieldsMetadataBackendLibs; + private fieldsMetadataService: FieldsMetadataService; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + + this.fieldsMetadataService = new FieldsMetadataService(this.logger); + } + + public setup(core: FieldsMetadataPluginCoreSetup, plugins: FieldsMetadataServerPluginSetupDeps) { + const fieldsMetadata = this.fieldsMetadataService.setup(); + + this.libs = { + getStartServices: () => core.getStartServices(), + logger: this.logger, + plugins, + router: core.http.createRouter(), + }; + + // Register server side APIs + initFieldsMetadataServer(this.libs); + + return { + fieldsMetadata, + }; + } + + public start(core: CoreStart, plugins: FieldsMetadataServerPluginStartDeps) { + const fieldsMetadata = this.fieldsMetadataService.start(); + + return { fieldsMetadata }; + } + + public stop() {} +} diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts new file mode 100644 index 0000000000000..167390fd3be7c --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createValidationFunction } from '../../../common/runtime_types'; +import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; +import * as fieldsMetadataV1 from '../../../common/fields_metadata/v1'; +import { FieldsMetadataBackendLibs } from '../../lib/shared_types'; + +export const initFindFieldsMetadataRoute = ({ + router, + getStartServices, +}: FieldsMetadataBackendLibs) => { + router.versioned + .get({ + access: 'internal', + path: FIND_FIELDS_METADATA_URL, + }) + .addVersion( + { + version: '1', + validate: { + request: { + query: createValidationFunction(fieldsMetadataV1.findFieldsMetadataRequestQueryRT), + }, + }, + }, + async (_requestContext, request, response) => { + const { fieldNames } = request.query; + + const { fieldsMetadata } = (await getStartServices())[2]; + const fieldsMetadataClient = fieldsMetadata.getClient(); + + try { + const fields = fieldsMetadataClient.find({ fieldNames }); + + return response.ok({ + body: fieldsMetadataV1.findFieldsMetadataResponsePayloadRT.encode({ fields }), + }); + } catch (error) { + return response.customError({ + statusCode: error.statusCode ?? 500, + body: { + message: error.message ?? 'An unexpected error occurred', + }, + }); + } + } + ); +}; diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/index.ts new file mode 100644 index 0000000000000..df72fb7c02e32 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { FieldsMetadataBackendLibs } from '../../lib/shared_types'; +import { initFindFieldsMetadataRoute } from './find_fields_metadata'; + +export const initFieldsMetadataRoutes = (libs: FieldsMetadataBackendLibs) => { + initFindFieldsMetadataRoute(libs); +}; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.mock.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.mock.ts new file mode 100644 index 0000000000000..3f5b6dac385bb --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.mock.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IFieldsMetadataClient } from './types'; + +export const createFieldsMetadataClientMock = (): jest.Mocked => ({ + getByName: jest.fn(), + find: jest.fn(), +}); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts new file mode 100644 index 0000000000000..b13f9fa973167 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldsMetadataClient } from './fields_metadata_client'; + +describe('FieldsMetadataClient class', () => { + const fieldsMetadataClient = FieldsMetadataClient.create(); + + it('#getByName resolves a single ecs field', () => { + const timestampField = fieldsMetadataClient.getByName('@timestamp'); + + expect(timestampField.hasOwnProperty('dashed_name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('description')).toBeTruthy(); + expect(timestampField.hasOwnProperty('example')).toBeTruthy(); + expect(timestampField.hasOwnProperty('flat_name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('level')).toBeTruthy(); + expect(timestampField.hasOwnProperty('name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('normalize')).toBeTruthy(); + expect(timestampField.hasOwnProperty('required')).toBeTruthy(); + expect(timestampField.hasOwnProperty('short')).toBeTruthy(); + expect(timestampField.hasOwnProperty('type')).toBeTruthy(); + }); + + it('#find resolves a dictionary of matching fields', async () => { + const fields = fieldsMetadataClient.find({ + fieldNames: ['@timestamp', 'message', 'not-existing-field'], + }); + + expect(fields.hasOwnProperty('@timestamp')).toBeTruthy(); + expect(fields.hasOwnProperty('message')).toBeTruthy(); + expect(fields.hasOwnProperty('not-existing-field')).toBeFalsy(); + }); +}); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts new file mode 100644 index 0000000000000..6b4b232fcca53 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EcsFlat } from '@elastic/ecs'; +import { Logger } from '@kbn/core/server'; +import { FieldName, FieldMetadata, EcsFieldMetadata } from '../../../common'; +import { EcsFieldsSourceClient } from './source_clients/ecs_fields_source_client'; +import { IntegrationsFieldsSourceClient } from './source_clients/integration_fields_source_client'; +import { IFieldsMetadataClient } from './types'; + +interface FieldsMetadataClientDeps { + logger: Logger; + ecsFieldsSourceClient: EcsFieldsSourceClient; + integrationFieldsSourceClient: IntegrationsFieldsSourceClient; +} + +export class FieldsMetadataClient implements IFieldsMetadataClient { + private constructor( + private readonly logger: Logger, + private readonly ecsFieldsSourceClient: EcsFieldsSourceClient, + private readonly integrationFieldsSourceClient: IntegrationsFieldsSourceClient + ) {} + + getByName(fieldName: TFieldName): FieldMetadata | undefined { + this.logger.debug(`Retrieving field metadata for: ${fieldName}`); + + const field = this.ecsFieldsSourceClient.getByName(fieldName); + + // TODO: enable resolution for integration field + // if (!field) { + // field = this.integrationFieldsSourceClient.getByName(fieldName); + // } + + return field; + } + + find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { + if (!fieldNames) { + return EcsFlat; + } + + const res = fieldNames.reduce((fieldsMetadata, fieldName) => { + const field = this.getByName(fieldName); + + if (field) { + fieldsMetadata[fieldName] = field; + } + + return fieldsMetadata; + }, {} as Record); + + return res; + } + + public static create({ + logger, + ecsFieldsSourceClient, + integrationFieldsSourceClient, + }: FieldsMetadataClientDeps) { + return new FieldsMetadataClient(logger, ecsFieldsSourceClient, integrationFieldsSourceClient); + } +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts new file mode 100644 index 0000000000000..631019c64f967 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createFieldsMetadataClientMock } from './fields_metadata_client.mock'; +import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; + +export const createFieldsMetadataServiceSetupMock = + (): jest.Mocked => ({}); + +export const createFieldsMetadataServiceStartMock = + (): jest.Mocked => ({ + getClient: jest.fn(() => createFieldsMetadataClientMock()), + }); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts new file mode 100644 index 0000000000000..4dfa596c38155 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EcsFlat as ecsFields } from '@elastic/ecs'; +import { Logger } from '@kbn/core/server'; +import { FieldsMetadataClient } from './fields_metadata_client'; +import { EcsFieldsSourceClient } from './source_clients/ecs_fields_source_client'; +import { IntegrationsFieldsSourceClient } from './source_clients/integration_fields_source_client'; +import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; + +export class FieldsMetadataService { + private packageService: any; // TODO: update types + + constructor(private readonly logger: Logger) {} + + public setup(): FieldsMetadataServiceSetup { + return { + registerPackageService: (packageService) => { + this.packageService = packageService; + }, + }; + } + + public start(): FieldsMetadataServiceStart { + const { logger, packageService } = this; + + const ecsFieldsSourceClient = EcsFieldsSourceClient.create({ ecsFields }); + const integrationFieldsSourceClient = IntegrationsFieldsSourceClient.create({ packageService }); + + return { + getClient() { + return FieldsMetadataClient.create({ + logger, + ecsFieldsSourceClient, + integrationFieldsSourceClient, + }); + }, + }; + } +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/index.ts new file mode 100644 index 0000000000000..51f5348b1be93 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { FieldsMetadataService } from './fields_metadata_service'; +export { FieldsMetadataClient } from './fields_metadata_client'; +export type { + FieldsMetadataServiceSetup, + FieldsMetadataServiceStart, + FieldsMetadataServiceStartDeps, +} from './types'; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts new file mode 100644 index 0000000000000..33d22a27909b3 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EcsFieldMetadata, + EcsFieldName, + FieldMetadata, + FieldName, + TEcsFields, +} from '../../../../common'; +import { ISourceClient } from './types'; + +interface EcsFieldsSourceClientDeps { + ecsFields: TEcsFields; +} + +export class EcsFieldsSourceClient implements ISourceClient { + private constructor(private readonly ecsFields: TEcsFields) {} + + getByName(fieldName: TFieldName): EcsFieldMetadata | undefined { + return fieldName in this.ecsFields ? this.ecsFields[fieldName as EcsFieldName] : undefined; + } + + find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { + if (!fieldNames) { + return this.ecsFields; + } + + const res = fieldNames.reduce((fieldsMetadata, fieldName) => { + const field = this.getByName(fieldName); + + if (field) { + fieldsMetadata[fieldName] = field; + } + + return fieldsMetadata; + }, {} as Record); + + return res; + } + + public static create({ ecsFields }: EcsFieldsSourceClientDeps) { + return new EcsFieldsSourceClient(ecsFields); + } +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts new file mode 100644 index 0000000000000..6ef89bc7a9cae --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldMetadata, FieldName } from '../../../../common'; +import { ISourceClient } from './types'; + +interface IntegrationsFieldsSourceClientDeps { + packageService: PackageService; +} + +export class IntegrationsFieldsSourceClient implements ISourceClient { + private constructor(private readonly packageClient: PackageClient) {} + + getByName(fieldName: TFieldName) { + throw new Error('TODO: Implement the IntegrationsFieldsSourceClient#getByName'); + } + + find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { + throw new Error('TODO: Implement the IntegrationsFieldsSourceClient#getByName'); + } + + public static create({ packageService }: IntegrationsFieldsSourceClientDeps) { + if (!packageService) { + return { + getByName: () => undefined, + find: () => ({}), + }; + } + + const packageClient = packageService.asInternalUser; + + return new IntegrationsFieldsSourceClient(packageClient); + } +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts new file mode 100644 index 0000000000000..e056c4acb0774 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldMetadata, FieldName } from '../../../../common'; + +export interface ISourceClient { + getByName(fieldName: TFieldName): FieldMetadata | undefined; + find(params: { + fieldNames?: TFieldName[]; + }): Record; +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts new file mode 100644 index 0000000000000..d8900cf9c0817 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FieldName, FieldMetadata } from '../../../common'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataServiceStartDeps {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataServiceSetup {} + +export interface FieldsMetadataServiceStart { + getClient(): IFieldsMetadataClient; +} + +export interface IFieldsMetadataClient { + getByName(fieldName: FieldName): FieldMetadata | undefined; + find(params: { + fieldNames?: TFieldName[]; + }): Record; +} diff --git a/x-pack/plugins/fields_metadata/server/types.ts b/x-pack/plugins/fields_metadata/server/types.ts new file mode 100644 index 0000000000000..c7ffc26598640 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/types.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup } from '@kbn/core/server'; + +import { + FieldsMetadataServiceSetup, + FieldsMetadataServiceStart, +} from './services/fields_metadata/types'; + +export type FieldsMetadataPluginCoreSetup = CoreSetup< + FieldsMetadataServerPluginStartDeps, + FieldsMetadataPluginStart +>; +export type FieldsMetadataPluginStartServicesAccessor = + FieldsMetadataPluginCoreSetup['getStartServices']; + +export interface FieldsMetadataPluginSetup { + fieldsMetadata: FieldsMetadataServiceSetup; +} + +export interface FieldsMetadataPluginStart { + fieldsMetadata: FieldsMetadataServiceStart; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataServerPluginSetupDeps {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldsMetadataServerPluginStartDeps {} diff --git a/x-pack/plugins/fields_metadata/tsconfig.json b/x-pack/plugins/fields_metadata/tsconfig.json new file mode 100644 index 0000000000000..aaa085d9c2401 --- /dev/null +++ b/x-pack/plugins/fields_metadata/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "public/**/*", + "server/**/*", + "types/**/*" + ], + "exclude": ["target/**/*"], + "kbn_references": [] +} From cfcada83f49c8306b6256396e53ec1abb689c16d Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 08:44:31 +0200 Subject: [PATCH 02/50] feat(io-ts-utils): add arrayToString util --- packages/kbn-io-ts-utils/index.ts | 1 + .../src/array_to_string_rt/index.test.ts | 59 +++++++++++++++++++ .../src/array_to_string_rt/index.ts | 24 ++++++++ 3 files changed, 84 insertions(+) create mode 100644 packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts create mode 100644 packages/kbn-io-ts-utils/src/array_to_string_rt/index.ts diff --git a/packages/kbn-io-ts-utils/index.ts b/packages/kbn-io-ts-utils/index.ts index 2a6c02f6cdf17..786d39ad6a3d6 100644 --- a/packages/kbn-io-ts-utils/index.ts +++ b/packages/kbn-io-ts-utils/index.ts @@ -9,6 +9,7 @@ export type { IndexPattern } from './src/index_pattern_rt'; export type { NonEmptyString, NonEmptyStringBrand } from './src/non_empty_string_rt'; +export { arrayToStringRt } from './src/array_to_string_rt'; export { deepExactRt } from './src/deep_exact_rt'; export { indexPatternRt } from './src/index_pattern_rt'; export { jsonRt } from './src/json_rt'; diff --git a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts new file mode 100644 index 0000000000000..a2d0e1f62e864 --- /dev/null +++ b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as rt from 'io-ts'; +import { arrayToStringRt } from '.'; +import { isRight, Either, isLeft, fold } from 'fp-ts/lib/Either'; +import { Right } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { identity } from 'fp-ts/lib/function'; + +function getValueOrThrow>(either: TEither): Right { + const value = pipe( + either, + fold(() => { + throw new Error('cannot get right value of left'); + }, identity) + ); + + return value as Right; +} + +describe('arrayToStringRt', () => { + it('should validate strings', () => { + expect(isRight(arrayToStringRt.decode(''))).toBe(true); + expect(isRight(arrayToStringRt.decode('message'))).toBe(true); + expect(isRight(arrayToStringRt.decode('message,event.original'))).toBe(true); + expect(isLeft(arrayToStringRt.decode({}))).toBe(true); + expect(isLeft(arrayToStringRt.decode(true))).toBe(true); + }); + + it('should return array of strings when decoding', () => { + expect(getValueOrThrow(arrayToStringRt.decode(''))).toEqual(['']); + expect(getValueOrThrow(arrayToStringRt.decode('message'))).toEqual(['message']); + expect(getValueOrThrow(arrayToStringRt.decode('message,event.original'))).toEqual([ + 'message', + 'event.original', + ]); + }); + + it('should be pipable', () => { + const piped = arrayToStringRt.pipe(rt.array(rt.string)); + + const validInput = ['message', 'event.original']; + const invalidInput = {}; + + const valid = piped.decode(validInput.join(',')); + const invalid = piped.decode(invalidInput); + + expect(isRight(valid)).toBe(true); + expect(getValueOrThrow(valid)).toEqual(validInput); + + expect(isLeft(invalid)).toBe(true); + }); +}); diff --git a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.ts b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.ts new file mode 100644 index 0000000000000..98c1af8b6ba09 --- /dev/null +++ b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { either } from 'fp-ts/lib/Either'; +import * as rt from 'io-ts'; + +export const arrayToStringRt = new rt.Type( + 'arrayToString', + rt.array(rt.unknown).is, + (input, context) => + either.chain(rt.string.validate(input, context), (str) => { + try { + return rt.success(str.split(',')); + } catch (e) { + return rt.failure(input, context); + } + }), + (arr) => arr.join(',') +); From 93fb2fbc7becba975302eeef1d8faafb411b441b Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 08:48:23 +0200 Subject: [PATCH 03/50] chore(fields-metadata): update plugin aliases --- package.json | 1 + tsconfig.base.json | 2 ++ 2 files changed, 3 insertions(+) diff --git a/package.json b/package.json index 04316640a2952..ec61fb813744b 100644 --- a/package.json +++ b/package.json @@ -484,6 +484,7 @@ "@kbn/field-formats-plugin": "link:src/plugins/field_formats", "@kbn/field-types": "link:packages/kbn-field-types", "@kbn/field-utils": "link:packages/kbn-field-utils", + "@kbn/fields-metadata-plugin": "link:x-pack/plugins/fields_metadata", "@kbn/file-upload-plugin": "link:x-pack/plugins/file_upload", "@kbn/files-example-plugin": "link:examples/files_example", "@kbn/files-management-plugin": "link:src/plugins/files_management", diff --git a/tsconfig.base.json b/tsconfig.base.json index dd82d55b1b0ad..94e42fdf56f5f 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -884,6 +884,8 @@ "@kbn/field-types/*": ["packages/kbn-field-types/*"], "@kbn/field-utils": ["packages/kbn-field-utils"], "@kbn/field-utils/*": ["packages/kbn-field-utils/*"], + "@kbn/fields-metadata-plugin": ["x-pack/plugins/fields_metadata"], + "@kbn/fields-metadata-plugin/*": ["x-pack/plugins/fields_metadata/*"], "@kbn/file-upload-plugin": ["x-pack/plugins/file_upload"], "@kbn/file-upload-plugin/*": ["x-pack/plugins/file_upload/*"], "@kbn/files-example-plugin": ["examples/files_example"], From 3f59e8f6fde781365e6d87eb19a6c6872b47bb5f Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 09:09:28 +0200 Subject: [PATCH 04/50] chore(fields-metadata): add code ownership --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d160dda238f8d..bad6f25619196 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -534,6 +534,7 @@ examples/locator_examples @elastic/appex-sharedux examples/locator_explorer @elastic/appex-sharedux packages/kbn-logging @elastic/kibana-core packages/kbn-logging-mocks @elastic/kibana-core +x-pack/plugins/fields_metadata @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_data_access @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_explorer @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_shared @elastic/obs-ux-logs-team From d46a0dd626872c3fc808c268153160e60a5a85b4 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 10:13:34 +0200 Subject: [PATCH 05/50] refactor(unified-doc-viewer): use fieldsMetadata for flyout descriptions --- src/plugins/unified_doc_viewer/kibana.jsonc | 1 + .../logs_overview_highlights.tsx | 38 +++++++++++++++++++ .../sub_components/highlight_field.tsx | 5 ++- .../highlight_field_description.tsx | 8 ++-- .../unified_doc_viewer/public/plugin.tsx | 5 ++- .../unified_doc_viewer/public/types.ts | 2 + .../common/fields_metadata/types.ts | 1 + .../use_fields_metadata.ts | 25 +++++++++--- .../plugins/fields_metadata/public/index.ts | 4 +- .../plugins/fields_metadata/public/mocks.tsx | 6 +-- .../plugins/fields_metadata/public/types.ts | 15 ++++---- yarn.lock | 4 ++ 12 files changed, 91 insertions(+), 23 deletions(-) diff --git a/src/plugins/unified_doc_viewer/kibana.jsonc b/src/plugins/unified_doc_viewer/kibana.jsonc index c5ab9f0c4e632..2361a10120e9b 100644 --- a/src/plugins/unified_doc_viewer/kibana.jsonc +++ b/src/plugins/unified_doc_viewer/kibana.jsonc @@ -9,5 +9,6 @@ "browser": true, "requiredBundles": ["kibanaUtils"], "requiredPlugins": ["data", "discoverShared", "fieldFormats"], + "optionalPlugins": ["fieldsMetadata"] } } diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx index a7d998b0f1eba..de68a4cfa67c4 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx @@ -13,6 +13,7 @@ import { i18n } from '@kbn/i18n'; import { DataTableRecord, LogDocumentOverview, fieldConstants } from '@kbn/discover-utils'; import { HighlightField } from './sub_components/highlight_field'; import { HighlightSection } from './sub_components/highlight_section'; +import { getUnifiedDocViewerServices } from '../../plugin'; export function LogsOverviewHighlights({ formattedDoc, @@ -21,6 +22,29 @@ export function LogsOverviewHighlights({ formattedDoc: LogDocumentOverview; flattenedDoc: DataTableRecord['flattened']; }) { + const { + fieldsMetadata: { useFieldsMetadata }, + } = getUnifiedDocViewerServices(); + + const { fieldsMetadata } = useFieldsMetadata({ + fieldNames: [ + fieldConstants.SERVICE_NAME_FIELD, + fieldConstants.HOST_NAME_FIELD, + fieldConstants.TRACE_ID_FIELD, + fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD, + fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD, + fieldConstants.CLOUD_PROVIDER_FIELD, + fieldConstants.CLOUD_REGION_FIELD, + fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD, + fieldConstants.CLOUD_PROJECT_ID_FIELD, + fieldConstants.CLOUD_INSTANCE_ID_FIELD, + fieldConstants.LOG_FILE_PATH_FIELD, + fieldConstants.DATASTREAM_DATASET_FIELD, + fieldConstants.DATASTREAM_NAMESPACE_FIELD, + fieldConstants.AGENT_NAME_FIELD, + ], + }); + const getHighlightProps = (field: keyof LogDocumentOverview) => ({ field, formattedValue: formattedDoc[field], @@ -38,6 +62,7 @@ export function LogsOverviewHighlights({ )} @@ -45,6 +70,7 @@ export function LogsOverviewHighlights({ )} @@ -52,6 +78,7 @@ export function LogsOverviewHighlights({ )} @@ -59,6 +86,7 @@ export function LogsOverviewHighlights({ )} @@ -66,6 +94,7 @@ export function LogsOverviewHighlights({ )} @@ -79,6 +108,7 @@ export function LogsOverviewHighlights({ )} @@ -100,6 +131,7 @@ export function LogsOverviewHighlights({ )} @@ -107,6 +139,7 @@ export function LogsOverviewHighlights({ )} @@ -114,6 +147,7 @@ export function LogsOverviewHighlights({ )} @@ -127,6 +161,7 @@ export function LogsOverviewHighlights({ )} @@ -134,6 +169,7 @@ export function LogsOverviewHighlights({ )} @@ -141,6 +177,7 @@ export function LogsOverviewHighlights({ @@ -149,6 +186,7 @@ export function LogsOverviewHighlights({ )} diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx index 176f8cf2b2a77..5b1472ce9db37 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx @@ -11,12 +11,14 @@ import { css } from '@emotion/react'; import React, { ReactNode } from 'react'; import { dynamic } from '@kbn/shared-ux-utility'; import { euiThemeVars } from '@kbn/ui-theme'; +import { FieldMetadata } from '@kbn/fields-metadata-plugin/common'; import { HoverActionPopover } from './hover_popover_action'; const HighlightFieldDescription = dynamic(() => import('./highlight_field_description')); interface HighlightFieldProps { field: string; + fieldMetadata?: FieldMetadata; formattedValue?: string; icon?: ReactNode; label: string; @@ -26,6 +28,7 @@ interface HighlightFieldProps { export function HighlightField({ field, + fieldMetadata, formattedValue, icon, label, @@ -39,7 +42,7 @@ export function HighlightField({ {label} - + UseFieldsMetadataReturnType; + export const createUseFieldsMetadataHook = ({ fieldsMetadataClient, -}: UseFieldsMetadataFactoryDeps) => { - return ({ fieldNames }: Params) => { - const [{ error, loading, value: fieldsMetadata }, load] = useAsyncFn( +}: UseFieldsMetadataFactoryDeps): UseFieldsMetadataHook => { + return ({ fieldNames }) => { + const serializedFieldNames = JSON.stringify(fieldNames); + + const [{ error, loading, value }, load] = useAsyncFn( () => fieldsMetadataClient.find({ fieldNames }), - [fieldNames] + [serializedFieldNames] ); useEffect(() => { load(); }, [load]); - return { fieldsMetadata, loading, error }; + return { fieldsMetadata: value?.fields, loading, error }; }; }; diff --git a/x-pack/plugins/fields_metadata/public/index.ts b/x-pack/plugins/fields_metadata/public/index.ts index e05935884742f..d22aa71e214dc 100644 --- a/x-pack/plugins/fields_metadata/public/index.ts +++ b/x-pack/plugins/fields_metadata/public/index.ts @@ -8,8 +8,8 @@ import { FieldsMetadataPlugin } from './plugin'; export type { - FieldsMetadataClientSetupExports, - FieldsMetadataClientStartExports, + FieldsMetadataClientSetup, + FieldsMetadataClientStart, FieldsMetadataClientSetupDeps, FieldsMetadataClientStartDeps, } from './types'; diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index 151e36eb7d178..4d6b542cf5c00 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -6,12 +6,12 @@ */ import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; -import { FieldsMetadataClientStartExports } from './types'; +import { FieldsMetadataClientStart } from './types'; export const createFieldsMetadataPluginStartMock = - (): jest.Mocked => ({ + (): jest.Mocked => ({ fieldsMetadata: createFieldsMetadataServiceStartMock(), }); -export const _ensureTypeCompatibility = (): FieldsMetadataClientStartExports => +export const _ensureTypeCompatibility = (): FieldsMetadataClientStart => createFieldsMetadataPluginStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/types.ts b/x-pack/plugins/fields_metadata/public/types.ts index a3f0639690764..cb7ae3f493291 100644 --- a/x-pack/plugins/fields_metadata/public/types.ts +++ b/x-pack/plugins/fields_metadata/public/types.ts @@ -6,14 +6,15 @@ */ import type { CoreSetup, CoreStart, Plugin as PluginClass } from '@kbn/core/public'; - -import { IFieldsMetadataClient } from './services/fields_metadata'; +import type { UseFieldsMetadataHook } from './hooks/use_fields_metadata/use_fields_metadata'; +import type { IFieldsMetadataClient } from './services/fields_metadata'; // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FieldsMetadataClientSetupExports {} +export interface FieldsMetadataClientSetup {} -export interface FieldsMetadataClientStartExports { +export interface FieldsMetadataClientStart { client: IFieldsMetadataClient; + useFieldsMetadata: UseFieldsMetadataHook; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -24,12 +25,12 @@ export interface FieldsMetadataClientStartDeps {} export type FieldsMetadataClientCoreSetup = CoreSetup< FieldsMetadataClientStartDeps, - FieldsMetadataClientStartExports + FieldsMetadataClientStart >; export type FieldsMetadataClientCoreStart = CoreStart; export type FieldsMetadataClientPluginClass = PluginClass< - FieldsMetadataClientSetupExports, - FieldsMetadataClientStartExports, + FieldsMetadataClientSetup, + FieldsMetadataClientStart, FieldsMetadataClientSetupDeps, FieldsMetadataClientStartDeps >; diff --git a/yarn.lock b/yarn.lock index 63f50a8f16e93..3babf3691de8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4803,6 +4803,10 @@ version "0.0.0" uid "" +"@kbn/fields-metadata-plugin@link:x-pack/plugins/fields_metadata": + version "0.0.0" + uid "" + "@kbn/file-upload-plugin@link:x-pack/plugins/file_upload": version "0.0.0" uid "" From 89ebdfb5b19b546d141f4afa167c29a21cf2a896 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 10:28:58 +0200 Subject: [PATCH 06/50] feat(fields-metadata): add client service caching layer --- .../fields_metadata/common/hashed_cache.ts | 41 +++++++++++++++++++ .../fields_metadata/fields_metadata_client.ts | 27 ++++++++---- 2 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/fields_metadata/common/hashed_cache.ts diff --git a/x-pack/plugins/fields_metadata/common/hashed_cache.ts b/x-pack/plugins/fields_metadata/common/hashed_cache.ts new file mode 100644 index 0000000000000..4b5ac5c614472 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/hashed_cache.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import LRUCache from 'lru-cache'; +import hash from 'object-hash'; +export interface IHashedCache { + get(key: KeyType): ValueType | undefined; + set(key: KeyType, value: ValueType): boolean; + has(key: KeyType): boolean; + reset(): void; +} + +export class HashedCache { + private cache: LRUCache; + + constructor(options: LRUCache.Options = { max: 500 }) { + this.cache = new LRUCache(options); + } + + public get(key: KeyType): ValueType | undefined { + const serializedKey = hash(key); + return this.cache.get(serializedKey); + } + + public set(key: KeyType, value: ValueType) { + const serializedKey = hash(key); + return this.cache.set(serializedKey, value); + } + + public has(key: KeyType): boolean { + const serializedKey = hash(key); + return this.cache.has(serializedKey); + } + + public reset() { + return this.cache.reset(); + } +} diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts index a8384c23a0ece..bf76e14542791 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -6,6 +6,7 @@ */ import { HttpStart } from '@kbn/core/public'; +import { HashedCache } from '../../../common/hashed_cache'; import { FindFieldsMetadataRequestQuery, findFieldsMetadataRequestQueryRT, @@ -17,25 +18,37 @@ import { decodeOrThrow } from '../../../common/runtime_types'; import { IFieldsMetadataClient } from './types'; export class FieldsMetadataClient implements IFieldsMetadataClient { - constructor(private readonly http: HttpStart) {} + private cache: HashedCache; - public async find({ - fieldNames, - }: FindFieldsMetadataRequestQuery): Promise { - const query = findFieldsMetadataRequestQueryRT.encode({ fieldNames }); + constructor(private readonly http: HttpStart) { + this.cache = new HashedCache(); + } + + public async find( + params: FindFieldsMetadataRequestQuery + ): Promise { + // Initially lookup for existing results given request parameters + if (this.cache.has(params)) { + return this.cache.get(params) as FindFieldsMetadataResponsePayload; + } + + const query = findFieldsMetadataRequestQueryRT.encode(params); const response = await this.http .get(FIND_FIELDS_METADATA_URL, { query, version: '1' }) .catch((error) => { - throw new Error(`Failed to fetch ecs fields ${fieldNames?.join() ?? ''}: ${error}`); + throw new Error(`Failed to fetch ecs fields ${params.fieldNames?.join() ?? ''}: ${error}`); }); const data = decodeOrThrow( findFieldsMetadataResponsePayloadRT, (message: string) => - new Error(`Failed to decode ecs fields ${fieldNames?.join() ?? ''}: ${message}"`) + new Error(`Failed to decode ecs fields ${params.fieldNames?.join() ?? ''}: ${message}"`) )(response); + // Store cached results for given request parameters + this.cache.set(params, data); + return data; } } From c5c8726bd2a153e891dafb9963faacf9a297aa19 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 18:10:24 +0200 Subject: [PATCH 07/50] feat(fields-metadata): add attributes filtering --- .../logs_overview_highlights.tsx | 1 + .../sub_components/highlight_field.tsx | 4 +- .../highlight_field_description.tsx | 8 +- .../fields_metadata/models/field_metadata.ts | 40 ++++++++ .../models/fields_metadata_dictionary.ts | 28 ++++++ .../common/fields_metadata/types.ts | 98 ++++++++++--------- .../v1/find_fields_metadata.ts | 5 +- .../plugins/fields_metadata/common/index.ts | 11 ++- .../use_fields_metadata.ts | 14 +-- .../fields_metadata/find_fields_metadata.ts | 15 ++- .../fields_metadata/fields_metadata_client.ts | 28 +++--- .../fields_metadata_service.ts | 8 +- .../repositories/ecs_fields_repository.ts | 53 ++++++++++ .../integration_fields_repository.ts} | 4 +- .../{source_clients => repositories}/types.ts | 9 +- .../ecs_fields_source_client.ts | 49 ---------- .../server/services/fields_metadata/types.ts | 6 +- 17 files changed, 240 insertions(+), 141 deletions(-) create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts rename x-pack/plugins/fields_metadata/server/services/fields_metadata/{source_clients/integration_fields_source_client.ts => repositories/integration_fields_repository.ts} (90%) rename x-pack/plugins/fields_metadata/server/services/fields_metadata/{source_clients => repositories}/types.ts (52%) delete mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx index de68a4cfa67c4..6adc3a9b7c07a 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx @@ -27,6 +27,7 @@ export function LogsOverviewHighlights({ } = getUnifiedDocViewerServices(); const { fieldsMetadata } = useFieldsMetadata({ + attributes: ['flat_name', 'short', 'type'], fieldNames: [ fieldConstants.SERVICE_NAME_FIELD, fieldConstants.HOST_NAME_FIELD, diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx index 5b1472ce9db37..ddc8995c52952 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx @@ -11,14 +11,14 @@ import { css } from '@emotion/react'; import React, { ReactNode } from 'react'; import { dynamic } from '@kbn/shared-ux-utility'; import { euiThemeVars } from '@kbn/ui-theme'; -import { FieldMetadata } from '@kbn/fields-metadata-plugin/common'; +import { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; import { HoverActionPopover } from './hover_popover_action'; const HighlightFieldDescription = dynamic(() => import('./highlight_field_description')); interface HighlightFieldProps { field: string; - fieldMetadata?: FieldMetadata; + fieldMetadata?: PartialFieldMetadataPlain; formattedValue?: string; icon?: ReactNode; label: string; diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field_description.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field_description.tsx index df867cab8c44c..dbc02e04e43c5 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field_description.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field_description.tsx @@ -7,11 +7,15 @@ */ import { EuiFlexGroup, EuiIconTip } from '@elastic/eui'; -import { FieldMetadata } from '@kbn/fields-metadata-plugin/common'; +import { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; import { FieldIcon } from '@kbn/react-field'; import React from 'react'; -export function HighlightFieldDescription({ fieldMetadata }: { fieldMetadata?: FieldMetadata }) { +export function HighlightFieldDescription({ + fieldMetadata, +}: { + fieldMetadata?: PartialFieldMetadataPlain; +}) { if (!fieldMetadata) return null; const { flat_name: fieldName, short, type } = fieldMetadata; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts new file mode 100644 index 0000000000000..0fe3a7236190f --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pick from 'lodash/pick'; +import { FieldAttribute, FieldMetadataPlain } from '../types'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface FieldMetadata extends FieldMetadataPlain {} +export class FieldMetadata { + private constructor(fieldMetadata: FieldMetadataPlain) { + Object.assign(this, fieldMetadata); + } + + public pick(props: FieldAttribute[]) { + return pick(this, props); + } + + public toPlain() { + return Object.assign({}, this); + } + + public static create(fieldMetadata: FieldMetadataPlain) { + const fieldMetadataProps = { + ...fieldMetadata, + dashed_name: fieldMetadata.dashed_name ?? FieldMetadata.toDashedName(fieldMetadata.flat_name), + normalize: fieldMetadata.normalize ?? [], + short: fieldMetadata.short ?? fieldMetadata.description, + }; + + return new FieldMetadata(fieldMetadataProps); + } + + private static toDashedName(flatName: string) { + return flatName.split('.').join('-'); + } +} diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts new file mode 100644 index 0000000000000..b7999b0586392 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import mapValues from 'lodash/mapValues'; +import { FieldAttribute, FieldMetadataPlain, PartialFieldMetadataPlain } from '../types'; +import { FieldMetadata } from './field_metadata'; + +export type PartialFieldsMetadataMap = Record; + +export class FieldsMetadataDictionary { + private constructor(private readonly fields: PartialFieldsMetadataMap) {} + + pick(attributes: FieldAttribute[]): Record { + return mapValues(this.fields, (field) => field?.pick(attributes)); + } + + toPlain(): Record { + return mapValues(this.fields, (field) => field?.toPlain()); + } + + public static create(fields: PartialFieldsMetadataMap = {}) { + return new FieldsMetadataDictionary(fields); + } +} diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index 6e5c6ebc13194..3347f8ddeb1f3 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -25,59 +25,65 @@ export const multiFieldRT = rt.type({ type: rt.string, }); -export const ecsFieldMetadataRT = rt.intersection([ - rt.type({ - dashed_name: rt.string, - description: rt.string, - flat_name: rt.string, - level: rt.string, - name: rt.string, - normalize: rt.array(rt.string), - short: rt.string, - type: rt.string, - }), - rt.partial({ - allowed_values: allowedValueRT, - beta: rt.string, - doc_values: rt.boolean, - example: rt.unknown, - expected_values: rt.array(rt.string), - format: rt.string, - ignore_above: rt.number, - index: rt.boolean, - input_format: rt.string, - multi_fields: rt.array(multiFieldRT), - object_type: rt.string, - original_fieldset: rt.string, - output_format: rt.string, - output_precision: rt.number, - pattern: rt.string, - required: rt.boolean, - scaling_factor: rt.number, - }), +const requiredBaseMetadataPlainRT = rt.type({ + description: rt.string, + flat_name: rt.string, + name: rt.string, + type: rt.string, +}); + +const optionalBaseMetadataPlainRT = rt.partial({ + description: rt.string, + flat_name: rt.string, + name: rt.string, + type: rt.string, +}); + +const optionalMetadataPlainRT = rt.partial({ + allowed_values: rt.array(allowedValueRT), + beta: rt.string, + dashed_name: rt.string, + doc_values: rt.boolean, + example: rt.unknown, + expected_values: rt.array(rt.string), + format: rt.string, + ignore_above: rt.number, + index: rt.boolean, + input_format: rt.string, + level: rt.string, + multi_fields: rt.array(multiFieldRT), + normalize: rt.array(rt.string), + object_type: rt.string, + original_fieldset: rt.string, + output_format: rt.string, + output_precision: rt.number, + pattern: rt.string, + required: rt.boolean, + scaling_factor: rt.number, + short: rt.string, +}); + +export const partialFieldMetadataPlainRT = rt.intersection([ + optionalBaseMetadataPlainRT, + optionalMetadataPlainRT, ]); -export const integrationFieldMetadataRT = rt.intersection([ - rt.type({ - name: rt.string, - description: rt.string, - type: rt.string, - flat_name: rt.string, - short: rt.string, - }), - rt.partial({ - example: rt.unknown, - }), +export const fieldMetadataPlainRT = rt.intersection([ + requiredBaseMetadataPlainRT, + optionalMetadataPlainRT, ]); -export const fieldMetadataRT = rt.union([ecsFieldMetadataRT, integrationFieldMetadataRT]); +export const fieldAttributeRT = rt.union([ + rt.keyof(requiredBaseMetadataPlainRT.props), + rt.keyof(optionalMetadataPlainRT.props), +]); export type TEcsFields = typeof EcsFlat; export type EcsFieldName = keyof TEcsFields; export type IntegrationFieldName = string; -export type EcsFieldMetadata = TEcsFields[EcsFieldName]; -export type IntegrationFieldMetadata = rt.TypeOf; - export type FieldName = EcsFieldName | (IntegrationFieldName & {}); -export type FieldMetadata = EcsFieldMetadata | IntegrationFieldMetadata; +export type FieldMetadataPlain = rt.TypeOf; +export type PartialFieldMetadataPlain = rt.TypeOf; + +export type FieldAttribute = rt.TypeOf; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts index e7d400c036b07..fe5f6cec92679 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts @@ -7,16 +7,17 @@ import { arrayToStringRt } from '@kbn/io-ts-utils'; import * as rt from 'io-ts'; -import { fieldMetadataRT } from '../types'; +import { fieldAttributeRT, partialFieldMetadataPlainRT } from '../types'; export const findFieldsMetadataRequestQueryRT = rt.exact( rt.partial({ + attributes: arrayToStringRt.pipe(rt.array(fieldAttributeRT)), fieldNames: arrayToStringRt.pipe(rt.array(rt.string)), }) ); export const findFieldsMetadataResponsePayloadRT = rt.type({ - fields: rt.record(rt.string, fieldMetadataRT), + fields: rt.record(rt.string, partialFieldMetadataPlainRT), }); export type FindFieldsMetadataRequestQuery = rt.TypeOf; diff --git a/x-pack/plugins/fields_metadata/common/index.ts b/x-pack/plugins/fields_metadata/common/index.ts index 70582b4f99541..8daf749f74261 100644 --- a/x-pack/plugins/fields_metadata/common/index.ts +++ b/x-pack/plugins/fields_metadata/common/index.ts @@ -5,13 +5,16 @@ * 2.0. */ -export { fieldMetadataRT } from './fields_metadata/types'; +export { fieldMetadataPlainRT } from './fields_metadata/types'; export type { - EcsFieldMetadata, EcsFieldName, - FieldMetadata, + FieldAttribute, + FieldMetadataPlain, FieldName, - IntegrationFieldMetadata, IntegrationFieldName, + PartialFieldMetadataPlain, TEcsFields, } from './fields_metadata/types'; + +export { FieldMetadata } from './fields_metadata/models/field_metadata'; +export { FieldsMetadataDictionary } from './fields_metadata/models/fields_metadata_dictionary'; diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 50456e0d1dbda..ea0bb707f9759 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -7,8 +7,9 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; +import hash from 'object-hash'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; -import { FieldName } from '../../../common'; +import { FieldAttribute, FieldName } from '../../../common'; import { IFieldsMetadataClient } from '../../services/fields_metadata'; interface UseFieldsMetadataFactoryDeps { @@ -16,7 +17,8 @@ interface UseFieldsMetadataFactoryDeps { } interface UseFieldsMetadataParams { - fieldNames: FieldName[]; + attributes?: FieldAttribute[]; + fieldNames?: FieldName[]; } interface UseFieldsMetadataReturnType { @@ -32,12 +34,12 @@ export type UseFieldsMetadataHook = ( export const createUseFieldsMetadataHook = ({ fieldsMetadataClient, }: UseFieldsMetadataFactoryDeps): UseFieldsMetadataHook => { - return ({ fieldNames }) => { - const serializedFieldNames = JSON.stringify(fieldNames); + return (params = {}) => { + const serializedParams = hash(params); const [{ error, loading, value }, load] = useAsyncFn( - () => fieldsMetadataClient.find({ fieldNames }), - [serializedFieldNames] + () => fieldsMetadataClient.find(params), + [serializedParams] ); useEffect(() => { diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index 167390fd3be7c..84dd42d9ec5f6 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -9,6 +9,7 @@ import { createValidationFunction } from '../../../common/runtime_types'; import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; import * as fieldsMetadataV1 from '../../../common/fields_metadata/v1'; import { FieldsMetadataBackendLibs } from '../../lib/shared_types'; +import { FindFieldsMetadataResponsePayload } from '../../../common/fields_metadata/v1'; export const initFindFieldsMetadataRoute = ({ router, @@ -29,16 +30,24 @@ export const initFindFieldsMetadataRoute = ({ }, }, async (_requestContext, request, response) => { - const { fieldNames } = request.query; + const { attributes, fieldNames } = request.query; const { fieldsMetadata } = (await getStartServices())[2]; const fieldsMetadataClient = fieldsMetadata.getClient(); try { - const fields = fieldsMetadataClient.find({ fieldNames }); + const fieldsDictionary = fieldsMetadataClient.find({ fieldNames }); + + const responsePayload: FindFieldsMetadataResponsePayload = { fields: {} }; + + if (attributes) { + responsePayload.fields = fieldsDictionary.pick(attributes); + } else { + responsePayload.fields = fieldsDictionary.toPlain(); + } return response.ok({ - body: fieldsMetadataV1.findFieldsMetadataResponsePayloadRT.encode({ fields }), + body: fieldsMetadataV1.findFieldsMetadataResponsePayloadRT.encode(responsePayload), }); } catch (error) { return response.customError({ diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts index 6b4b232fcca53..b28fcf482518e 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -7,28 +7,32 @@ import { EcsFlat } from '@elastic/ecs'; import { Logger } from '@kbn/core/server'; -import { FieldName, FieldMetadata, EcsFieldMetadata } from '../../../common'; -import { EcsFieldsSourceClient } from './source_clients/ecs_fields_source_client'; -import { IntegrationsFieldsSourceClient } from './source_clients/integration_fields_source_client'; +import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; +import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; +import { IntegrationsFieldsSourceClient } from './repositories/integration_fields_repository'; import { IFieldsMetadataClient } from './types'; interface FieldsMetadataClientDeps { logger: Logger; - ecsFieldsSourceClient: EcsFieldsSourceClient; + ecsFieldsRepository: EcsFieldsRepository; integrationFieldsSourceClient: IntegrationsFieldsSourceClient; } +interface FindOptions { + fieldNames?: FieldName[]; +} + export class FieldsMetadataClient implements IFieldsMetadataClient { private constructor( private readonly logger: Logger, - private readonly ecsFieldsSourceClient: EcsFieldsSourceClient, + private readonly ecsFieldsRepository: EcsFieldsRepository, private readonly integrationFieldsSourceClient: IntegrationsFieldsSourceClient ) {} getByName(fieldName: TFieldName): FieldMetadata | undefined { this.logger.debug(`Retrieving field metadata for: ${fieldName}`); - const field = this.ecsFieldsSourceClient.getByName(fieldName); + const field = this.ecsFieldsRepository.getByName(fieldName); // TODO: enable resolution for integration field // if (!field) { @@ -38,12 +42,12 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { return field; } - find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { + find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary { if (!fieldNames) { - return EcsFlat; + return this.ecsFieldsRepository.find(); } - const res = fieldNames.reduce((fieldsMetadata, fieldName) => { + const fields = fieldNames.reduce((fieldsMetadata, fieldName) => { const field = this.getByName(fieldName); if (field) { @@ -53,14 +57,14 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { return fieldsMetadata; }, {} as Record); - return res; + return FieldsMetadataDictionary.create(fields); } public static create({ logger, - ecsFieldsSourceClient, + ecsFieldsRepository, integrationFieldsSourceClient, }: FieldsMetadataClientDeps) { - return new FieldsMetadataClient(logger, ecsFieldsSourceClient, integrationFieldsSourceClient); + return new FieldsMetadataClient(logger, ecsFieldsRepository, integrationFieldsSourceClient); } } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index 4dfa596c38155..2e8b2bb446084 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -8,8 +8,8 @@ import { EcsFlat as ecsFields } from '@elastic/ecs'; import { Logger } from '@kbn/core/server'; import { FieldsMetadataClient } from './fields_metadata_client'; -import { EcsFieldsSourceClient } from './source_clients/ecs_fields_source_client'; -import { IntegrationsFieldsSourceClient } from './source_clients/integration_fields_source_client'; +import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; +import { IntegrationsFieldsSourceClient } from './repositories/integration_fields_repository'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; export class FieldsMetadataService { @@ -28,14 +28,14 @@ export class FieldsMetadataService { public start(): FieldsMetadataServiceStart { const { logger, packageService } = this; - const ecsFieldsSourceClient = EcsFieldsSourceClient.create({ ecsFields }); + const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); const integrationFieldsSourceClient = IntegrationsFieldsSourceClient.create({ packageService }); return { getClient() { return FieldsMetadataClient.create({ logger, - ecsFieldsSourceClient, + ecsFieldsRepository, integrationFieldsSourceClient, }); }, diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts new file mode 100644 index 0000000000000..53f254e01f154 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import mapValues from 'lodash/mapValues'; +import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary'; +import { FieldMetadata, FieldName, TEcsFields } from '../../../../common'; +import { IFieldsRepository } from './types'; + +interface EcsFieldsRepositoryDeps { + ecsFields: TEcsFields; +} + +interface FindOptions { + fieldNames?: FieldName[]; +} + +export class EcsFieldsRepository implements IFieldsRepository { + private readonly ecsFields: Record; + + private constructor(ecsFields: TEcsFields) { + this.ecsFields = mapValues(ecsFields, (field) => FieldMetadata.create(field)); + } + + getByName(fieldName: FieldName): FieldMetadata | undefined { + return this.ecsFields[fieldName]; + } + + find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary { + if (!fieldNames) { + return FieldsMetadataDictionary.create(this.ecsFields); + } + + const fields = fieldNames.reduce((fieldsMetadata, fieldName) => { + const field = this.getByName(fieldName); + + if (field) { + fieldsMetadata[fieldName] = field; + } + + return fieldsMetadata; + }, {} as Record); + + return FieldsMetadataDictionary.create(fields); + } + + public static create({ ecsFields }: EcsFieldsRepositoryDeps) { + return new EcsFieldsRepository(ecsFields); + } +} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts similarity index 90% rename from x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts rename to x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 6ef89bc7a9cae..3c0033b331e20 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/integration_fields_source_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -6,13 +6,13 @@ */ import { FieldMetadata, FieldName } from '../../../../common'; -import { ISourceClient } from './types'; +import { IFieldsRepository } from './types'; interface IntegrationsFieldsSourceClientDeps { packageService: PackageService; } -export class IntegrationsFieldsSourceClient implements ISourceClient { +export class IntegrationsFieldsSourceClient implements IFieldsRepository { private constructor(private readonly packageClient: PackageClient) {} getByName(fieldName: TFieldName) { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts similarity index 52% rename from x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts rename to x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index e056c4acb0774..cd2ace8f0e0a5 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,11 +5,10 @@ * 2.0. */ -import { FieldMetadata, FieldName } from '../../../../common'; +import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_dictionary'; +import type { FieldMetadata, FieldName } from '../../../../common'; -export interface ISourceClient { +export interface IFieldsRepository { getByName(fieldName: TFieldName): FieldMetadata | undefined; - find(params: { - fieldNames?: TFieldName[]; - }): Record; + find(params: { fieldNames?: TFieldName[] }): FieldsMetadataDictionary; } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts deleted file mode 100644 index 33d22a27909b3..0000000000000 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/source_clients/ecs_fields_source_client.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - EcsFieldMetadata, - EcsFieldName, - FieldMetadata, - FieldName, - TEcsFields, -} from '../../../../common'; -import { ISourceClient } from './types'; - -interface EcsFieldsSourceClientDeps { - ecsFields: TEcsFields; -} - -export class EcsFieldsSourceClient implements ISourceClient { - private constructor(private readonly ecsFields: TEcsFields) {} - - getByName(fieldName: TFieldName): EcsFieldMetadata | undefined { - return fieldName in this.ecsFields ? this.ecsFields[fieldName as EcsFieldName] : undefined; - } - - find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { - if (!fieldNames) { - return this.ecsFields; - } - - const res = fieldNames.reduce((fieldsMetadata, fieldName) => { - const field = this.getByName(fieldName); - - if (field) { - fieldsMetadata[fieldName] = field; - } - - return fieldsMetadata; - }, {} as Record); - - return res; - } - - public static create({ ecsFields }: EcsFieldsSourceClientDeps) { - return new EcsFieldsSourceClient(ecsFields); - } -} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts index d8900cf9c0817..1ff48e6c1bf24 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { FieldName, FieldMetadata } from '../../../common'; +import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldsMetadataServiceStartDeps {} @@ -19,7 +19,5 @@ export interface FieldsMetadataServiceStart { export interface IFieldsMetadataClient { getByName(fieldName: FieldName): FieldMetadata | undefined; - find(params: { - fieldNames?: TFieldName[]; - }): Record; + find(params: { fieldNames?: FieldName[] }): FieldsMetadataDictionary; } From 8d1d6f6a3a700475308f4443db005a3c444c511d Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 20 May 2024 18:34:51 +0200 Subject: [PATCH 08/50] chore(fields-metadata): update tsconfig --- src/plugins/unified_doc_viewer/tsconfig.json | 3 ++- x-pack/plugins/fields_metadata/tsconfig.json | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/plugins/unified_doc_viewer/tsconfig.json b/src/plugins/unified_doc_viewer/tsconfig.json index 1a0487b317129..43cfe7945f188 100644 --- a/src/plugins/unified_doc_viewer/tsconfig.json +++ b/src/plugins/unified_doc_viewer/tsconfig.json @@ -29,7 +29,8 @@ "@kbn/custom-icons", "@kbn/react-field", "@kbn/ui-theme", - "@kbn/discover-shared-plugin" + "@kbn/discover-shared-plugin", + "@kbn/fields-metadata-plugin" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/fields_metadata/tsconfig.json b/x-pack/plugins/fields_metadata/tsconfig.json index aaa085d9c2401..bc251c1b6ed79 100644 --- a/x-pack/plugins/fields_metadata/tsconfig.json +++ b/x-pack/plugins/fields_metadata/tsconfig.json @@ -11,5 +11,11 @@ "types/**/*" ], "exclude": ["target/**/*"], - "kbn_references": [] + "kbn_references": [ + "@kbn/core", + "@kbn/io-ts-utils", + "@kbn/logging", + "@kbn/core-http-request-handler-context-server", + "@kbn/core-http-server", + ] } From 9b7b72a6b84f603db51148447ad194452f0faeaa Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 21 May 2024 09:15:40 +0200 Subject: [PATCH 09/50] feat(fields-metadata): add source attribute --- .../fields_metadata/common/fields_metadata/types.ts | 13 +++++++------ .../fields_metadata/fields_metadata_client.ts | 11 +++++------ .../fields_metadata/fields_metadata_service.ts | 6 +++--- .../repositories/ecs_fields_repository.ts | 4 +++- .../repositories/integration_fields_repository.ts | 12 ++++++------ 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index 3347f8ddeb1f3..3b86cd5427ecc 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -8,6 +8,11 @@ import { EcsFlat } from '@elastic/ecs'; import * as rt from 'io-ts'; +export const fieldSourceRT = rt.keyof({ + ecs: null, + integration: null, +}); + export const allowedValueRT = rt.intersection([ rt.type({ description: rt.string, @@ -29,15 +34,11 @@ const requiredBaseMetadataPlainRT = rt.type({ description: rt.string, flat_name: rt.string, name: rt.string, + source: fieldSourceRT, type: rt.string, }); -const optionalBaseMetadataPlainRT = rt.partial({ - description: rt.string, - flat_name: rt.string, - name: rt.string, - type: rt.string, -}); +const optionalBaseMetadataPlainRT = rt.partial(requiredBaseMetadataPlainRT.props); const optionalMetadataPlainRT = rt.partial({ allowed_values: rt.array(allowedValueRT), diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts index b28fcf482518e..f4ba65a60cfdd 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -5,17 +5,16 @@ * 2.0. */ -import { EcsFlat } from '@elastic/ecs'; import { Logger } from '@kbn/core/server'; import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; -import { IntegrationsFieldsSourceClient } from './repositories/integration_fields_repository'; +import { IntegrationsFieldsRepository } from './repositories/integration_fields_repository'; import { IFieldsMetadataClient } from './types'; interface FieldsMetadataClientDeps { logger: Logger; ecsFieldsRepository: EcsFieldsRepository; - integrationFieldsSourceClient: IntegrationsFieldsSourceClient; + integrationFieldsRepository: IntegrationsFieldsRepository; } interface FindOptions { @@ -26,7 +25,7 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { private constructor( private readonly logger: Logger, private readonly ecsFieldsRepository: EcsFieldsRepository, - private readonly integrationFieldsSourceClient: IntegrationsFieldsSourceClient + private readonly integrationFieldsRepository: IntegrationsFieldsRepository ) {} getByName(fieldName: TFieldName): FieldMetadata | undefined { @@ -63,8 +62,8 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { public static create({ logger, ecsFieldsRepository, - integrationFieldsSourceClient, + integrationFieldsRepository, }: FieldsMetadataClientDeps) { - return new FieldsMetadataClient(logger, ecsFieldsRepository, integrationFieldsSourceClient); + return new FieldsMetadataClient(logger, ecsFieldsRepository, integrationFieldsRepository); } } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index 2e8b2bb446084..4a8ebafd72d08 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -9,7 +9,7 @@ import { EcsFlat as ecsFields } from '@elastic/ecs'; import { Logger } from '@kbn/core/server'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; -import { IntegrationsFieldsSourceClient } from './repositories/integration_fields_repository'; +import { IntegrationsFieldsRepository } from './repositories/integration_fields_repository'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; export class FieldsMetadataService { @@ -29,14 +29,14 @@ export class FieldsMetadataService { const { logger, packageService } = this; const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); - const integrationFieldsSourceClient = IntegrationsFieldsSourceClient.create({ packageService }); + const integrationFieldsRepository = IntegrationsFieldsRepository.create({ packageService }); return { getClient() { return FieldsMetadataClient.create({ logger, ecsFieldsRepository, - integrationFieldsSourceClient, + integrationFieldsRepository, }); }, }; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts index 53f254e01f154..5a6f1aacab0ca 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts @@ -22,7 +22,9 @@ export class EcsFieldsRepository implements IFieldsRepository { private readonly ecsFields: Record; private constructor(ecsFields: TEcsFields) { - this.ecsFields = mapValues(ecsFields, (field) => FieldMetadata.create(field)); + this.ecsFields = mapValues(ecsFields, (field) => + FieldMetadata.create({ ...field, source: 'ecs' }) + ); } getByName(fieldName: FieldName): FieldMetadata | undefined { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 3c0033b331e20..90ca85b3f27df 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -8,22 +8,22 @@ import { FieldMetadata, FieldName } from '../../../../common'; import { IFieldsRepository } from './types'; -interface IntegrationsFieldsSourceClientDeps { +interface IntegrationsFieldsRepositoryDeps { packageService: PackageService; } -export class IntegrationsFieldsSourceClient implements IFieldsRepository { +export class IntegrationsFieldsRepository implements IFieldsRepository { private constructor(private readonly packageClient: PackageClient) {} getByName(fieldName: TFieldName) { - throw new Error('TODO: Implement the IntegrationsFieldsSourceClient#getByName'); + throw new Error('TODO: Implement the IntegrationsFieldsRepository#getByName'); } find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { - throw new Error('TODO: Implement the IntegrationsFieldsSourceClient#getByName'); + throw new Error('TODO: Implement the IntegrationsFieldsRepository#getByName'); } - public static create({ packageService }: IntegrationsFieldsSourceClientDeps) { + public static create({ packageService }: IntegrationsFieldsRepositoryDeps) { if (!packageService) { return { getByName: () => undefined, @@ -33,6 +33,6 @@ export class IntegrationsFieldsSourceClient implements IFieldsRepository { const packageClient = packageService.asInternalUser; - return new IntegrationsFieldsSourceClient(packageClient); + return new IntegrationsFieldsRepository(packageClient); } } From 7ee8810b89014a31d9981724acf565d3071fb34b Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 21 May 2024 15:27:14 +0200 Subject: [PATCH 10/50] feat(fleet): mvp integration fields extractor --- x-pack/plugins/fleet/kibana.jsonc | 3 +- x-pack/plugins/fleet/server/plugin.ts | 8 +- .../server/services/epm/package_service.ts | 5 +- .../register_integration_fields_extractor.ts | 152 ++++++++++++++++++ 4 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts diff --git a/x-pack/plugins/fleet/kibana.jsonc b/x-pack/plugins/fleet/kibana.jsonc index c5e2eea368fed..3211a6a96ebbb 100644 --- a/x-pack/plugins/fleet/kibana.jsonc +++ b/x-pack/plugins/fleet/kibana.jsonc @@ -23,7 +23,8 @@ "taskManager", "files", "uiActions", - "dashboard" + "dashboard", + "fieldsMetadata" ], "optionalPlugins": [ "features", diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 02e6d7baf9577..0547dfac67aca 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -41,6 +41,7 @@ import type { SecurityPluginStart, } from '@kbn/security-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import type { FieldsMetadataServerSetup } from '@kbn/fields-metadata-plugin/server'; import type { TaskManagerSetupContract, TaskManagerStartContract, @@ -125,6 +126,7 @@ import { PolicyWatcher } from './services/agent_policy_watch'; import { getPackageSpecTagId } from './services/epm/kibana/assets/tag_assets'; import { FleetMetricsTask } from './services/metrics/fleet_metrics_task'; import { fetchAgentMetrics } from './services/metrics/fetch_agent_metrics'; +import { registerIntegrationFieldsExtractor } from './services/register_integration_fields_extractor'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -135,6 +137,7 @@ export interface FleetSetupDeps { spaces?: SpacesPluginStart; telemetry?: TelemetryPluginSetup; taskManager: TaskManagerSetupContract; + fieldsMetadata: FieldsMetadataServerSetup; } export interface FleetStartDeps { @@ -279,7 +282,7 @@ export class FleetPlugin }); } - public setup(core: CoreSetup, deps: FleetSetupDeps) { + public setup(core: CoreSetup, deps: FleetSetupDeps) { this.httpSetup = core.http; this.encryptedSavedObjectsSetup = deps.encryptedSavedObjects; this.cloud = deps.cloud; @@ -574,6 +577,9 @@ export class FleetPlugin taskManager: deps.taskManager, logFactory: this.initializerContext.logger, }); + + // Register fields metadata extractor + registerIntegrationFieldsExtractor({ core, fieldsMetadata: deps.fieldsMetadata }); } public start(core: CoreStart, plugins: FleetStartDeps): FleetStartContract { diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index b7174fa7883e7..2aa12040d8269 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -78,10 +78,7 @@ export interface PackageClient { bundledPackage: BundledPackage ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>; - getPackage( - packageName: string, - packageVersion: string - ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>; + getPackage(packageName: string, packageVersion: string): ReturnType; getPackages(params?: { excludeInstallStatus?: false; diff --git a/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts b/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts new file mode 100644 index 0000000000000..e9f17d0f19953 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { load } from 'js-yaml'; +import type { CoreSetup } from '@kbn/core/server'; +import type { FieldsMetadataServerSetup } from '@kbn/fields-metadata-plugin/server'; + +import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; + +import type { FleetStartContract, FleetStartDeps } from '../plugin'; +import type { AssetsMap, RegistryDataStream } from '../types'; + +interface RegistrationDeps { + core: CoreSetup; + fieldsMetadata: FieldsMetadataServerSetup; +} + +type IntegrationFieldMetadata = Pick< + PartialFieldMetadataPlain, + 'description' | 'flat_name' | 'name' | 'type' +>; + +type InputField = + | { + name: string; + type: string; + description?: string; + } + | { + name: string; + type: 'group'; + fields: InputField[]; + }; + +export const registerIntegrationFieldsExtractor = ({ core, fieldsMetadata }: RegistrationDeps) => { + fieldsMetadata.registerIntegrationFieldsExtractor( + async ({ integration, dataset }: { integration: string; dataset?: string }) => { + const [_core, _startDeps, { packageService }] = await core.getStartServices(); + + // Attempt retrieving latest integration version + const latestPackage = await packageService.asInternalUser.fetchFindLatestPackage(integration); + + const { name, version } = latestPackage; + const resolvedIntegration = await packageService.asInternalUser.getPackage(name, version); + + if (!resolvedIntegration) { + throw new Error('The integration assets you are looking for cannot be retrieved.'); + } + + const dataStreamsMap = resolveDataStreamsMap(resolvedIntegration.packageInfo.data_streams); + + const { assetsMap } = resolvedIntegration; + + if (dataStreamsMap.has(dataset)) { + const dataStream = dataStreamsMap.get(dataset); + + return resolveDataStreamFields({ dataStream, assetsMap }); + } else { + return [...dataStreamsMap.values()].reduce( + (integrationDataStreamsFields, dataStream) => + Object.assign( + integrationDataStreamsFields, + resolveDataStreamFields({ dataStream, assetsMap }) + ), + {} + ); + } + } + ); +}; + +const EXCLUDED_FILES = ['ecs.yml']; + +const isFieldsAsset = (assetPath: string, datasetPath: string) => { + return new RegExp( + `.*\/data_stream\/${datasetPath}\/fields\/(?!(${EXCLUDED_FILES.join('|')})$).*\.yml`, + 'i' + ).test(assetPath); +}; + +const getFieldAssetPaths = (assetsMap: AssetsMap, datasetPath: string) => { + return [...assetsMap.keys()].filter((path) => isFieldsAsset(path, datasetPath)); +}; + +const flattenFields = ( + fields: InputField[], + prefix = '' +): Record => { + return fields.reduce((acc, field) => { + const fqn = prefix ? `${prefix}.${field.name}` : field.name; + + if (isGroupField(field)) { + return Object.assign(acc, flattenFields(field.fields || [], fqn)); + } + + const integrationFieldMetadata = { + description: field.description, + flat_name: fqn, + name: field.name, + type: field.type, + }; + + acc[fqn] = integrationFieldMetadata; + return acc; + }, {} as Record); +}; + +const isGroupField = (field: InputField): field is Extract => { + return field.type === 'group'; +}; + +const resolveDataStreamsMap = (dataStreams?: RegistryDataStream[]) => { + if (!dataStreams) return new Map(); + + return dataStreams.reduce((dataStreamsMap, dataStream) => { + dataStreamsMap.set(dataStream.dataset, { + datasetName: dataStream.dataset, + datasetPath: dataStream.path, + }); + return dataStreamsMap; + }, new Map() as Map); +}; + +const resolveDataStreamFields = ({ + dataStream, + assetsMap, +}: { + dataStream: RegistryDataStream; + assetsMap: AssetsMap; +}) => { + const { datasetName, datasetPath } = dataStream; + const fieldsAssetPaths = getFieldAssetPaths(assetsMap, datasetPath); + + const fields = fieldsAssetPaths.reduce((fieldsMap, path) => { + const fieldsAsset = assetsMap.get(path); + if (fieldsAsset) { + const fieldsAssetJSON = load(fieldsAsset.toString('utf8')); + const flattenedFields = flattenFields(fieldsAssetJSON); + Object.assign(fieldsMap, flattenedFields); + } + + return fieldsMap; + }, {} as Record); + + return { + [datasetName]: fields, + }; +}; From 23b9e807fefa5b4d4160501a1430dfe344831545 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 21 May 2024 15:28:03 +0200 Subject: [PATCH 11/50] refactor(fields-metadata): renaming and update exports --- .../common/fields_metadata/common.ts | 2 +- .../plugins/fields_metadata/server/index.ts | 2 +- .../plugins/fields_metadata/server/mocks.ts | 6 ++-- .../plugins/fields_metadata/server/plugin.ts | 13 ++++---- .../fields_metadata/find_fields_metadata.ts | 4 +-- .../fields_metadata/fields_metadata_client.ts | 8 ++--- .../fields_metadata_service.ts | 13 +++++--- .../repositories/ecs_fields_repository.ts | 2 +- .../integration_fields_repository.ts | 32 +++++++++---------- .../fields_metadata/repositories/types.ts | 24 +++++++++++--- .../server/services/fields_metadata/types.ts | 6 ++-- .../plugins/fields_metadata/server/types.ts | 12 +++---- 12 files changed, 72 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts index fd1769dda63e6..4b5c58958d08b 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const FIND_FIELDS_METADATA_URL = '/api/fields_metadata'; +export const FIND_FIELDS_METADATA_URL = '/internal/fields_metadata'; diff --git a/x-pack/plugins/fields_metadata/server/index.ts b/x-pack/plugins/fields_metadata/server/index.ts index 212fcc95b646b..a0673d7f58217 100644 --- a/x-pack/plugins/fields_metadata/server/index.ts +++ b/x-pack/plugins/fields_metadata/server/index.ts @@ -7,7 +7,7 @@ import { PluginInitializerContext } from '@kbn/core/server'; -export type { FieldsMetadataPluginSetup, FieldsMetadataPluginStart } from './types'; +export type { FieldsMetadataServerSetup, FieldsMetadataServerStart } from './types'; export async function plugin(context: PluginInitializerContext) { const { FieldsMetadataPlugin } = await import('./plugin'); diff --git a/x-pack/plugins/fields_metadata/server/mocks.ts b/x-pack/plugins/fields_metadata/server/mocks.ts index ed65ff2f88b65..aaec4c0a85afd 100644 --- a/x-pack/plugins/fields_metadata/server/mocks.ts +++ b/x-pack/plugins/fields_metadata/server/mocks.ts @@ -9,10 +9,10 @@ import { createFieldsMetadataServiceSetupMock, createFieldsMetadataServiceStartMock, } from './services/fields_metadata/fields_metadata_service.mock'; -import { FieldsMetadataPluginSetup, FieldsMetadataPluginStart } from './types'; +import { FieldsMetadataServerSetup, FieldsMetadataServerStart } from './types'; const createFieldsMetadataSetupMock = () => { - const fieldsMetadataSetupMock: jest.Mocked = { + const fieldsMetadataSetupMock: jest.Mocked = { fieldsMetadata: createFieldsMetadataServiceSetupMock(), }; @@ -20,7 +20,7 @@ const createFieldsMetadataSetupMock = () => { }; const createFieldsMetadataStartMock = () => { - const fieldsMetadataStartMock: jest.Mocked = { + const fieldsMetadataStartMock: jest.Mocked = { fieldsMetadata: createFieldsMetadataServiceStartMock(), }; return fieldsMetadataStartMock; diff --git a/x-pack/plugins/fields_metadata/server/plugin.ts b/x-pack/plugins/fields_metadata/server/plugin.ts index b28ec952de016..68eda678edeaa 100644 --- a/x-pack/plugins/fields_metadata/server/plugin.ts +++ b/x-pack/plugins/fields_metadata/server/plugin.ts @@ -9,8 +9,8 @@ import { PluginInitializerContext, CoreStart, Plugin, Logger } from '@kbn/core/s import { FieldsMetadataPluginCoreSetup, - FieldsMetadataPluginSetup, - FieldsMetadataPluginStart, + FieldsMetadataServerSetup, + FieldsMetadataServerStart, FieldsMetadataServerPluginSetupDeps, FieldsMetadataServerPluginStartDeps, } from './types'; @@ -21,8 +21,8 @@ import { FieldsMetadataBackendLibs } from './lib/shared_types'; export class FieldsMetadataPlugin implements Plugin< - FieldsMetadataPluginSetup, - FieldsMetadataPluginStart, + FieldsMetadataServerSetup, + FieldsMetadataServerStart, FieldsMetadataServerPluginSetupDeps, FieldsMetadataServerPluginStartDeps > @@ -51,14 +51,15 @@ export class FieldsMetadataPlugin initFieldsMetadataServer(this.libs); return { - fieldsMetadata, + registerIntegrationFieldsExtractor: fieldsMetadata.registerIntegrationFieldsExtractor, }; } public start(core: CoreStart, plugins: FieldsMetadataServerPluginStartDeps) { const fieldsMetadata = this.fieldsMetadataService.start(); + const client = fieldsMetadata.getClient(); - return { fieldsMetadata }; + return { client }; } public stop() {} diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index 84dd42d9ec5f6..f2754dc0f99b7 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -32,8 +32,8 @@ export const initFindFieldsMetadataRoute = ({ async (_requestContext, request, response) => { const { attributes, fieldNames } = request.query; - const { fieldsMetadata } = (await getStartServices())[2]; - const fieldsMetadataClient = fieldsMetadata.getClient(); + const { client } = (await getStartServices())[2]; + const fieldsMetadataClient = client; try { const fieldsDictionary = fieldsMetadataClient.find({ fieldNames }); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts index f4ba65a60cfdd..d84b1b8477108 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -31,12 +31,12 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { getByName(fieldName: TFieldName): FieldMetadata | undefined { this.logger.debug(`Retrieving field metadata for: ${fieldName}`); - const field = this.ecsFieldsRepository.getByName(fieldName); + let field = this.ecsFieldsRepository.getByName(fieldName); // TODO: enable resolution for integration field - // if (!field) { - // field = this.integrationFieldsSourceClient.getByName(fieldName); - // } + if (!field) { + field = this.integrationFieldsRepository.getByName(fieldName); + } return field; } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index 4a8ebafd72d08..df75e3286e066 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -10,26 +10,29 @@ import { Logger } from '@kbn/core/server'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; import { IntegrationsFieldsRepository } from './repositories/integration_fields_repository'; +import { IntegrationFieldsExtractor } from './repositories/types'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; export class FieldsMetadataService { - private packageService: any; // TODO: update types + private integrationFieldsExtractor: IntegrationFieldsExtractor = () => ({}); constructor(private readonly logger: Logger) {} public setup(): FieldsMetadataServiceSetup { return { - registerPackageService: (packageService) => { - this.packageService = packageService; + registerIntegrationFieldsExtractor: (extractor: IntegrationFieldsExtractor) => { + this.integrationFieldsExtractor = extractor; }, }; } public start(): FieldsMetadataServiceStart { - const { logger, packageService } = this; + const { logger, integrationFieldsExtractor } = this; const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); - const integrationFieldsRepository = IntegrationsFieldsRepository.create({ packageService }); + const integrationFieldsRepository = IntegrationsFieldsRepository.create({ + integrationFieldsExtractor, + }); return { getClient() { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts index 5a6f1aacab0ca..35471257d1562 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts @@ -27,7 +27,7 @@ export class EcsFieldsRepository implements IFieldsRepository { ); } - getByName(fieldName: FieldName): FieldMetadata | undefined { + getByName(fieldName: FieldName) { return this.ecsFields[fieldName]; } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 90ca85b3f27df..9ea67abb39671 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -6,33 +6,31 @@ */ import { FieldMetadata, FieldName } from '../../../../common'; -import { IFieldsRepository } from './types'; +import { IFieldsRepository, IntegrationFieldsExtractor } from './types'; interface IntegrationsFieldsRepositoryDeps { - packageService: PackageService; + integrationFieldsExtractor: IntegrationFieldsExtractor; } export class IntegrationsFieldsRepository implements IFieldsRepository { - private constructor(private readonly packageClient: PackageClient) {} + private constructor(private readonly fieldsExtractor: IntegrationFieldsExtractor) {} - getByName(fieldName: TFieldName) { - throw new Error('TODO: Implement the IntegrationsFieldsRepository#getByName'); + async getByName(fieldName: TFieldName) { + const data = await this.fieldsExtractor({ + integration: '1password', + // dataset: 'audit_events', + }); + + console.log(JSON.stringify(data, null, 2)); + + return undefined; } - find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { + async find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { throw new Error('TODO: Implement the IntegrationsFieldsRepository#getByName'); } - public static create({ packageService }: IntegrationsFieldsRepositoryDeps) { - if (!packageService) { - return { - getByName: () => undefined, - find: () => ({}), - }; - } - - const packageClient = packageService.asInternalUser; - - return new IntegrationsFieldsRepository(packageClient); + public static create({ integrationFieldsExtractor }: IntegrationsFieldsRepositoryDeps) { + return new IntegrationsFieldsRepository(integrationFieldsExtractor); } } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index cd2ace8f0e0a5..0c38176fa65ec 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,10 +5,26 @@ * 2.0. */ -import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_dictionary'; -import type { FieldMetadata, FieldName } from '../../../../common'; +import type { + FieldMetadata, + FieldName, + FieldsMetadataDictionary, + PartialFieldMetadataPlain, +} from '../../../../common'; export interface IFieldsRepository { - getByName(fieldName: TFieldName): FieldMetadata | undefined; - find(params: { fieldNames?: TFieldName[] }): FieldsMetadataDictionary; + getByName( + fieldName: TFieldName + ): Promise | FieldMetadata | undefined; + find(params: { + fieldNames?: TFieldName[]; + }): Promise | FieldsMetadataDictionary; } + +export type IntegrationFieldsExtractor = ({ + integration, + dataset, +}: { + integration: string; + dataset?: string; +}) => Promise>; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts index 1ff48e6c1bf24..496308f5e3a3d 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts @@ -6,12 +6,14 @@ */ import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; +import { IntegrationFieldsExtractor } from './repositories/types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldsMetadataServiceStartDeps {} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FieldsMetadataServiceSetup {} +export interface FieldsMetadataServiceSetup { + registerIntegrationFieldsExtractor: (extractor: IntegrationFieldsExtractor) => void; +} export interface FieldsMetadataServiceStart { getClient(): IFieldsMetadataClient; diff --git a/x-pack/plugins/fields_metadata/server/types.ts b/x-pack/plugins/fields_metadata/server/types.ts index c7ffc26598640..6bc3583f3f598 100644 --- a/x-pack/plugins/fields_metadata/server/types.ts +++ b/x-pack/plugins/fields_metadata/server/types.ts @@ -9,22 +9,22 @@ import type { CoreSetup } from '@kbn/core/server'; import { FieldsMetadataServiceSetup, - FieldsMetadataServiceStart, + IFieldsMetadataClient, } from './services/fields_metadata/types'; export type FieldsMetadataPluginCoreSetup = CoreSetup< FieldsMetadataServerPluginStartDeps, - FieldsMetadataPluginStart + FieldsMetadataServerStart >; export type FieldsMetadataPluginStartServicesAccessor = FieldsMetadataPluginCoreSetup['getStartServices']; -export interface FieldsMetadataPluginSetup { - fieldsMetadata: FieldsMetadataServiceSetup; +export interface FieldsMetadataServerSetup { + registerIntegrationFieldsExtractor: FieldsMetadataServiceSetup['registerIntegrationFieldsExtractor']; } -export interface FieldsMetadataPluginStart { - fieldsMetadata: FieldsMetadataServiceStart; +export interface FieldsMetadataServerStart { + client: IFieldsMetadataClient; } // eslint-disable-next-line @typescript-eslint/no-empty-interface From 6775428c9bc0902554ac195258cf6bd2e381e7ad Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 21 May 2024 17:15:00 +0200 Subject: [PATCH 12/50] feat(fields-metadata): get to work integration fields mvp --- .../common/fields_metadata/types.ts | 2 +- .../v1/find_fields_metadata.ts | 2 + .../fields_metadata/find_fields_metadata.ts | 8 +- .../fields_metadata/fields_metadata_client.ts | 42 ++++--- .../fields_metadata_service.ts | 4 +- .../repositories/ecs_fields_repository.ts | 5 +- .../integration_fields_repository.ts | 108 +++++++++++++++--- .../fields_metadata/repositories/types.ts | 24 ++-- .../server/services/fields_metadata/types.ts | 10 +- 9 files changed, 142 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index 3b86cd5427ecc..b121dba5349d2 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -31,7 +31,6 @@ export const multiFieldRT = rt.type({ }); const requiredBaseMetadataPlainRT = rt.type({ - description: rt.string, flat_name: rt.string, name: rt.string, source: fieldSourceRT, @@ -44,6 +43,7 @@ const optionalMetadataPlainRT = rt.partial({ allowed_values: rt.array(allowedValueRT), beta: rt.string, dashed_name: rt.string, + description: rt.string, doc_values: rt.boolean, example: rt.unknown, expected_values: rt.array(rt.string), diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts index fe5f6cec92679..fd06a941109dc 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts @@ -13,6 +13,8 @@ export const findFieldsMetadataRequestQueryRT = rt.exact( rt.partial({ attributes: arrayToStringRt.pipe(rt.array(fieldAttributeRT)), fieldNames: arrayToStringRt.pipe(rt.array(rt.string)), + integration: rt.string, + dataset: rt.string, }) ); diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index f2754dc0f99b7..dd0bcbaf6177f 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -30,13 +30,17 @@ export const initFindFieldsMetadataRoute = ({ }, }, async (_requestContext, request, response) => { - const { attributes, fieldNames } = request.query; + const { attributes, fieldNames, integration, dataset } = request.query; const { client } = (await getStartServices())[2]; const fieldsMetadataClient = client; try { - const fieldsDictionary = fieldsMetadataClient.find({ fieldNames }); + const fieldsDictionary = await fieldsMetadataClient.find({ + fieldNames, + integration, + dataset, + }); const responsePayload: FindFieldsMetadataResponsePayload = { fields: {} }; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts index d84b1b8477108..e152c14347927 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -8,53 +8,57 @@ import { Logger } from '@kbn/core/server'; import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; -import { IntegrationsFieldsRepository } from './repositories/integration_fields_repository'; -import { IFieldsMetadataClient } from './types'; +import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; +import { IntegrationFieldsSearchParams } from './repositories/types'; +import { FindFieldsMetadataOptions, IFieldsMetadataClient } from './types'; interface FieldsMetadataClientDeps { logger: Logger; ecsFieldsRepository: EcsFieldsRepository; - integrationFieldsRepository: IntegrationsFieldsRepository; -} - -interface FindOptions { - fieldNames?: FieldName[]; + integrationFieldsRepository: IntegrationFieldsRepository; } export class FieldsMetadataClient implements IFieldsMetadataClient { private constructor( private readonly logger: Logger, private readonly ecsFieldsRepository: EcsFieldsRepository, - private readonly integrationFieldsRepository: IntegrationsFieldsRepository + private readonly integrationFieldsRepository: IntegrationFieldsRepository ) {} - getByName(fieldName: TFieldName): FieldMetadata | undefined { + async getByName( + fieldName: TFieldName, + { integration, dataset }: Partial = {} + ): Promise { this.logger.debug(`Retrieving field metadata for: ${fieldName}`); + // 1. Try resolving from ecs static metadata let field = this.ecsFieldsRepository.getByName(fieldName); - // TODO: enable resolution for integration field - if (!field) { - field = this.integrationFieldsRepository.getByName(fieldName); + // 2. Try searching for the fiels in the Elastic Package Registry + if (!field && integration) { + field = await this.integrationFieldsRepository.getByName(fieldName, { integration, dataset }); } return field; } - find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary { + async find({ + fieldNames, + integration, + dataset, + }: FindFieldsMetadataOptions = {}): Promise { if (!fieldNames) { return this.ecsFieldsRepository.find(); } - const fields = fieldNames.reduce((fieldsMetadata, fieldName) => { - const field = this.getByName(fieldName); + const fields: Record = {}; + for (const fieldName of fieldNames) { + const field = await this.getByName(fieldName, { integration, dataset }); if (field) { - fieldsMetadata[fieldName] = field; + fields[fieldName] = field; } - - return fieldsMetadata; - }, {} as Record); + } return FieldsMetadataDictionary.create(fields); } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index df75e3286e066..6934b3a61e7ef 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -9,7 +9,7 @@ import { EcsFlat as ecsFields } from '@elastic/ecs'; import { Logger } from '@kbn/core/server'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; -import { IntegrationsFieldsRepository } from './repositories/integration_fields_repository'; +import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; import { IntegrationFieldsExtractor } from './repositories/types'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; @@ -30,7 +30,7 @@ export class FieldsMetadataService { const { logger, integrationFieldsExtractor } = this; const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); - const integrationFieldsRepository = IntegrationsFieldsRepository.create({ + const integrationFieldsRepository = IntegrationFieldsRepository.create({ integrationFieldsExtractor, }); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts index 35471257d1562..05f34bbf0ee60 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts @@ -8,7 +8,6 @@ import mapValues from 'lodash/mapValues'; import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary'; import { FieldMetadata, FieldName, TEcsFields } from '../../../../common'; -import { IFieldsRepository } from './types'; interface EcsFieldsRepositoryDeps { ecsFields: TEcsFields; @@ -18,7 +17,7 @@ interface FindOptions { fieldNames?: FieldName[]; } -export class EcsFieldsRepository implements IFieldsRepository { +export class EcsFieldsRepository { private readonly ecsFields: Record; private constructor(ecsFields: TEcsFields) { @@ -27,7 +26,7 @@ export class EcsFieldsRepository implements IFieldsRepository { ); } - getByName(fieldName: FieldName) { + getByName(fieldName: FieldName): FieldMetadata | undefined { return this.ecsFields[fieldName]; } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 9ea67abb39671..628b88b77e25c 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -5,32 +5,108 @@ * 2.0. */ -import { FieldMetadata, FieldName } from '../../../../common'; -import { IFieldsRepository, IntegrationFieldsExtractor } from './types'; +import { HashedCache } from '../../../../common/hashed_cache'; +import { FieldMetadata, FieldMetadataPlain, IntegrationFieldName } from '../../../../common'; +import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; -interface IntegrationsFieldsRepositoryDeps { +interface IntegrationFieldsRepositoryDeps { integrationFieldsExtractor: IntegrationFieldsExtractor; } -export class IntegrationsFieldsRepository implements IFieldsRepository { - private constructor(private readonly fieldsExtractor: IntegrationFieldsExtractor) {} +export class IntegrationFieldsRepository { + private cache: HashedCache>>; - async getByName(fieldName: TFieldName) { - const data = await this.fieldsExtractor({ - integration: '1password', - // dataset: 'audit_events', - }); + private constructor(private readonly fieldsExtractor: IntegrationFieldsExtractor) { + this.cache = new HashedCache(); + } + + async getByName( + fieldName: IntegrationFieldName, + { integration, dataset }: IntegrationFieldsSearchParams + ): Promise { + let field = this.getCachedField(fieldName, { integration, dataset }); - console.log(JSON.stringify(data, null, 2)); + if (!field) { + await this.fieldsExtractor({ integration, dataset }) + .then(this.mapExtractedFieldsToFieldMetadataInstances) + .then((fieldMetadataTree) => this.storeFieldsInCache(fieldMetadataTree, integration)); - return undefined; + field = this.getCachedField(fieldName, { integration, dataset }); + } + + return field; } - async find({ fieldNames }: { fieldNames?: FieldName[] } = {}): Record { - throw new Error('TODO: Implement the IntegrationsFieldsRepository#getByName'); + async find({ fieldNames }: { fieldNames?: IntegrationFieldName[] } = {}) { + throw new Error('TODO: Implement the IntegrationFieldsRepository#getByName'); } - public static create({ integrationFieldsExtractor }: IntegrationsFieldsRepositoryDeps) { - return new IntegrationsFieldsRepository(integrationFieldsExtractor); + public static create({ integrationFieldsExtractor }: IntegrationFieldsRepositoryDeps) { + return new IntegrationFieldsRepository(integrationFieldsExtractor); } + + private getCachedField( + fieldName: IntegrationFieldName, + { integration, dataset }: IntegrationFieldsSearchParams + ): FieldMetadata | undefined { + const cachedIntegration = this.cache.get(integration); + + // 1. Integration fields were never fetched + if (!cachedIntegration) { + return undefined; + } + + // 2. Dataset is passed but was never fetched before + if (dataset && !cachedIntegration.hasOwnProperty(dataset)) { + return undefined; + } + + // 3. Dataset is passed and it was previously fetched, should return the field + if (dataset && cachedIntegration.hasOwnProperty(dataset)) { + const targetDataset = cachedIntegration[dataset]; + return targetDataset[fieldName]; + } + + // 4. Dataset is not passed, we attempt search on all stored datasets + if (!dataset) { + // Merge all the available datasets into a unique field list. Overriding fields might occur in the process. + const cachedDatasetsFields = Object.assign({}, ...Object.values(cachedIntegration)); + return cachedDatasetsFields[fieldName]; + } + } + + private storeFieldsInCache = ( + extractedFieldsMetadata: Record>, + integration: string + ): void => { + const cachedIntegration = this.cache.get(integration); + + if (!cachedIntegration) { + this.cache.set(integration, extractedFieldsMetadata); + } else { + this.cache.set(integration, { ...cachedIntegration, ...extractedFieldsMetadata }); + } + }; + + private mapExtractedFieldsToFieldMetadataInstances = ( + extractedFields: Record> + ) => { + return Object.entries(extractedFields).reduce( + (integrationGroup, [datasetName, datasetGroup]) => { + integrationGroup[datasetName] = Object.entries(datasetGroup).reduce( + (datasetGroupResult, [extractedFieldName, extractedField]) => { + datasetGroupResult[extractedFieldName] = FieldMetadata.create({ + ...extractedField, + source: 'integration', + }); + return datasetGroupResult; + }, + {} as Record + ); + + return integrationGroup; + }, + {} as Record> + ); + }; } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index 0c38176fa65ec..e2f38e7acba81 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,26 +5,16 @@ * 2.0. */ -import type { - FieldMetadata, - FieldName, - FieldsMetadataDictionary, - PartialFieldMetadataPlain, -} from '../../../../common'; +import type { FieldMetadataPlain } from '../../../../common'; -export interface IFieldsRepository { - getByName( - fieldName: TFieldName - ): Promise | FieldMetadata | undefined; - find(params: { - fieldNames?: TFieldName[]; - }): Promise | FieldsMetadataDictionary; +export interface IntegrationFieldsSearchParams { + integration: string; + dataset?: string; } +export type ExtractedIntegrationFields = Record>; + export type IntegrationFieldsExtractor = ({ integration, dataset, -}: { - integration: string; - dataset?: string; -}) => Promise>; +}: IntegrationFieldsSearchParams) => Promise; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts index 496308f5e3a3d..ef092b3612aee 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts @@ -6,7 +6,7 @@ */ import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; -import { IntegrationFieldsExtractor } from './repositories/types'; +import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './repositories/types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldsMetadataServiceStartDeps {} @@ -19,7 +19,11 @@ export interface FieldsMetadataServiceStart { getClient(): IFieldsMetadataClient; } +export interface FindFieldsMetadataOptions extends Partial { + fieldNames?: FieldName[]; +} + export interface IFieldsMetadataClient { - getByName(fieldName: FieldName): FieldMetadata | undefined; - find(params: { fieldNames?: FieldName[] }): FieldsMetadataDictionary; + getByName(fieldName: FieldName): Promise; + find(params: FindFieldsMetadataOptions): Promise; } From 1b23ef434c67f0839fdcbb0545595e43917ede3b Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 21 May 2024 18:47:17 +0200 Subject: [PATCH 13/50] refactor(fields-metadata): update cache key --- .../integration_fields_repository.ts | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 628b88b77e25c..d4dec3afeb224 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -5,10 +5,10 @@ * 2.0. */ +import hash from 'object-hash'; import { HashedCache } from '../../../../common/hashed_cache'; import { FieldMetadata, FieldMetadataPlain, IntegrationFieldName } from '../../../../common'; import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; - interface IntegrationFieldsRepositoryDeps { integrationFieldsExtractor: IntegrationFieldsExtractor; } @@ -27,9 +27,7 @@ export class IntegrationFieldsRepository { let field = this.getCachedField(fieldName, { integration, dataset }); if (!field) { - await this.fieldsExtractor({ integration, dataset }) - .then(this.mapExtractedFieldsToFieldMetadataInstances) - .then((fieldMetadataTree) => this.storeFieldsInCache(fieldMetadataTree, integration)); + await this.extractFields({ integration, dataset }); field = this.getCachedField(fieldName, { integration, dataset }); } @@ -45,11 +43,28 @@ export class IntegrationFieldsRepository { return new IntegrationFieldsRepository(integrationFieldsExtractor); } + private async extractFields({ + integration, + dataset, + }: IntegrationFieldsSearchParams): Promise { + const cacheKey = this.getCacheKey({ integration, dataset }); + const cachedIntegration = this.cache.get(cacheKey); + + if (cachedIntegration) { + return undefined; + } + + return this.fieldsExtractor({ integration, dataset }) + .then(this.mapExtractedFieldsToFieldMetadataInstances) + .then((fieldMetadataTree) => this.storeFieldsInCache(cacheKey, fieldMetadataTree)); + } + private getCachedField( fieldName: IntegrationFieldName, { integration, dataset }: IntegrationFieldsSearchParams ): FieldMetadata | undefined { - const cachedIntegration = this.cache.get(integration); + const cacheKey = this.getCacheKey({ integration, dataset }); + const cachedIntegration = this.cache.get(cacheKey); // 1. Integration fields were never fetched if (!cachedIntegration) { @@ -76,18 +91,20 @@ export class IntegrationFieldsRepository { } private storeFieldsInCache = ( - extractedFieldsMetadata: Record>, - integration: string + cacheKey: string, + extractedFieldsMetadata: Record> ): void => { - const cachedIntegration = this.cache.get(integration); + const cachedIntegration = this.cache.get(cacheKey); if (!cachedIntegration) { - this.cache.set(integration, extractedFieldsMetadata); + this.cache.set(cacheKey, extractedFieldsMetadata); } else { - this.cache.set(integration, { ...cachedIntegration, ...extractedFieldsMetadata }); + this.cache.set(cacheKey, { ...cachedIntegration, ...extractedFieldsMetadata }); } }; + private getCacheKey = (params: IntegrationFieldsSearchParams) => hash(params); + private mapExtractedFieldsToFieldMetadataInstances = ( extractedFields: Record> ) => { From 2167cecb4c8d4a5f799db4a27707924077e384a3 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 22 May 2024 12:51:10 +0200 Subject: [PATCH 14/50] refactor(fleet): move implementation to package service and add smoke test --- .../fields_metadata/repositories/types.ts | 4 +- .../services/epm/package_service.mock.ts | 1 + .../services/epm/package_service.test.ts | 17 ++- .../server/services/epm/package_service.ts | 20 ++- .../server/services/epm/packages/utils.ts | 112 ++++++++++++++ .../server/services/epm/registry/index.ts | 51 ++++++- .../register_integration_fields_extractor.ts | 137 +----------------- 7 files changed, 206 insertions(+), 136 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index e2f38e7acba81..6d19e0ae2d5ae 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,14 +5,14 @@ * 2.0. */ -import type { FieldMetadataPlain } from '../../../../common'; +import type { PartialFieldMetadataPlain } from '../../../../common'; export interface IntegrationFieldsSearchParams { integration: string; dataset?: string; } -export type ExtractedIntegrationFields = Record>; +export type ExtractedIntegrationFields = Record>; export type IntegrationFieldsExtractor = ({ integration, diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts index 3eb689dfa1a2a..9a2d57fad3ccb 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts @@ -14,6 +14,7 @@ const createClientMock = (): jest.Mocked => ({ fetchFindLatestPackage: jest.fn(), readBundledPackage: jest.fn(), getPackage: jest.fn(), + getPackageFieldsMetadata: jest.fn(), getPackages: jest.fn(), reinstallEsAssets: jest.fn(), }); diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 8bd2ace1d760c..479d355c00e68 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -44,6 +44,7 @@ const testKeys = [ 'ensureInstalledPackage', 'fetchFindLatestPackage', 'getPackage', + 'getPackageFieldsMetadata', 'reinstallEsAssets', 'readBundledPackage', ]; @@ -127,6 +128,20 @@ function getTest( }; break; case testKeys[4]: + test = { + method: mocks.packageClient.getPackageFieldsMetadata.bind(mocks.packageClient), + args: [{ packageName: 'package_name', datasetName: 'dataset_name' }], + spy: jest.spyOn(epmRegistry, 'getPackageFieldsMetadata'), + spyArgs: [{ packageName: 'package_name', datasetName: 'dataset_name' }, undefined], + spyResponse: { + dataset_name: { field_1: { flat_name: 'field_1', type: 'keyword' } }, + }, + expectedReturnValue: { + dataset_name: { field_1: { flat_name: 'field_1', type: 'keyword' } }, + }, + }; + break; + case testKeys[5]: const pkg: InstallablePackage = { format_version: '1.0.0', name: 'package name', @@ -172,7 +187,7 @@ function getTest( ], }; break; - case testKeys[5]: + case testKeys[6]: const bundledPackage = { name: 'package name', version: '8.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 2aa12040d8269..ec2ef2e14e92f 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -37,6 +37,7 @@ import { INSTALL_PACKAGES_AUTHZ, READ_PACKAGE_INFO_AUTHZ } from '../../routes/ep import type { InstallResult } from '../../../common'; import type { FetchFindLatestPackageOptions } from './registry'; +import { getPackageFieldsMetadata } from './registry'; import * as Registry from './registry'; import { fetchFindLatestPackageOrThrow, getPackage } from './registry'; @@ -78,7 +79,16 @@ export interface PackageClient { bundledPackage: BundledPackage ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>; - getPackage(packageName: string, packageVersion: string): ReturnType; + getPackage( + packageName: string, + packageVersion: string, + options?: Parameters['2'] + ): ReturnType; + + getPackageFieldsMetadata( + params: Parameters['0'], + options?: Parameters['1'] + ): ReturnType; getPackages(params?: { excludeInstallStatus?: false; @@ -224,6 +234,14 @@ class PackageClientImpl implements PackageClient { return getPackage(packageName, packageVersion, options); } + public async getPackageFieldsMetadata( + params: Parameters['0'], + options?: Parameters['1'] + ) { + await this.#runPreflight(READ_PACKAGE_INFO_AUTHZ); + return getPackageFieldsMetadata(params, options); + } + public async getPackages(params?: { excludeInstallStatus?: false; category?: CategoryId; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts index 0cb97ca007daf..ba5a2d73ee83d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts @@ -6,6 +6,118 @@ */ import { withSpan } from '@kbn/apm-utils'; +import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; + +import { load } from 'js-yaml'; + +import type { RegistryDataStream } from '../../../../common'; +import type { AssetsMap } from '../../../../common/types'; + +interface PackageFieldMetadata extends PartialFieldMetadataPlain { + name: string; +} + +type InputField = + | PackageFieldMetadata + | { + name: string; + type: 'group'; + fields: InputField[]; + }; export const withPackageSpan = (stepName: string, func: () => Promise) => withSpan({ name: stepName, type: 'package' }, func); + +const normalizeFields = ( + fields: InputField[], + prefix = '' +): Record => { + return fields.reduce((normalizedFields, field) => { + const flatName = prefix ? `${prefix}.${field.name}` : field.name; + // Recursively resolve field groups + if (isGroupField(field)) { + return Object.assign(normalizedFields, normalizeFields(field.fields || [], flatName)); + } + + normalizedFields[flatName] = createIntegrationField(field, flatName); + + return normalizedFields; + }, {} as Record); +}; + +const createIntegrationField = ( + field: Omit, + flatName: string +) => ({ + ...field, + flat_name: flatName, +}); + +const isGroupField = (field: InputField): field is Extract => { + return field.type === 'group'; +}; + +export const resolveDataStreamsMap = ( + dataStreams?: RegistryDataStream[] +): Map => { + if (!dataStreams) return new Map(); + + return dataStreams.reduce((dataStreamsMap, dataStream) => { + dataStreamsMap.set(dataStream.dataset, dataStream); + return dataStreamsMap; + }, new Map() as Map); +}; + +export const resolveDataStreamFields = ({ + dataStream, + assetsMap, + excludedFieldsAssets, +}: { + dataStream: RegistryDataStream; + assetsMap: AssetsMap; + excludedFieldsAssets?: string[]; +}) => { + const { dataset, path } = dataStream; + const dataStreamFieldsAssetPaths = getDataStreamFieldsAssetPaths( + assetsMap, + path, + excludedFieldsAssets + ); + + const fields = dataStreamFieldsAssetPaths.reduce((dataStreamFields, fieldsAssetPath) => { + const fieldsAssetBuffer = assetsMap.get(fieldsAssetPath); + + if (fieldsAssetBuffer) { + const fieldsAssetJSON = load(fieldsAssetBuffer.toString('utf8')); + const normalizedFields = normalizeFields(fieldsAssetJSON); + Object.assign(dataStreamFields, normalizedFields); + } + + return dataStreamFields; + }, {} as Record); + + return { + [dataset]: fields, + }; +}; + +const isFieldsAsset = ( + assetPath: string, + dataStreamPath: string, + excludedFieldsAssets: string[] = [] +) => { + return new RegExp( + `.*\/data_stream\/${dataStreamPath}\/fields\/(?!(${excludedFieldsAssets.join('|')})$).*\.yml`, + 'i' + ).test(assetPath); +}; + +const getDataStreamFieldsAssetPaths = ( + assetsMap: AssetsMap, + dataStreamPath: string, + excludedFieldsAssets?: string[] +) => { + return [...assetsMap.keys()].filter((path) => + isFieldsAsset(path, dataStreamPath, excludedFieldsAssets) + ); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 9dfd307725945..e8b5cb556aa15 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -13,6 +13,8 @@ import semverGte from 'semver/functions/gte'; import type { Response } from 'node-fetch'; import type { Logger } from '@kbn/logging'; +import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; + import { splitPkgKey as split } from '../../../../common/services'; import { KibanaAssetType } from '../../../types'; @@ -48,7 +50,7 @@ import { import { getBundledPackageByName } from '../packages/bundled_packages'; -import { withPackageSpan } from '../packages/utils'; +import { resolveDataStreamFields, resolveDataStreamsMap, withPackageSpan } from '../packages/utils'; import { verifyPackageArchiveSignature } from '../packages/package_verification'; @@ -352,6 +354,53 @@ export async function getPackage( return { paths, packageInfo, assetsMap, verificationResult }; } +export async function getPackageFieldsMetadata( + params: { packageName: string; datasetName?: string }, + options: { excludedFieldsAssets?: string[] } = {} +): Promise>> { + const { packageName, datasetName } = params; + const { excludedFieldsAssets = ['ecs.yml'] } = options; + + // Attempt retrieving latest package name and version + const latestPackage = await fetchFindLatestPackageOrThrow(packageName); + const { name, version } = latestPackage; + + // Attempt retrieving latest package + const resolvedPackage = await getPackage(name, version); + + if (!resolvedPackage) { + throw new Error('The package you are looking for cannot be retrieved.'); + } + + // We need to collect all the available data streams for the package. + // In case a dataset is specified from the parameter, it will load the fields only for that specific dataset. + // As a fallback case, we'll try to read the fields for all the data streams in the package. + const dataStreamsMap = resolveDataStreamsMap(resolvedPackage.packageInfo.data_streams); + + const { assetsMap } = resolvedPackage; + + const dataStream = datasetName ? dataStreamsMap.get(datasetName) : null; + + if (dataStream) { + // Resolve a single data stream fields when the `datasetName` parameter is specified + return resolveDataStreamFields({ dataStream, assetsMap, excludedFieldsAssets }); + } else { + // Resolve and merge all the integration data streams fields otherwise + return [...dataStreamsMap.values()].reduce( + (packageDataStreamsFields, currentDataStream) => + Object.assign( + packageDataStreamsFields, + resolveDataStreamFields({ + dataStream: currentDataStream, + assetsMap, + excludedFieldsAssets, + }) + ), + {} + ); + } +} + function ensureContentType(archivePath: string) { const contentType = mime.lookup(archivePath); diff --git a/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts b/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts index e9f17d0f19953..d06c1e528469d 100644 --- a/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts +++ b/x-pack/plugins/fleet/server/services/register_integration_fields_extractor.ts @@ -5,148 +5,23 @@ * 2.0. */ -import { load } from 'js-yaml'; import type { CoreSetup } from '@kbn/core/server'; import type { FieldsMetadataServerSetup } from '@kbn/fields-metadata-plugin/server'; -import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; - import type { FleetStartContract, FleetStartDeps } from '../plugin'; -import type { AssetsMap, RegistryDataStream } from '../types'; interface RegistrationDeps { core: CoreSetup; fieldsMetadata: FieldsMetadataServerSetup; } -type IntegrationFieldMetadata = Pick< - PartialFieldMetadataPlain, - 'description' | 'flat_name' | 'name' | 'type' ->; - -type InputField = - | { - name: string; - type: string; - description?: string; - } - | { - name: string; - type: 'group'; - fields: InputField[]; - }; - export const registerIntegrationFieldsExtractor = ({ core, fieldsMetadata }: RegistrationDeps) => { - fieldsMetadata.registerIntegrationFieldsExtractor( - async ({ integration, dataset }: { integration: string; dataset?: string }) => { - const [_core, _startDeps, { packageService }] = await core.getStartServices(); - - // Attempt retrieving latest integration version - const latestPackage = await packageService.asInternalUser.fetchFindLatestPackage(integration); - - const { name, version } = latestPackage; - const resolvedIntegration = await packageService.asInternalUser.getPackage(name, version); - - if (!resolvedIntegration) { - throw new Error('The integration assets you are looking for cannot be retrieved.'); - } - - const dataStreamsMap = resolveDataStreamsMap(resolvedIntegration.packageInfo.data_streams); - - const { assetsMap } = resolvedIntegration; + fieldsMetadata.registerIntegrationFieldsExtractor(async ({ integration, dataset }) => { + const [_core, _startDeps, { packageService }] = await core.getStartServices(); - if (dataStreamsMap.has(dataset)) { - const dataStream = dataStreamsMap.get(dataset); - - return resolveDataStreamFields({ dataStream, assetsMap }); - } else { - return [...dataStreamsMap.values()].reduce( - (integrationDataStreamsFields, dataStream) => - Object.assign( - integrationDataStreamsFields, - resolveDataStreamFields({ dataStream, assetsMap }) - ), - {} - ); - } - } - ); -}; - -const EXCLUDED_FILES = ['ecs.yml']; - -const isFieldsAsset = (assetPath: string, datasetPath: string) => { - return new RegExp( - `.*\/data_stream\/${datasetPath}\/fields\/(?!(${EXCLUDED_FILES.join('|')})$).*\.yml`, - 'i' - ).test(assetPath); -}; - -const getFieldAssetPaths = (assetsMap: AssetsMap, datasetPath: string) => { - return [...assetsMap.keys()].filter((path) => isFieldsAsset(path, datasetPath)); -}; - -const flattenFields = ( - fields: InputField[], - prefix = '' -): Record => { - return fields.reduce((acc, field) => { - const fqn = prefix ? `${prefix}.${field.name}` : field.name; - - if (isGroupField(field)) { - return Object.assign(acc, flattenFields(field.fields || [], fqn)); - } - - const integrationFieldMetadata = { - description: field.description, - flat_name: fqn, - name: field.name, - type: field.type, - }; - - acc[fqn] = integrationFieldMetadata; - return acc; - }, {} as Record); -}; - -const isGroupField = (field: InputField): field is Extract => { - return field.type === 'group'; -}; - -const resolveDataStreamsMap = (dataStreams?: RegistryDataStream[]) => { - if (!dataStreams) return new Map(); - - return dataStreams.reduce((dataStreamsMap, dataStream) => { - dataStreamsMap.set(dataStream.dataset, { - datasetName: dataStream.dataset, - datasetPath: dataStream.path, + return packageService.asInternalUser.getPackageFieldsMetadata({ + packageName: integration, + datasetName: dataset, }); - return dataStreamsMap; - }, new Map() as Map); -}; - -const resolveDataStreamFields = ({ - dataStream, - assetsMap, -}: { - dataStream: RegistryDataStream; - assetsMap: AssetsMap; -}) => { - const { datasetName, datasetPath } = dataStream; - const fieldsAssetPaths = getFieldAssetPaths(assetsMap, datasetPath); - - const fields = fieldsAssetPaths.reduce((fieldsMap, path) => { - const fieldsAsset = assetsMap.get(path); - if (fieldsAsset) { - const fieldsAssetJSON = load(fieldsAsset.toString('utf8')); - const flattenedFields = flattenFields(fieldsAssetJSON); - Object.assign(fieldsMap, flattenedFields); - } - - return fieldsMap; - }, {} as Record); - - return { - [datasetName]: fields, - }; + }); }; From 5c8cd73773344197c6607f307df36a4b069bdb79 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 22 May 2024 13:19:05 +0200 Subject: [PATCH 15/50] refactor(fields-metadata): update FieldMetadata defaults --- .../fields_metadata/models/field_metadata.ts | 24 +++++++++++++++---- .../common/fields_metadata/types.ts | 1 + .../use_fields_metadata.ts | 11 ++++----- .../integration_fields_repository.ts | 9 +++++-- .../fields_metadata/repositories/types.ts | 4 ++-- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts index 0fe3a7236190f..1738ee5db773d 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts @@ -5,8 +5,10 @@ * 2.0. */ +/* eslint-disable @typescript-eslint/naming-convention */ + import pick from 'lodash/pick'; -import { FieldAttribute, FieldMetadataPlain } from '../types'; +import { FieldAttribute, FieldMetadataPlain, PartialFieldMetadataPlain } from '../types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldMetadata extends FieldMetadataPlain {} @@ -23,12 +25,24 @@ export class FieldMetadata { return Object.assign({}, this); } - public static create(fieldMetadata: FieldMetadataPlain) { + public static create(fieldMetadata: PartialFieldMetadataPlain) { + const name = fieldMetadata.name ?? ''; + const flat_name = fieldMetadata.flat_name ?? name; + const dashed_name = fieldMetadata.dashed_name ?? FieldMetadata.toDashedName(flat_name); + const normalize = fieldMetadata.normalize ?? []; + const short = fieldMetadata.short ?? fieldMetadata.description; + const source = fieldMetadata.source ?? 'unknown'; + const type = fieldMetadata.type ?? 'unknown'; + const fieldMetadataProps = { ...fieldMetadata, - dashed_name: fieldMetadata.dashed_name ?? FieldMetadata.toDashedName(fieldMetadata.flat_name), - normalize: fieldMetadata.normalize ?? [], - short: fieldMetadata.short ?? fieldMetadata.description, + name, + flat_name, + dashed_name, + normalize, + short, + source, + type, }; return new FieldMetadata(fieldMetadataProps); diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index b121dba5349d2..80e922e2f1300 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -11,6 +11,7 @@ import * as rt from 'io-ts'; export const fieldSourceRT = rt.keyof({ ecs: null, integration: null, + unknown: null, }); export const allowedValueRT = rt.intersection([ diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index ea0bb707f9759..b21fcfaff8394 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -8,18 +8,17 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; import hash from 'object-hash'; -import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; -import { FieldAttribute, FieldName } from '../../../common'; +import { + FindFieldsMetadataRequestQuery, + FindFieldsMetadataResponsePayload, +} from '../../../common/latest'; import { IFieldsMetadataClient } from '../../services/fields_metadata'; interface UseFieldsMetadataFactoryDeps { fieldsMetadataClient: IFieldsMetadataClient; } -interface UseFieldsMetadataParams { - attributes?: FieldAttribute[]; - fieldNames?: FieldName[]; -} +type UseFieldsMetadataParams = FindFieldsMetadataRequestQuery; interface UseFieldsMetadataReturnType { fieldsMetadata: FindFieldsMetadataResponsePayload['fields'] | undefined; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index d4dec3afeb224..d9febd15feb56 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -7,7 +7,12 @@ import hash from 'object-hash'; import { HashedCache } from '../../../../common/hashed_cache'; -import { FieldMetadata, FieldMetadataPlain, IntegrationFieldName } from '../../../../common'; +import { + FieldMetadata, + FieldMetadataPlain, + IntegrationFieldName, + PartialFieldMetadataPlain, +} from '../../../../common'; import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; interface IntegrationFieldsRepositoryDeps { integrationFieldsExtractor: IntegrationFieldsExtractor; @@ -106,7 +111,7 @@ export class IntegrationFieldsRepository { private getCacheKey = (params: IntegrationFieldsSearchParams) => hash(params); private mapExtractedFieldsToFieldMetadataInstances = ( - extractedFields: Record> + extractedFields: Record> ) => { return Object.entries(extractedFields).reduce( (integrationGroup, [datasetName, datasetGroup]) => { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index 6d19e0ae2d5ae..e2f38e7acba81 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,14 +5,14 @@ * 2.0. */ -import type { PartialFieldMetadataPlain } from '../../../../common'; +import type { FieldMetadataPlain } from '../../../../common'; export interface IntegrationFieldsSearchParams { integration: string; dataset?: string; } -export type ExtractedIntegrationFields = Record>; +export type ExtractedIntegrationFields = Record>; export type IntegrationFieldsExtractor = ({ integration, From 2a214aae97bc8c166f07aa9f83df84aa88cce0fe Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 22 May 2024 16:11:42 +0200 Subject: [PATCH 16/50] refactor(fields-metadata): update custom errors --- .../fields_metadata/find_fields_metadata.ts | 9 +++++ .../server/services/fields_metadata/errors.ts | 8 ++++ .../integration_fields_repository.ts | 38 +++++++++---------- .../fields_metadata/repositories/types.ts | 4 +- 4 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index dd0bcbaf6177f..b3a969d99837c 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -10,6 +10,7 @@ import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; import * as fieldsMetadataV1 from '../../../common/fields_metadata/v1'; import { FieldsMetadataBackendLibs } from '../../lib/shared_types'; import { FindFieldsMetadataResponsePayload } from '../../../common/fields_metadata/v1'; +import { PackageNotFoundError } from '../../services/fields_metadata/errors'; export const initFindFieldsMetadataRoute = ({ router, @@ -54,6 +55,14 @@ export const initFindFieldsMetadataRoute = ({ body: fieldsMetadataV1.findFieldsMetadataResponsePayloadRT.encode(responsePayload), }); } catch (error) { + if (error instanceof PackageNotFoundError) { + return response.badRequest({ + body: { + message: error.message, + }, + }); + } + return response.customError({ statusCode: error.statusCode ?? 500, body: { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts new file mode 100644 index 0000000000000..ea99d78cd2d66 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class PackageNotFoundError extends Error {} diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index d9febd15feb56..c0ff34f298bb9 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -5,21 +5,19 @@ * 2.0. */ -import hash from 'object-hash'; import { HashedCache } from '../../../../common/hashed_cache'; -import { - FieldMetadata, - FieldMetadataPlain, - IntegrationFieldName, - PartialFieldMetadataPlain, -} from '../../../../common'; +import { FieldMetadata, IntegrationFieldName, PartialFieldMetadataPlain } from '../../../../common'; import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; +import { PackageNotFoundError } from '../errors'; interface IntegrationFieldsRepositoryDeps { integrationFieldsExtractor: IntegrationFieldsExtractor; } export class IntegrationFieldsRepository { - private cache: HashedCache>>; + private cache: HashedCache< + IntegrationFieldsSearchParams, + Record> + >; private constructor(private readonly fieldsExtractor: IntegrationFieldsExtractor) { this.cache = new HashedCache(); @@ -32,7 +30,11 @@ export class IntegrationFieldsRepository { let field = this.getCachedField(fieldName, { integration, dataset }); if (!field) { - await this.extractFields({ integration, dataset }); + try { + await this.extractFields({ integration, dataset }); + } catch (error) { + throw new PackageNotFoundError(error.message); + } field = this.getCachedField(fieldName, { integration, dataset }); } @@ -40,10 +42,6 @@ export class IntegrationFieldsRepository { return field; } - async find({ fieldNames }: { fieldNames?: IntegrationFieldName[] } = {}) { - throw new Error('TODO: Implement the IntegrationFieldsRepository#getByName'); - } - public static create({ integrationFieldsExtractor }: IntegrationFieldsRepositoryDeps) { return new IntegrationFieldsRepository(integrationFieldsExtractor); } @@ -60,7 +58,7 @@ export class IntegrationFieldsRepository { } return this.fieldsExtractor({ integration, dataset }) - .then(this.mapExtractedFieldsToFieldMetadataInstances) + .then(this.mapExtractedFieldsToFieldMetadataTree) .then((fieldMetadataTree) => this.storeFieldsInCache(cacheKey, fieldMetadataTree)); } @@ -96,7 +94,7 @@ export class IntegrationFieldsRepository { } private storeFieldsInCache = ( - cacheKey: string, + cacheKey: IntegrationFieldsSearchParams, extractedFieldsMetadata: Record> ): void => { const cachedIntegration = this.cache.get(cacheKey); @@ -108,17 +106,17 @@ export class IntegrationFieldsRepository { } }; - private getCacheKey = (params: IntegrationFieldsSearchParams) => hash(params); + private getCacheKey = (params: IntegrationFieldsSearchParams) => params; - private mapExtractedFieldsToFieldMetadataInstances = ( + private mapExtractedFieldsToFieldMetadataTree = ( extractedFields: Record> ) => { return Object.entries(extractedFields).reduce( (integrationGroup, [datasetName, datasetGroup]) => { integrationGroup[datasetName] = Object.entries(datasetGroup).reduce( - (datasetGroupResult, [extractedFieldName, extractedField]) => { - datasetGroupResult[extractedFieldName] = FieldMetadata.create({ - ...extractedField, + (datasetGroupResult, [fieldName, field]) => { + datasetGroupResult[fieldName] = FieldMetadata.create({ + ...field, source: 'integration', }); return datasetGroupResult; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index e2f38e7acba81..6d19e0ae2d5ae 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,14 +5,14 @@ * 2.0. */ -import type { FieldMetadataPlain } from '../../../../common'; +import type { PartialFieldMetadataPlain } from '../../../../common'; export interface IntegrationFieldsSearchParams { integration: string; dataset?: string; } -export type ExtractedIntegrationFields = Record>; +export type ExtractedIntegrationFields = Record>; export type IntegrationFieldsExtractor = ({ integration, From 1940bbde5f283b818482b157c069634fe5bcb471 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 22 May 2024 18:34:03 +0200 Subject: [PATCH 17/50] refactor(fields-metadata): update typing issues --- .../public/hooks/use_fields_metadata/index.ts | 8 ++ .../use_fields_metadata.test.ts | 115 ++++++++++++++++++ .../use_fields_metadata.ts | 4 +- .../plugins/fields_metadata/public/mocks.tsx | 8 +- .../plugins/fields_metadata/public/plugin.ts | 2 +- .../plugins/fields_metadata/server/mocks.ts | 20 ++- .../fields_metadata_client.test.ts | 33 ++++- .../fields_metadata_service.mock.ts | 4 +- .../fields_metadata_service.ts | 2 +- 9 files changed, 172 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts index e69de29bb2d1d..6aacffe118884 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './use_fields_metadata'; diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts index e69de29bb2d1d..a0951604cd7bc 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { ALERT_STATUS, ValidFeatureId } from '@kbn/rule-data-utils'; + +import { createUseFieldsMetadataHook } from './use_fields_metadata'; +import { KibanaReactContextValue, useKibana } from '@kbn/kibana-react-plugin/public'; +import { coreMock } from '@kbn/core/public/mocks'; +import { CoreStart } from '@kbn/core/public'; +import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; +import { createFieldsMetadataClientMock } from '../../services/fields_metadata/fields_metadata_client.mock'; +import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; + +const mockedFieldsMetadataResponse: FindFieldsMetadataResponsePayload = { + fields: { + '@timestamp': { + dashed_name: 'timestamp', + description: + 'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', + flat_name: '@timestamp', + level: 'core', + name: '@timestamp', + normalize: [], + short: 'Date/time when the event originated.', + type: 'date', + source: 'ecs', + }, + }, +}; + +const expectedResult = { + activeAlertCount: 2, + recoveredAlertCount: 20, +}; + +jest.mock('@kbn/kibana-react-plugin/public'); + +const fieldsMetadataClient = createFieldsMetadataClientMock(); +fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); + +const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataClient }); + +describe('useFieldsMetadata', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + // it('should return the mocked data from API', async () => { + // mockedPostAPI.mockResolvedValue(mockedFieldsMetadataResponse); + + // const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata({ featureIds })); + + // expect(result.current.loading).toBe(true); + // expect(result.current.alertsCount).toEqual(undefined); + + // await waitForNextUpdate(); + + // const { alertsCount, loading, error } = result.current; + // expect(alertsCount).toEqual(expectedResult); + // expect(loading).toBeFalsy(); + // expect(error).toBeFalsy(); + // }); + + // it('should call API with correct input', async () => { + // const ruleId = 'c95bc120-1d56-11ed-9cc7-e7214ada1128'; + // const query = { + // term: { + // 'kibana.alert.rule.uuid': ruleId, + // }, + // }; + // mockedPostAPI.mockResolvedValue(mockedFieldsMetadataResponse); + + // const { waitForNextUpdate } = renderHook(() => + // useFieldsMetadata({ + // featureIds, + // query, + // }) + // ); + + // await waitForNextUpdate(); + + // const body = JSON.stringify({ + // aggs: { + // count: { + // terms: { field: ALERT_STATUS }, + // }, + // }, + // feature_ids: featureIds, + // query, + // size: 0, + // }); + + // expect(mockedPostAPI).toHaveBeenCalledWith( + // FIND_FIELDS_METADATA_URL, + // expect.objectContaining({ body }) + // ); + // }); + + // it('should return error if API call fails', async () => { + // const error = new Error('Fetch Alerts Count Failed'); + // mockedPostAPI.mockRejectedValueOnce(error); + + // const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata({ featureIds })); + + // await waitForNextUpdate(); + + // expect(result.current.error?.message).toMatch(error.message); + // }); +}); diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index b21fcfaff8394..3d6315a5037c7 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -18,9 +18,9 @@ interface UseFieldsMetadataFactoryDeps { fieldsMetadataClient: IFieldsMetadataClient; } -type UseFieldsMetadataParams = FindFieldsMetadataRequestQuery; +export type UseFieldsMetadataParams = FindFieldsMetadataRequestQuery; -interface UseFieldsMetadataReturnType { +export interface UseFieldsMetadataReturnType { fieldsMetadata: FindFieldsMetadataResponsePayload['fields'] | undefined; loading: boolean; error: Error | undefined; diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index 4d6b542cf5c00..09e6f8172427e 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -8,10 +8,10 @@ import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; import { FieldsMetadataClientStart } from './types'; -export const createFieldsMetadataPluginStartMock = - (): jest.Mocked => ({ - fieldsMetadata: createFieldsMetadataServiceStartMock(), - }); +export const createFieldsMetadataPluginStartMock = (): jest.Mocked => ({ + client: createFieldsMetadataServiceStartMock().client, + useFieldsMetadata: jest.fn(), +}); export const _ensureTypeCompatibility = (): FieldsMetadataClientStart => createFieldsMetadataPluginStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts index ade8954335100..79e45a539064a 100644 --- a/x-pack/plugins/fields_metadata/public/plugin.ts +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -6,7 +6,7 @@ */ import { CoreStart } from '@kbn/core/public'; -import { createUseFieldsMetadataHook } from './hooks/use_fields_metadata/use_fields_metadata'; +import { createUseFieldsMetadataHook } from './hooks/use_fields_metadata'; import { FieldsMetadataService } from './services/fields_metadata'; import { FieldsMetadataClientCoreSetup, diff --git a/x-pack/plugins/fields_metadata/server/mocks.ts b/x-pack/plugins/fields_metadata/server/mocks.ts index aaec4c0a85afd..60eefff36d750 100644 --- a/x-pack/plugins/fields_metadata/server/mocks.ts +++ b/x-pack/plugins/fields_metadata/server/mocks.ts @@ -11,20 +11,14 @@ import { } from './services/fields_metadata/fields_metadata_service.mock'; import { FieldsMetadataServerSetup, FieldsMetadataServerStart } from './types'; -const createFieldsMetadataSetupMock = () => { - const fieldsMetadataSetupMock: jest.Mocked = { - fieldsMetadata: createFieldsMetadataServiceSetupMock(), - }; +const createFieldsMetadataSetupMock = (): jest.Mocked => ({ + registerIntegrationFieldsExtractor: + createFieldsMetadataServiceSetupMock().registerIntegrationFieldsExtractor, +}); - return fieldsMetadataSetupMock; -}; - -const createFieldsMetadataStartMock = () => { - const fieldsMetadataStartMock: jest.Mocked = { - fieldsMetadata: createFieldsMetadataServiceStartMock(), - }; - return fieldsMetadataStartMock; -}; +const createFieldsMetadataStartMock = (): jest.Mocked => ({ + client: createFieldsMetadataServiceStartMock().getClient(), +}); export const fieldsMetadataPluginMock = { createSetupContract: createFieldsMetadataSetupMock, diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts index b13f9fa973167..97cf572d54c22 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts @@ -4,11 +4,40 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { TEcsFields } from '../../../common'; +import { loggerMock } from '@kbn/logging-mocks'; import { FieldsMetadataClient } from './fields_metadata_client'; +import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; +import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; + +const ecsFields = { + '@timestamp': { + dashed_name: 'timestamp', + description: + 'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', + flat_name: '@timestamp', + level: 'core', + name: '@timestamp', + normalize: [], + required: !0, + short: 'Date/time when the event originated.', + type: 'date', + }, +} as TEcsFields; describe('FieldsMetadataClient class', () => { - const fieldsMetadataClient = FieldsMetadataClient.create(); + const logger = loggerMock.create(); + const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); + const integrationFieldsRepository = IntegrationFieldsRepository.create({ + integrationFieldsExtractor: () => Promise.resolve({}), + }); + + const fieldsMetadataClient = FieldsMetadataClient.create({ + ecsFieldsRepository, + integrationFieldsRepository, + logger, + }); it('#getByName resolves a single ecs field', () => { const timestampField = fieldsMetadataClient.getByName('@timestamp'); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts index 631019c64f967..6fab587c9ca7a 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.mock.ts @@ -9,7 +9,9 @@ import { createFieldsMetadataClientMock } from './fields_metadata_client.mock'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; export const createFieldsMetadataServiceSetupMock = - (): jest.Mocked => ({}); + (): jest.Mocked => ({ + registerIntegrationFieldsExtractor: jest.fn(), + }); export const createFieldsMetadataServiceStartMock = (): jest.Mocked => ({ diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index 6934b3a61e7ef..391da465e9a1f 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -14,7 +14,7 @@ import { IntegrationFieldsExtractor } from './repositories/types'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; export class FieldsMetadataService { - private integrationFieldsExtractor: IntegrationFieldsExtractor = () => ({}); + private integrationFieldsExtractor: IntegrationFieldsExtractor = () => Promise.resolve({}); constructor(private readonly logger: Logger) {} From 065918e40d19d38cb8306e059aa0c68ce91fd9f9 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 22 May 2024 18:39:24 +0200 Subject: [PATCH 18/50] test(fields-metadata): add first useFieldsMetadata test --- .../use_fields_metadata.test.ts | 62 +++++++------------ .../use_fields_metadata.ts | 2 +- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts index a0951604cd7bc..db0a45ad98750 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts @@ -6,40 +6,28 @@ */ import { renderHook } from '@testing-library/react-hooks'; -import { ALERT_STATUS, ValidFeatureId } from '@kbn/rule-data-utils'; import { createUseFieldsMetadataHook } from './use_fields_metadata'; -import { KibanaReactContextValue, useKibana } from '@kbn/kibana-react-plugin/public'; -import { coreMock } from '@kbn/core/public/mocks'; -import { CoreStart } from '@kbn/core/public'; -import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; import { createFieldsMetadataClientMock } from '../../services/fields_metadata/fields_metadata_client.mock'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; -const mockedFieldsMetadataResponse: FindFieldsMetadataResponsePayload = { - fields: { - '@timestamp': { - dashed_name: 'timestamp', - description: - 'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.', - example: '2016-05-23T08:05:34.853Z', - flat_name: '@timestamp', - level: 'core', - name: '@timestamp', - normalize: [], - short: 'Date/time when the event originated.', - type: 'date', - source: 'ecs', - }, +const fields: FindFieldsMetadataResponsePayload['fields'] = { + '@timestamp': { + dashed_name: 'timestamp', + description: + 'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', + flat_name: '@timestamp', + level: 'core', + name: '@timestamp', + normalize: [], + short: 'Date/time when the event originated.', + type: 'date', + source: 'ecs', }, }; -const expectedResult = { - activeAlertCount: 2, - recoveredAlertCount: 20, -}; - -jest.mock('@kbn/kibana-react-plugin/public'); +const mockedFieldsMetadataResponse: FindFieldsMetadataResponsePayload = { fields }; const fieldsMetadataClient = createFieldsMetadataClientMock(); fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); @@ -51,21 +39,19 @@ describe('useFieldsMetadata', () => { jest.clearAllMocks(); }); - // it('should return the mocked data from API', async () => { - // mockedPostAPI.mockResolvedValue(mockedFieldsMetadataResponse); - - // const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata({ featureIds })); + it('should return the fields record from API', async () => { + const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata()); - // expect(result.current.loading).toBe(true); - // expect(result.current.alertsCount).toEqual(undefined); + expect(result.current.loading).toBe(true); + expect(result.current.fieldsMetadata).toEqual(undefined); - // await waitForNextUpdate(); + await waitForNextUpdate(); - // const { alertsCount, loading, error } = result.current; - // expect(alertsCount).toEqual(expectedResult); - // expect(loading).toBeFalsy(); - // expect(error).toBeFalsy(); - // }); + const { fieldsMetadata, loading, error } = result.current; + expect(fieldsMetadata).toEqual(fields); + expect(loading).toBeFalsy(); + expect(error).toBeFalsy(); + }); // it('should call API with correct input', async () => { // const ruleId = 'c95bc120-1d56-11ed-9cc7-e7214ada1128'; diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 3d6315a5037c7..ae28dc417fdc2 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -27,7 +27,7 @@ export interface UseFieldsMetadataReturnType { } export type UseFieldsMetadataHook = ( - params: UseFieldsMetadataParams + params?: UseFieldsMetadataParams ) => UseFieldsMetadataReturnType; export const createUseFieldsMetadataHook = ({ From 8caf30b95f85902127e6fc22013bfced05178d40 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 09:50:12 +0200 Subject: [PATCH 19/50] test(fields-metadata): update useFieldsMetadata test --- x-pack/plugins/fields_metadata/jest.config.js | 2 +- .../use_fields_metadata.test.ts | 79 +++++++------------ .../use_fields_metadata.ts | 16 ++-- 3 files changed, 41 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/fields_metadata/jest.config.js b/x-pack/plugins/fields_metadata/jest.config.js index 47ba6f1a113bd..3c1d51b335696 100644 --- a/x-pack/plugins/fields_metadata/jest.config.js +++ b/x-pack/plugins/fields_metadata/jest.config.js @@ -7,7 +7,7 @@ module.exports = { preset: '@kbn/test', - rootDir: '../../../..', + rootDir: '../../..', roots: ['/x-pack/plugins/fields_metadata'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/fields_metadata', coverageReporters: ['text', 'html'], diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts index db0a45ad98750..3321ec1737076 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts @@ -7,7 +7,7 @@ import { renderHook } from '@testing-library/react-hooks'; -import { createUseFieldsMetadataHook } from './use_fields_metadata'; +import { createUseFieldsMetadataHook, UseFieldsMetadataParams } from './use_fields_metadata'; import { createFieldsMetadataClientMock } from '../../services/fields_metadata/fields_metadata_client.mock'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; @@ -27,10 +27,9 @@ const fields: FindFieldsMetadataResponsePayload['fields'] = { }, }; -const mockedFieldsMetadataResponse: FindFieldsMetadataResponsePayload = { fields }; +const mockedFieldsMetadataResponse = { fields }; const fieldsMetadataClient = createFieldsMetadataClientMock(); -fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataClient }); @@ -39,7 +38,8 @@ describe('useFieldsMetadata', () => { jest.clearAllMocks(); }); - it('should return the fields record from API', async () => { + it('should return the fieldsMetadata value from the API', async () => { + fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata()); expect(result.current.loading).toBe(true); @@ -53,49 +53,30 @@ describe('useFieldsMetadata', () => { expect(error).toBeFalsy(); }); - // it('should call API with correct input', async () => { - // const ruleId = 'c95bc120-1d56-11ed-9cc7-e7214ada1128'; - // const query = { - // term: { - // 'kibana.alert.rule.uuid': ruleId, - // }, - // }; - // mockedPostAPI.mockResolvedValue(mockedFieldsMetadataResponse); - - // const { waitForNextUpdate } = renderHook(() => - // useFieldsMetadata({ - // featureIds, - // query, - // }) - // ); - - // await waitForNextUpdate(); - - // const body = JSON.stringify({ - // aggs: { - // count: { - // terms: { field: ALERT_STATUS }, - // }, - // }, - // feature_ids: featureIds, - // query, - // size: 0, - // }); - - // expect(mockedPostAPI).toHaveBeenCalledWith( - // FIND_FIELDS_METADATA_URL, - // expect.objectContaining({ body }) - // ); - // }); - - // it('should return error if API call fails', async () => { - // const error = new Error('Fetch Alerts Count Failed'); - // mockedPostAPI.mockRejectedValueOnce(error); - - // const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata({ featureIds })); - - // await waitForNextUpdate(); - - // expect(result.current.error?.message).toMatch(error.message); - // }); + it('should call the fieldsMetadata service with the passed parameters', async () => { + fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); + const params: UseFieldsMetadataParams = { + attributes: ['description', 'short'], + fieldNames: ['@timestamp', 'agent.name'], + integration: 'integration_name', + dataset: 'dataset_name', + }; + + const { waitForNextUpdate } = renderHook(() => useFieldsMetadata(params)); + + await waitForNextUpdate(); + + expect(fieldsMetadataClient.find).toHaveBeenCalledWith(params); + }); + + it('should return an error if the API call fails', async () => { + const error = new Error('Fetch fields metadata Failed'); + fieldsMetadataClient.find.mockRejectedValueOnce(error); + + const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata()); + + await waitForNextUpdate(); + + expect(result.current.error?.message).toMatch(error.message); + }); }); diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index ae28dc417fdc2..698e5e4da83d7 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -8,22 +8,26 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; import hash from 'object-hash'; -import { - FindFieldsMetadataRequestQuery, - FindFieldsMetadataResponsePayload, -} from '../../../common/latest'; +import { FieldAttribute, FieldName } from '../../../common'; +import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; import { IFieldsMetadataClient } from '../../services/fields_metadata'; interface UseFieldsMetadataFactoryDeps { fieldsMetadataClient: IFieldsMetadataClient; } -export type UseFieldsMetadataParams = FindFieldsMetadataRequestQuery; +export interface UseFieldsMetadataParams { + attributes?: FieldAttribute[]; + fieldNames?: FieldName[]; + integration?: string; + dataset?: string; +} export interface UseFieldsMetadataReturnType { fieldsMetadata: FindFieldsMetadataResponsePayload['fields'] | undefined; loading: boolean; error: Error | undefined; + reload: ReturnType[1]; } export type UseFieldsMetadataHook = ( @@ -45,6 +49,6 @@ export const createUseFieldsMetadataHook = ({ load(); }, [load]); - return { fieldsMetadata: value?.fields, loading, error }; + return { fieldsMetadata: value?.fields, loading, error, reload: load }; }; }; From 2771b81a0f933e114106487309319e6538d38d44 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 10:48:12 +0200 Subject: [PATCH 20/50] test(fields-metadata): update FieldsMetadataClient test --- .../fields_metadata_client.test.ts | 159 +++++++++++++++--- 1 file changed, 132 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts index 97cf572d54c22..0d35dc70fd678 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { TEcsFields } from '../../../common'; +import { FieldMetadata, TEcsFields } from '../../../common'; import { loggerMock } from '@kbn/logging-mocks'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; @@ -26,41 +26,146 @@ const ecsFields = { }, } as TEcsFields; +const integrationFields = { + '1password.item_usages': { + 'onepassword.client.platform_version': { + name: 'platform_version', + type: 'keyword', + description: + 'The version of the browser or computer where the 1Password app is installed, or the CPU of the machine where the 1Password command-line tool is installed', + flat_name: 'onepassword.client.platform_version', + source: 'integration', + dashed_name: 'onepassword-client-platform_version', + normalize: [], + short: + 'The version of the browser or computer where the 1Password app is installed, or the CPU of the machine where the 1Password command-line tool is installed', + }, + }, +}; + describe('FieldsMetadataClient class', () => { const logger = loggerMock.create(); const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); - const integrationFieldsRepository = IntegrationFieldsRepository.create({ - integrationFieldsExtractor: () => Promise.resolve({}), - }); + const integrationFieldsExtractor = jest.fn(); + integrationFieldsExtractor.mockImplementation(() => Promise.resolve(integrationFields)); - const fieldsMetadataClient = FieldsMetadataClient.create({ - ecsFieldsRepository, - integrationFieldsRepository, - logger, + let integrationFieldsRepository: IntegrationFieldsRepository; + let fieldsMetadataClient: FieldsMetadataClient; + + beforeEach(() => { + integrationFieldsExtractor.mockClear(); + integrationFieldsRepository = IntegrationFieldsRepository.create({ + integrationFieldsExtractor, + }); + fieldsMetadataClient = FieldsMetadataClient.create({ + ecsFieldsRepository, + integrationFieldsRepository, + logger, + }); }); - it('#getByName resolves a single ecs field', () => { - const timestampField = fieldsMetadataClient.getByName('@timestamp'); - - expect(timestampField.hasOwnProperty('dashed_name')).toBeTruthy(); - expect(timestampField.hasOwnProperty('description')).toBeTruthy(); - expect(timestampField.hasOwnProperty('example')).toBeTruthy(); - expect(timestampField.hasOwnProperty('flat_name')).toBeTruthy(); - expect(timestampField.hasOwnProperty('level')).toBeTruthy(); - expect(timestampField.hasOwnProperty('name')).toBeTruthy(); - expect(timestampField.hasOwnProperty('normalize')).toBeTruthy(); - expect(timestampField.hasOwnProperty('required')).toBeTruthy(); - expect(timestampField.hasOwnProperty('short')).toBeTruthy(); - expect(timestampField.hasOwnProperty('type')).toBeTruthy(); + describe('#getByName', () => { + it('should resolve a single ECS FieldMetadata instance by default', async () => { + const timestampFieldInstance = await fieldsMetadataClient.getByName('@timestamp'); + + expect(integrationFieldsExtractor).not.toHaveBeenCalled(); + + expectToBeDefined(timestampFieldInstance); + expect(timestampFieldInstance).toBeInstanceOf(FieldMetadata); + + const timestampField = timestampFieldInstance.toPlain(); + + expect(timestampField.hasOwnProperty('dashed_name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('description')).toBeTruthy(); + expect(timestampField.hasOwnProperty('example')).toBeTruthy(); + expect(timestampField.hasOwnProperty('flat_name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('level')).toBeTruthy(); + expect(timestampField.hasOwnProperty('name')).toBeTruthy(); + expect(timestampField.hasOwnProperty('normalize')).toBeTruthy(); + expect(timestampField.hasOwnProperty('required')).toBeTruthy(); + expect(timestampField.hasOwnProperty('short')).toBeTruthy(); + expect(timestampField.hasOwnProperty('type')).toBeTruthy(); + }); + + it('should attempt resolving the field from an integration if it does not exist in ECS and the integration and dataset params are provided', async () => { + const onePasswordFieldInstance = await fieldsMetadataClient.getByName( + 'onepassword.client.platform_version', + { integration: '1password', dataset: '1password.item_usages' } + ); + + expect(integrationFieldsExtractor).toHaveBeenCalled(); + + expectToBeDefined(onePasswordFieldInstance); + expect(onePasswordFieldInstance).toBeInstanceOf(FieldMetadata); + + const onePasswordField = onePasswordFieldInstance.toPlain(); + + expect(onePasswordField.hasOwnProperty('name')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('type')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('description')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('flat_name')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('source')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('dashed_name')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('normalize')).toBeTruthy(); + expect(onePasswordField.hasOwnProperty('short')).toBeTruthy(); + }); + + it('should not resolve the field from an integration if the integration and dataset params are not provided', async () => { + const onePasswordFieldInstance = await fieldsMetadataClient.getByName( + 'onepassword.client.platform_version' + ); + + expect(integrationFieldsExtractor).not.toHaveBeenCalled(); + expect(onePasswordFieldInstance).toBeUndefined(); + }); }); - it('#find resolves a dictionary of matching fields', async () => { - const fields = fieldsMetadataClient.find({ - fieldNames: ['@timestamp', 'message', 'not-existing-field'], + describe('#find', () => { + it('should resolve a FieldsMetadataDictionary of matching fields', async () => { + const fieldsDictionaryInstance = await fieldsMetadataClient.find({ + fieldNames: ['@timestamp'], + }); + + expect(integrationFieldsExtractor).not.toHaveBeenCalled(); + + const fields = fieldsDictionaryInstance.toPlain(); + + expect(fields.hasOwnProperty('@timestamp')).toBeTruthy(); + }); + + it('should resolve a FieldsMetadataDictionary of matching fields, including integration fields when integration and dataset params are provided', async () => { + const fieldsDictionaryInstance = await fieldsMetadataClient.find({ + fieldNames: ['@timestamp', 'onepassword.client.platform_version'], + integration: '1password', + dataset: '1password.item_usages', + }); + + expect(integrationFieldsExtractor).toHaveBeenCalled(); + + const fields = fieldsDictionaryInstance.toPlain(); + + expect(fields.hasOwnProperty('@timestamp')).toBeTruthy(); + expect(fields.hasOwnProperty('onepassword.client.platform_version')).toBeTruthy(); }); - expect(fields.hasOwnProperty('@timestamp')).toBeTruthy(); - expect(fields.hasOwnProperty('message')).toBeTruthy(); - expect(fields.hasOwnProperty('not-existing-field')).toBeFalsy(); + it('should resolve a FieldsMetadataDictionary of matching fields, skipping unmatched fields', async () => { + const fieldsDictionaryInstance = await fieldsMetadataClient.find({ + fieldNames: ['@timestamp', 'onepassword.client.platform_version', 'not-existing-field'], + integration: '1password', + dataset: '1password.item_usages', + }); + + expect(integrationFieldsExtractor).toHaveBeenCalled(); + + const fields = fieldsDictionaryInstance.toPlain(); + + expect(fields.hasOwnProperty('@timestamp')).toBeTruthy(); + expect(fields.hasOwnProperty('onepassword.client.platform_version')).toBeTruthy(); + expect(fields.hasOwnProperty('not-existing-field')).toBeFalsy(); + }); }); }); + +function expectToBeDefined(value: T | undefined): asserts value is T { + expect(value).toBeDefined(); +} From 77e28f7a2d124edb0bf9fdd2b59b17d462fa547d Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 11:31:31 +0200 Subject: [PATCH 21/50] refactor(fields-metadata): remove stop lifecycle --- x-pack/plugins/fields_metadata/public/plugin.ts | 2 -- x-pack/plugins/fields_metadata/server/plugin.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts index 79e45a539064a..a99a3438685d0 100644 --- a/x-pack/plugins/fields_metadata/public/plugin.ts +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -40,6 +40,4 @@ export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { useFieldsMetadata, }; } - - public stop() {} } diff --git a/x-pack/plugins/fields_metadata/server/plugin.ts b/x-pack/plugins/fields_metadata/server/plugin.ts index 68eda678edeaa..25c35c65106a2 100644 --- a/x-pack/plugins/fields_metadata/server/plugin.ts +++ b/x-pack/plugins/fields_metadata/server/plugin.ts @@ -61,6 +61,4 @@ export class FieldsMetadataPlugin return { client }; } - - public stop() {} } From 0b891f619721a75b8a18d9b9c45ca52a0d189e19 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 11:32:47 +0200 Subject: [PATCH 22/50] chore(fields-metadata): add bundle limits --- packages/kbn-optimizer/limits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 47d3d0c55c290..e01799bccc615 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -179,3 +179,4 @@ pageLoadAssetSize: visTypeXy: 46868 visualizations: 90000 watcher: 43598 + fieldsMetadata: 82448 From 4b2ebe35ac44eb8c5e7a75844c839edfe8c6a8ef Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 14:04:41 +0200 Subject: [PATCH 23/50] refactor(fields-metadata): update start contracts --- .../common/fields_metadata/models/field_metadata.ts | 2 +- .../fields_metadata/models/fields_metadata_dictionary.ts | 4 ++-- x-pack/plugins/fields_metadata/server/mocks.ts | 2 +- x-pack/plugins/fields_metadata/server/plugin.ts | 5 ++--- .../server/routes/fields_metadata/find_fields_metadata.ts | 4 ++-- x-pack/plugins/fields_metadata/server/types.ts | 4 ++-- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts index 1738ee5db773d..beac6a9bfe673 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts @@ -17,7 +17,7 @@ export class FieldMetadata { Object.assign(this, fieldMetadata); } - public pick(props: FieldAttribute[]) { + public pick(props: FieldAttribute[]): PartialFieldMetadataPlain { return pick(this, props); } diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts index b7999b0586392..9039a1eb9f1ea 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts @@ -15,11 +15,11 @@ export class FieldsMetadataDictionary { private constructor(private readonly fields: PartialFieldsMetadataMap) {} pick(attributes: FieldAttribute[]): Record { - return mapValues(this.fields, (field) => field?.pick(attributes)); + return mapValues(this.fields, (field) => field.pick(attributes)); } toPlain(): Record { - return mapValues(this.fields, (field) => field?.toPlain()); + return mapValues(this.fields, (field) => field.toPlain()); } public static create(fields: PartialFieldsMetadataMap = {}) { diff --git a/x-pack/plugins/fields_metadata/server/mocks.ts b/x-pack/plugins/fields_metadata/server/mocks.ts index 60eefff36d750..657e73635ce85 100644 --- a/x-pack/plugins/fields_metadata/server/mocks.ts +++ b/x-pack/plugins/fields_metadata/server/mocks.ts @@ -17,7 +17,7 @@ const createFieldsMetadataSetupMock = (): jest.Mocked }); const createFieldsMetadataStartMock = (): jest.Mocked => ({ - client: createFieldsMetadataServiceStartMock().getClient(), + getClient: createFieldsMetadataServiceStartMock().getClient, }); export const fieldsMetadataPluginMock = { diff --git a/x-pack/plugins/fields_metadata/server/plugin.ts b/x-pack/plugins/fields_metadata/server/plugin.ts index 25c35c65106a2..da7ede5efba3f 100644 --- a/x-pack/plugins/fields_metadata/server/plugin.ts +++ b/x-pack/plugins/fields_metadata/server/plugin.ts @@ -55,10 +55,9 @@ export class FieldsMetadataPlugin }; } - public start(core: CoreStart, plugins: FieldsMetadataServerPluginStartDeps) { + public start(_core: CoreStart, _plugins: FieldsMetadataServerPluginStartDeps) { const fieldsMetadata = this.fieldsMetadataService.start(); - const client = fieldsMetadata.getClient(); - return { client }; + return { getClient: fieldsMetadata.getClient }; } } diff --git a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts index b3a969d99837c..5e518618d98d8 100644 --- a/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/server/routes/fields_metadata/find_fields_metadata.ts @@ -33,8 +33,8 @@ export const initFindFieldsMetadataRoute = ({ async (_requestContext, request, response) => { const { attributes, fieldNames, integration, dataset } = request.query; - const { client } = (await getStartServices())[2]; - const fieldsMetadataClient = client; + const [_core, _startDeps, startContract] = await getStartServices(); + const fieldsMetadataClient = startContract.getClient(); try { const fieldsDictionary = await fieldsMetadataClient.find({ diff --git a/x-pack/plugins/fields_metadata/server/types.ts b/x-pack/plugins/fields_metadata/server/types.ts index 6bc3583f3f598..4e2bf7ce2c0b3 100644 --- a/x-pack/plugins/fields_metadata/server/types.ts +++ b/x-pack/plugins/fields_metadata/server/types.ts @@ -9,7 +9,7 @@ import type { CoreSetup } from '@kbn/core/server'; import { FieldsMetadataServiceSetup, - IFieldsMetadataClient, + FieldsMetadataServiceStart, } from './services/fields_metadata/types'; export type FieldsMetadataPluginCoreSetup = CoreSetup< @@ -24,7 +24,7 @@ export interface FieldsMetadataServerSetup { } export interface FieldsMetadataServerStart { - client: IFieldsMetadataClient; + getClient: FieldsMetadataServiceStart['getClient']; } // eslint-disable-next-line @typescript-eslint/no-empty-interface From ce75a676e631e3ce3ceedb19a1bcf9611e392fbf Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 23 May 2024 12:15:56 +0000 Subject: [PATCH 24/50] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/fields_metadata/tsconfig.json | 1 + x-pack/plugins/fleet/tsconfig.json | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/fields_metadata/tsconfig.json b/x-pack/plugins/fields_metadata/tsconfig.json index bc251c1b6ed79..91fc85b3024ea 100644 --- a/x-pack/plugins/fields_metadata/tsconfig.json +++ b/x-pack/plugins/fields_metadata/tsconfig.json @@ -17,5 +17,6 @@ "@kbn/logging", "@kbn/core-http-request-handler-context-server", "@kbn/core-http-server", + "@kbn/logging-mocks", ] } diff --git a/x-pack/plugins/fleet/tsconfig.json b/x-pack/plugins/fleet/tsconfig.json index 5e525bba938f2..e9150e7e1d559 100644 --- a/x-pack/plugins/fleet/tsconfig.json +++ b/x-pack/plugins/fleet/tsconfig.json @@ -108,5 +108,6 @@ "@kbn/zod-helpers", "@kbn/react-kibana-mount", "@kbn/react-kibana-context-render", + "@kbn/fields-metadata-plugin", ] } From 3702350cd6149bd94a84c05930fb063174566ff5 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 14:20:18 +0200 Subject: [PATCH 25/50] refactor(io-ts-utils): update test --- .../src/array_to_string_rt/index.test.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts index a2d0e1f62e864..4d4603fe269f2 100644 --- a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts +++ b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts @@ -9,19 +9,12 @@ import * as rt from 'io-ts'; import { arrayToStringRt } from '.'; import { isRight, Either, isLeft, fold } from 'fp-ts/lib/Either'; -import { Right } from 'fp-ts/lib/Either'; -import { pipe } from 'fp-ts/lib/pipeable'; import { identity } from 'fp-ts/lib/function'; -function getValueOrThrow>(either: TEither): Right { - const value = pipe( - either, - fold(() => { - throw new Error('cannot get right value of left'); - }, identity) - ); - - return value as Right; +function getValueOrThrow(either: Either) { + return fold(() => { + throw new Error('Cannot get right value of left'); + }, identity)(either); } describe('arrayToStringRt', () => { From e923f540d60400eb89b9dbc71f6ccbfbb5d8db57 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 23 May 2024 12:34:56 +0000 Subject: [PATCH 26/50] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d651c5ab76939..eacb7b774b7f2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -445,6 +445,7 @@ examples/field_formats_example @elastic/kibana-data-discovery src/plugins/field_formats @elastic/kibana-data-discovery packages/kbn-field-types @elastic/kibana-data-discovery packages/kbn-field-utils @elastic/kibana-data-discovery +x-pack/plugins/fields_metadata @elastic/obs-ux-logs-team x-pack/plugins/file_upload @elastic/kibana-gis examples/files_example @elastic/appex-sharedux src/plugins/files_management @elastic/appex-sharedux @@ -535,7 +536,6 @@ examples/locator_examples @elastic/appex-sharedux examples/locator_explorer @elastic/appex-sharedux packages/kbn-logging @elastic/kibana-core packages/kbn-logging-mocks @elastic/kibana-core -x-pack/plugins/fields_metadata @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_data_access @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_explorer @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/logs_shared @elastic/obs-ux-logs-team From 70039b6755b6fd0511934fd69e3fc18a84b3b73b Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 14:54:46 +0200 Subject: [PATCH 27/50] refactor(unified-doc-viewer): remove multiple optional chaining applying default value --- .../logs_overview_highlights.tsx | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx index 6adc3a9b7c07a..f22d17956639f 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx @@ -26,7 +26,7 @@ export function LogsOverviewHighlights({ fieldsMetadata: { useFieldsMetadata }, } = getUnifiedDocViewerServices(); - const { fieldsMetadata } = useFieldsMetadata({ + const { fieldsMetadata = {} } = useFieldsMetadata({ attributes: ['flat_name', 'short', 'type'], fieldNames: [ fieldConstants.SERVICE_NAME_FIELD, @@ -63,7 +63,7 @@ export function LogsOverviewHighlights({ )} @@ -71,7 +71,7 @@ export function LogsOverviewHighlights({ )} @@ -79,7 +79,7 @@ export function LogsOverviewHighlights({ )} @@ -87,7 +87,7 @@ export function LogsOverviewHighlights({ )} @@ -95,7 +95,7 @@ export function LogsOverviewHighlights({ )} @@ -109,7 +109,7 @@ export function LogsOverviewHighlights({ )} @@ -132,7 +132,7 @@ export function LogsOverviewHighlights({ )} @@ -140,7 +140,7 @@ export function LogsOverviewHighlights({ )} @@ -148,7 +148,7 @@ export function LogsOverviewHighlights({ )} @@ -162,7 +162,7 @@ export function LogsOverviewHighlights({ )} @@ -170,7 +170,7 @@ export function LogsOverviewHighlights({ )} @@ -178,7 +178,7 @@ export function LogsOverviewHighlights({ @@ -187,7 +187,7 @@ export function LogsOverviewHighlights({ )} From 891c0db967de510e2856559de23eccf5a28c3141 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 23 May 2024 12:56:14 +0000 Subject: [PATCH 28/50] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- docs/developer/plugin-list.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 5b6b7e4014bc7..9f2aa1c6860b3 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -578,6 +578,10 @@ activities. |The features plugin enhance Kibana with a per-feature privilege system. +|{kib-repo}blob/{branch}/x-pack/plugins/fields_metadata/README.md[fieldsMetadata] +|Exposes services for async usage and search of field metadata. + + |{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload] |WARNING: Missing README. From e6ac4c0f932b6e756cc0ffa0c640d98256a01957 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 15:11:10 +0200 Subject: [PATCH 29/50] refactor(unified-doc-viewer): lift conditional to render field description --- .../sub_components/highlight_field.tsx | 4 +++- .../sub_components/highlight_field_description.tsx | 6 +----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx index ddc8995c52952..b50540c14c7a6 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx @@ -36,13 +36,15 @@ export function HighlightField({ value, ...props }: HighlightFieldProps) { + const hasFieldDescription = !!fieldMetadata?.short; + return formattedValue && value ? (
{label} - + {hasFieldDescription ? : null} {type && } From 3950d42f286483590278e617e9d1062abaaa8083 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 15:25:31 +0200 Subject: [PATCH 30/50] chore(kbn-optimizer): update limits file --- packages/kbn-optimizer/limits.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index e01799bccc615..954cb099d608a 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -30,7 +30,7 @@ pageLoadAssetSize: data: 454087 datasetQuality: 50624 dataViewEditor: 28082 - dataViewFieldEditor: 27000 + dataViewFieldEditor: 42017 dataViewManagement: 5300 dataViews: 65000 dataVisualizer: 27530 @@ -61,6 +61,7 @@ pageLoadAssetSize: expressionXY: 45000 features: 21723 fieldFormats: 65209 + fieldsMetadata: 82448 files: 22673 filesManagement: 18683 fileUpload: 25664 @@ -179,4 +180,3 @@ pageLoadAssetSize: visTypeXy: 46868 visualizations: 90000 watcher: 43598 - fieldsMetadata: 82448 From 71946c5504729a0ebdb3e6087984b8d8817c091a Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 16:08:04 +0200 Subject: [PATCH 31/50] refactor(fields-metadata): minor types changes --- .../public/__mocks__/services.ts | 2 ++ .../unified_doc_viewer/public/plugin.tsx | 4 +-- .../unified_doc_viewer/public/types.ts | 4 +-- .../fields_metadata/models/field_metadata.ts | 10 +++---- .../models/fields_metadata_dictionary.ts | 6 ++--- .../common/fields_metadata/types.ts | 6 ++--- .../plugins/fields_metadata/public/index.ts | 8 +++--- .../plugins/fields_metadata/public/mocks.tsx | 6 ++--- .../plugins/fields_metadata/public/plugin.ts | 8 +++--- .../plugins/fields_metadata/public/types.ts | 26 +++++++++---------- .../integration_fields_repository.ts | 4 +-- .../fields_metadata/repositories/types.ts | 4 +-- .../server/services/epm/packages/utils.ts | 19 +++++--------- .../server/services/epm/registry/index.ts | 4 +-- 14 files changed, 52 insertions(+), 59 deletions(-) diff --git a/src/plugins/unified_doc_viewer/public/__mocks__/services.ts b/src/plugins/unified_doc_viewer/public/__mocks__/services.ts index e9dc74697f358..3d660aaa46a48 100644 --- a/src/plugins/unified_doc_viewer/public/__mocks__/services.ts +++ b/src/plugins/unified_doc_viewer/public/__mocks__/services.ts @@ -10,6 +10,7 @@ import { analyticsServiceMock } from '@kbn/core-analytics-browser-mocks'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { discoverSharedPluginMock } from '@kbn/discover-shared-plugin/public/mocks'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; +import { createFieldsMetadataPluginStartMock } from '@kbn/fields-metadata-plugin/public/mocks'; import { uiSettingsServiceMock } from '@kbn/core-ui-settings-browser-mocks'; import type { UnifiedDocViewerServices, UnifiedDocViewerStart } from '../types'; import { Storage } from '@kbn/kibana-utils-plugin/public'; @@ -24,6 +25,7 @@ export const mockUnifiedDocViewerServices: jest.Mocked data: dataPluginMock.createStartContract(), discoverShared: discoverSharedPluginMock.createStartContract(), fieldFormats: fieldFormatsMock, + fieldsMetadata: createFieldsMetadataPluginStartMock(), storage: new Storage(localStorage), uiSettings: uiSettingsServiceMock.createStartContract(), unifiedDocViewer: mockUnifiedDocViewer, diff --git a/src/plugins/unified_doc_viewer/public/plugin.tsx b/src/plugins/unified_doc_viewer/public/plugin.tsx index 0bf4731e8ee1c..524a02eec9ee9 100644 --- a/src/plugins/unified_doc_viewer/public/plugin.tsx +++ b/src/plugins/unified_doc_viewer/public/plugin.tsx @@ -18,7 +18,7 @@ import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { CoreStart } from '@kbn/core/public'; import { dynamic } from '@kbn/shared-ux-utility'; import { DiscoverSharedPublicStart } from '@kbn/discover-shared-plugin/public'; -import { FieldsMetadataClientStart } from '@kbn/fields-metadata-plugin/public'; +import { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public'; import type { UnifiedDocViewerServices } from './types'; export const [getUnifiedDocViewerServices, setUnifiedDocViewerServices] = @@ -51,7 +51,7 @@ export interface UnifiedDocViewerStartDeps { data: DataPublicPluginStart; discoverShared: DiscoverSharedPublicStart; fieldFormats: FieldFormatsStart; - fieldsMetadata: FieldsMetadataClientStart; + fieldsMetadata: FieldsMetadataPublicStart; } export class UnifiedDocViewerPublicPlugin diff --git a/src/plugins/unified_doc_viewer/public/types.ts b/src/plugins/unified_doc_viewer/public/types.ts index 0ccdaa823c4b4..c19c60da72b13 100644 --- a/src/plugins/unified_doc_viewer/public/types.ts +++ b/src/plugins/unified_doc_viewer/public/types.ts @@ -14,7 +14,7 @@ import type { AnalyticsServiceStart } from '@kbn/core-analytics-browser'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DiscoverSharedPublicStart } from '@kbn/discover-shared-plugin/public'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; -import type { FieldsMetadataClientStart } from '@kbn/fields-metadata-plugin/public'; +import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public'; import type { Storage } from '@kbn/kibana-utils-plugin/public'; import type { IUiSettingsClient } from '@kbn/core-ui-settings-browser'; import type { UnifiedDocViewerStart } from './plugin'; @@ -24,7 +24,7 @@ export interface UnifiedDocViewerServices { data: DataPublicPluginStart; discoverShared: DiscoverSharedPublicStart; fieldFormats: FieldFormatsStart; - fieldsMetadata: FieldsMetadataClientStart; + fieldsMetadata: FieldsMetadataPublicStart; storage: Storage; uiSettings: IUiSettingsClient; unifiedDocViewer: UnifiedDocViewerStart; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts index beac6a9bfe673..d12cbbaf15bc0 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts @@ -21,13 +21,12 @@ export class FieldMetadata { return pick(this, props); } - public toPlain() { + public toPlain(): FieldMetadataPlain { return Object.assign({}, this); } - public static create(fieldMetadata: PartialFieldMetadataPlain) { - const name = fieldMetadata.name ?? ''; - const flat_name = fieldMetadata.flat_name ?? name; + public static create(fieldMetadata: FieldMetadataPlain) { + const flat_name = fieldMetadata.flat_name ?? fieldMetadata.name; const dashed_name = fieldMetadata.dashed_name ?? FieldMetadata.toDashedName(flat_name); const normalize = fieldMetadata.normalize ?? []; const short = fieldMetadata.short ?? fieldMetadata.description; @@ -36,9 +35,8 @@ export class FieldMetadata { const fieldMetadataProps = { ...fieldMetadata, - name, - flat_name, dashed_name, + flat_name, normalize, short, source, diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts index 9039a1eb9f1ea..0ddda927de676 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts @@ -9,10 +9,10 @@ import mapValues from 'lodash/mapValues'; import { FieldAttribute, FieldMetadataPlain, PartialFieldMetadataPlain } from '../types'; import { FieldMetadata } from './field_metadata'; -export type PartialFieldsMetadataMap = Record; +export type FieldsMetadataMap = Record; export class FieldsMetadataDictionary { - private constructor(private readonly fields: PartialFieldsMetadataMap) {} + private constructor(private readonly fields: FieldsMetadataMap) {} pick(attributes: FieldAttribute[]): Record { return mapValues(this.fields, (field) => field.pick(attributes)); @@ -22,7 +22,7 @@ export class FieldsMetadataDictionary { return mapValues(this.fields, (field) => field.toPlain()); } - public static create(fields: PartialFieldsMetadataMap = {}) { + public static create(fields: FieldsMetadataMap) { return new FieldsMetadataDictionary(fields); } } diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index 80e922e2f1300..8a1327e363aad 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -32,10 +32,7 @@ export const multiFieldRT = rt.type({ }); const requiredBaseMetadataPlainRT = rt.type({ - flat_name: rt.string, name: rt.string, - source: fieldSourceRT, - type: rt.string, }); const optionalBaseMetadataPlainRT = rt.partial(requiredBaseMetadataPlainRT.props); @@ -48,6 +45,7 @@ const optionalMetadataPlainRT = rt.partial({ doc_values: rt.boolean, example: rt.unknown, expected_values: rt.array(rt.string), + flat_name: rt.string, format: rt.string, ignore_above: rt.number, index: rt.boolean, @@ -63,6 +61,8 @@ const optionalMetadataPlainRT = rt.partial({ required: rt.boolean, scaling_factor: rt.number, short: rt.string, + source: fieldSourceRT, + type: rt.string, }); export const partialFieldMetadataPlainRT = rt.intersection([ diff --git a/x-pack/plugins/fields_metadata/public/index.ts b/x-pack/plugins/fields_metadata/public/index.ts index d22aa71e214dc..11afff4fab172 100644 --- a/x-pack/plugins/fields_metadata/public/index.ts +++ b/x-pack/plugins/fields_metadata/public/index.ts @@ -8,10 +8,10 @@ import { FieldsMetadataPlugin } from './plugin'; export type { - FieldsMetadataClientSetup, - FieldsMetadataClientStart, - FieldsMetadataClientSetupDeps, - FieldsMetadataClientStartDeps, + FieldsMetadataPublicSetup, + FieldsMetadataPublicStart, + FieldsMetadataPublicSetupDeps, + FieldsMetadataPublicStartDeps, } from './types'; // This exports static code and TypeScript types, diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index 09e6f8172427e..6244b1309408b 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -6,12 +6,12 @@ */ import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; -import { FieldsMetadataClientStart } from './types'; +import { FieldsMetadataPublicStart } from './types'; -export const createFieldsMetadataPluginStartMock = (): jest.Mocked => ({ +export const createFieldsMetadataPluginStartMock = (): jest.Mocked => ({ client: createFieldsMetadataServiceStartMock().client, useFieldsMetadata: jest.fn(), }); -export const _ensureTypeCompatibility = (): FieldsMetadataClientStart => +export const _ensureTypeCompatibility = (): FieldsMetadataPublicStart => createFieldsMetadataPluginStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts index a99a3438685d0..37db9ae94a494 100644 --- a/x-pack/plugins/fields_metadata/public/plugin.ts +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -11,8 +11,8 @@ import { FieldsMetadataService } from './services/fields_metadata'; import { FieldsMetadataClientCoreSetup, FieldsMetadataClientPluginClass, - FieldsMetadataClientSetupDeps, - FieldsMetadataClientStartDeps, + FieldsMetadataPublicSetupDeps, + FieldsMetadataPublicStartDeps, } from './types'; export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { @@ -22,13 +22,13 @@ export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { this.fieldsMetadata = new FieldsMetadataService(); } - public setup(_: FieldsMetadataClientCoreSetup, pluginsSetup: FieldsMetadataClientSetupDeps) { + public setup(_: FieldsMetadataClientCoreSetup, pluginsSetup: FieldsMetadataPublicSetupDeps) { this.fieldsMetadata.setup(); return {}; } - public start(core: CoreStart, plugins: FieldsMetadataClientStartDeps) { + public start(core: CoreStart, plugins: FieldsMetadataPublicStartDeps) { const { http } = core; const { client } = this.fieldsMetadata.start({ http }); diff --git a/x-pack/plugins/fields_metadata/public/types.ts b/x-pack/plugins/fields_metadata/public/types.ts index cb7ae3f493291..34a8ee0dda61f 100644 --- a/x-pack/plugins/fields_metadata/public/types.ts +++ b/x-pack/plugins/fields_metadata/public/types.ts @@ -10,32 +10,32 @@ import type { UseFieldsMetadataHook } from './hooks/use_fields_metadata/use_fiel import type { IFieldsMetadataClient } from './services/fields_metadata'; // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FieldsMetadataClientSetup {} +export interface FieldsMetadataPublicSetup {} -export interface FieldsMetadataClientStart { +export interface FieldsMetadataPublicStart { client: IFieldsMetadataClient; useFieldsMetadata: UseFieldsMetadataHook; } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FieldsMetadataClientSetupDeps {} +export interface FieldsMetadataPublicSetupDeps {} // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FieldsMetadataClientStartDeps {} +export interface FieldsMetadataPublicStartDeps {} export type FieldsMetadataClientCoreSetup = CoreSetup< - FieldsMetadataClientStartDeps, - FieldsMetadataClientStart + FieldsMetadataPublicStartDeps, + FieldsMetadataPublicStart >; export type FieldsMetadataClientCoreStart = CoreStart; export type FieldsMetadataClientPluginClass = PluginClass< - FieldsMetadataClientSetup, - FieldsMetadataClientStart, - FieldsMetadataClientSetupDeps, - FieldsMetadataClientStartDeps + FieldsMetadataPublicSetup, + FieldsMetadataPublicStart, + FieldsMetadataPublicSetupDeps, + FieldsMetadataPublicStartDeps >; -export type FieldsMetadataClientStartServicesAccessor = +export type FieldsMetadataPublicStartServicesAccessor = FieldsMetadataClientCoreSetup['getStartServices']; -export type FieldsMetadataClientStartServices = - ReturnType; +export type FieldsMetadataPublicStartServices = + ReturnType; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index c0ff34f298bb9..59f4ee26a2e12 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -6,7 +6,7 @@ */ import { HashedCache } from '../../../../common/hashed_cache'; -import { FieldMetadata, IntegrationFieldName, PartialFieldMetadataPlain } from '../../../../common'; +import { FieldMetadata, FieldMetadataPlain, IntegrationFieldName } from '../../../../common'; import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; import { PackageNotFoundError } from '../errors'; interface IntegrationFieldsRepositoryDeps { @@ -109,7 +109,7 @@ export class IntegrationFieldsRepository { private getCacheKey = (params: IntegrationFieldsSearchParams) => params; private mapExtractedFieldsToFieldMetadataTree = ( - extractedFields: Record> + extractedFields: Record> ) => { return Object.entries(extractedFields).reduce( (integrationGroup, [datasetName, datasetGroup]) => { diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index 6d19e0ae2d5ae..e2f38e7acba81 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -5,14 +5,14 @@ * 2.0. */ -import type { PartialFieldMetadataPlain } from '../../../../common'; +import type { FieldMetadataPlain } from '../../../../common'; export interface IntegrationFieldsSearchParams { integration: string; dataset?: string; } -export type ExtractedIntegrationFields = Record>; +export type ExtractedIntegrationFields = Record>; export type IntegrationFieldsExtractor = ({ integration, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts index ba5a2d73ee83d..084dbbad6534b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts @@ -6,19 +6,15 @@ */ import { withSpan } from '@kbn/apm-utils'; -import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; +import type { FieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; import { load } from 'js-yaml'; import type { RegistryDataStream } from '../../../../common'; import type { AssetsMap } from '../../../../common/types'; -interface PackageFieldMetadata extends PartialFieldMetadataPlain { - name: string; -} - type InputField = - | PackageFieldMetadata + | FieldMetadataPlain | { name: string; type: 'group'; @@ -28,10 +24,7 @@ type InputField = export const withPackageSpan = (stepName: string, func: () => Promise) => withSpan({ name: stepName, type: 'package' }, func); -const normalizeFields = ( - fields: InputField[], - prefix = '' -): Record => { +const normalizeFields = (fields: InputField[], prefix = ''): Record => { return fields.reduce((normalizedFields, field) => { const flatName = prefix ? `${prefix}.${field.name}` : field.name; // Recursively resolve field groups @@ -42,11 +35,11 @@ const normalizeFields = ( normalizedFields[flatName] = createIntegrationField(field, flatName); return normalizedFields; - }, {} as Record); + }, {} as Record); }; const createIntegrationField = ( - field: Omit, + field: Omit, flatName: string ) => ({ ...field, @@ -94,7 +87,7 @@ export const resolveDataStreamFields = ({ } return dataStreamFields; - }, {} as Record); + }, {} as Record); return { [dataset]: fields, diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index e8b5cb556aa15..84e10f6bc08f7 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -13,7 +13,7 @@ import semverGte from 'semver/functions/gte'; import type { Response } from 'node-fetch'; import type { Logger } from '@kbn/logging'; -import type { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; +import type { FieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; import { splitPkgKey as split } from '../../../../common/services'; @@ -357,7 +357,7 @@ export async function getPackage( export async function getPackageFieldsMetadata( params: { packageName: string; datasetName?: string }, options: { excludedFieldsAssets?: string[] } = {} -): Promise>> { +): Promise>> { const { packageName, datasetName } = params; const { excludedFieldsAssets = ['ecs.yml'] } = options; From cf34cb43a18f3b355ec6dc37301a92889da7fa65 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 16:16:53 +0200 Subject: [PATCH 32/50] refactor(fields-metadata): cleanup unnecessary code --- x-pack/plugins/fields_metadata/public/index.ts | 10 ++++------ x-pack/plugins/fields_metadata/public/mocks.tsx | 3 --- x-pack/plugins/fields_metadata/public/plugin.ts | 11 +++-------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fields_metadata/public/index.ts b/x-pack/plugins/fields_metadata/public/index.ts index 11afff4fab172..74f73c2751f77 100644 --- a/x-pack/plugins/fields_metadata/public/index.ts +++ b/x-pack/plugins/fields_metadata/public/index.ts @@ -7,15 +7,13 @@ import { FieldsMetadataPlugin } from './plugin'; +export function plugin() { + return new FieldsMetadataPlugin(); +} + export type { FieldsMetadataPublicSetup, FieldsMetadataPublicStart, FieldsMetadataPublicSetupDeps, FieldsMetadataPublicStartDeps, } from './types'; - -// This exports static code and TypeScript types, -// as well as, Kibana Platform `plugin()` initializer. -export function plugin() { - return new FieldsMetadataPlugin(); -} diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index 6244b1309408b..bce1aef19192a 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -12,6 +12,3 @@ export const createFieldsMetadataPluginStartMock = (): jest.Mocked - createFieldsMetadataPluginStartMock(); diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts index 37db9ae94a494..d258368bad57c 100644 --- a/x-pack/plugins/fields_metadata/public/plugin.ts +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -8,12 +8,7 @@ import { CoreStart } from '@kbn/core/public'; import { createUseFieldsMetadataHook } from './hooks/use_fields_metadata'; import { FieldsMetadataService } from './services/fields_metadata'; -import { - FieldsMetadataClientCoreSetup, - FieldsMetadataClientPluginClass, - FieldsMetadataPublicSetupDeps, - FieldsMetadataPublicStartDeps, -} from './types'; +import { FieldsMetadataClientPluginClass } from './types'; export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { private fieldsMetadata: FieldsMetadataService; @@ -22,13 +17,13 @@ export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { this.fieldsMetadata = new FieldsMetadataService(); } - public setup(_: FieldsMetadataClientCoreSetup, pluginsSetup: FieldsMetadataPublicSetupDeps) { + public setup() { this.fieldsMetadata.setup(); return {}; } - public start(core: CoreStart, plugins: FieldsMetadataPublicStartDeps) { + public start(core: CoreStart) { const { http } = core; const { client } = this.fieldsMetadata.start({ http }); From 8107ded16ec3b0ec050306dddf4300bf34c9ec95 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 16:48:51 +0200 Subject: [PATCH 33/50] refactor(fields-metadata): improve client error handling --- .../common/fields_metadata/errors.ts | 24 +++++++++++++++++ .../common/fields_metadata/index.ts | 1 + .../fields_metadata/common/hashed_cache.ts | 1 + .../fields_metadata/fields_metadata_client.ts | 26 ++++++++++++++++--- .../server/services/fields_metadata/errors.ts | 8 +++++- 5 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/fields_metadata/common/fields_metadata/errors.ts diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/errors.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/errors.ts new file mode 100644 index 0000000000000..e4a2c02c33e44 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/errors.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable max-classes-per-file */ + +export class FetchFieldsMetadataError extends Error { + constructor(message: string, public cause?: Error) { + super(message); + Object.setPrototypeOf(this, new.target.prototype); + this.name = 'FetchFieldsMetadataError'; + } +} + +export class DecodeFieldsMetadataError extends Error { + constructor(message: string, public cause?: Error) { + super(message); + Object.setPrototypeOf(this, new.target.prototype); + this.name = 'DecodeFieldsMetadataError'; + } +} diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts index 4328ef2e38aa6..0590a318d13f9 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/index.ts @@ -6,4 +6,5 @@ */ export * from './common'; +export * from './errors'; export * from './types'; diff --git a/x-pack/plugins/fields_metadata/common/hashed_cache.ts b/x-pack/plugins/fields_metadata/common/hashed_cache.ts index 4b5ac5c614472..dfd5afe92c61a 100644 --- a/x-pack/plugins/fields_metadata/common/hashed_cache.ts +++ b/x-pack/plugins/fields_metadata/common/hashed_cache.ts @@ -6,6 +6,7 @@ */ import LRUCache from 'lru-cache'; import hash from 'object-hash'; + export interface IHashedCache { get(key: KeyType): ValueType | undefined; set(key: KeyType, value: ValueType): boolean; diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts index bf76e14542791..176721c704126 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -13,7 +13,12 @@ import { FindFieldsMetadataResponsePayload, findFieldsMetadataResponsePayloadRT, } from '../../../common/latest'; -import { FIND_FIELDS_METADATA_URL } from '../../../common/fields_metadata'; +import { + DecodeFieldsMetadataError, + FetchFieldsMetadataError, + FieldName, + FIND_FIELDS_METADATA_URL, +} from '../../../common/fields_metadata'; import { decodeOrThrow } from '../../../common/runtime_types'; import { IFieldsMetadataClient } from './types'; @@ -37,13 +42,17 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { const response = await this.http .get(FIND_FIELDS_METADATA_URL, { query, version: '1' }) .catch((error) => { - throw new Error(`Failed to fetch ecs fields ${params.fieldNames?.join() ?? ''}: ${error}`); + throw new FetchFieldsMetadataError( + `Failed to fetch fields ${truncateFieldNamesList(params.fieldNames)}: ${error.message}` + ); }); const data = decodeOrThrow( findFieldsMetadataResponsePayloadRT, (message: string) => - new Error(`Failed to decode ecs fields ${params.fieldNames?.join() ?? ''}: ${message}"`) + new DecodeFieldsMetadataError( + `Failed decoding fields ${truncateFieldNamesList(params.fieldNames)}: ${message}` + ) )(response); // Store cached results for given request parameters @@ -52,3 +61,14 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { return data; } } + +const truncateFieldNamesList = (fieldNames?: FieldName[]) => { + if (!fieldNames || fieldNames.length === 0) return ''; + + const visibleFields = fieldNames.slice(0, 3); + const additionalFieldsCount = fieldNames.length - visibleFields.length; + + return visibleFields + .join() + .concat(additionalFieldsCount > 0 ? `+${additionalFieldsCount} fields` : ''); +}; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts index ea99d78cd2d66..f91ebd745ec94 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/errors.ts @@ -5,4 +5,10 @@ * 2.0. */ -export class PackageNotFoundError extends Error {} +export class PackageNotFoundError extends Error { + constructor(message: string, public cause?: Error) { + super(message); + Object.setPrototypeOf(this, new.target.prototype); + this.name = 'PackageNotFoundError'; + } +} From bf78af05ea6eca6594d2849881a837b5ead92289 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 17:03:24 +0200 Subject: [PATCH 34/50] refactor(fields-metadata): minor mocks adjustments --- .../unified_doc_viewer/public/__mocks__/services.ts | 4 ++-- x-pack/plugins/fields_metadata/public/mocks.tsx | 6 +++++- .../fields_metadata/fields_metadata_service.mock.ts | 10 ++++------ x-pack/plugins/fields_metadata/server/mocks.ts | 10 +++++----- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/plugins/unified_doc_viewer/public/__mocks__/services.ts b/src/plugins/unified_doc_viewer/public/__mocks__/services.ts index 3d660aaa46a48..8496b919b38f0 100644 --- a/src/plugins/unified_doc_viewer/public/__mocks__/services.ts +++ b/src/plugins/unified_doc_viewer/public/__mocks__/services.ts @@ -10,7 +10,7 @@ import { analyticsServiceMock } from '@kbn/core-analytics-browser-mocks'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { discoverSharedPluginMock } from '@kbn/discover-shared-plugin/public/mocks'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; -import { createFieldsMetadataPluginStartMock } from '@kbn/fields-metadata-plugin/public/mocks'; +import { fieldsMetadataPluginPublicMock } from '@kbn/fields-metadata-plugin/public/mocks'; import { uiSettingsServiceMock } from '@kbn/core-ui-settings-browser-mocks'; import type { UnifiedDocViewerServices, UnifiedDocViewerStart } from '../types'; import { Storage } from '@kbn/kibana-utils-plugin/public'; @@ -25,7 +25,7 @@ export const mockUnifiedDocViewerServices: jest.Mocked data: dataPluginMock.createStartContract(), discoverShared: discoverSharedPluginMock.createStartContract(), fieldFormats: fieldFormatsMock, - fieldsMetadata: createFieldsMetadataPluginStartMock(), + fieldsMetadata: fieldsMetadataPluginPublicMock.createStartContract(), storage: new Storage(localStorage), uiSettings: uiSettingsServiceMock.createStartContract(), unifiedDocViewer: mockUnifiedDocViewer, diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index bce1aef19192a..d1fb85b8351be 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -8,7 +8,11 @@ import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; import { FieldsMetadataPublicStart } from './types'; -export const createFieldsMetadataPluginStartMock = (): jest.Mocked => ({ +const createFieldsMetadataPublicStartMock = (): jest.Mocked => ({ client: createFieldsMetadataServiceStartMock().client, useFieldsMetadata: jest.fn(), }); + +export const fieldsMetadataPluginPublicMock = { + createStartContract: createFieldsMetadataPublicStartMock, +}; diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts index bfef30e155fd1..76f892f63e80c 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts @@ -8,9 +8,7 @@ import { createFieldsMetadataClientMock } from './fields_metadata_client.mock'; import { FieldsMetadataServiceStart } from './types'; -export const createFieldsMetadataServiceStartMock = () => ({ - client: createFieldsMetadataClientMock(), -}); - -export const _ensureTypeCompatibility = (): FieldsMetadataServiceStart => - createFieldsMetadataServiceStartMock(); +export const createFieldsMetadataServiceStartMock = + (): jest.Mocked => ({ + client: createFieldsMetadataClientMock(), + }); diff --git a/x-pack/plugins/fields_metadata/server/mocks.ts b/x-pack/plugins/fields_metadata/server/mocks.ts index 657e73635ce85..b46ca661b6210 100644 --- a/x-pack/plugins/fields_metadata/server/mocks.ts +++ b/x-pack/plugins/fields_metadata/server/mocks.ts @@ -11,16 +11,16 @@ import { } from './services/fields_metadata/fields_metadata_service.mock'; import { FieldsMetadataServerSetup, FieldsMetadataServerStart } from './types'; -const createFieldsMetadataSetupMock = (): jest.Mocked => ({ +const createFieldsMetadataServerSetupMock = (): jest.Mocked => ({ registerIntegrationFieldsExtractor: createFieldsMetadataServiceSetupMock().registerIntegrationFieldsExtractor, }); -const createFieldsMetadataStartMock = (): jest.Mocked => ({ +const createFieldsMetadataServerStartMock = (): jest.Mocked => ({ getClient: createFieldsMetadataServiceStartMock().getClient, }); -export const fieldsMetadataPluginMock = { - createSetupContract: createFieldsMetadataSetupMock, - createStartContract: createFieldsMetadataStartMock, +export const fieldsMetadataPluginServerMock = { + createSetupContract: createFieldsMetadataServerSetupMock, + createStartContract: createFieldsMetadataServerStartMock, }; From ee9412be97457992deb3ff5c351657978ad293ef Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 17:39:26 +0200 Subject: [PATCH 35/50] refactor(fields-metadata): improve integration service readability --- .../integration_fields_repository.ts | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 59f4ee26a2e12..721ae74c060c4 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -6,18 +6,22 @@ */ import { HashedCache } from '../../../../common/hashed_cache'; -import { FieldMetadata, FieldMetadataPlain, IntegrationFieldName } from '../../../../common'; -import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './types'; +import { FieldMetadata, IntegrationFieldName } from '../../../../common'; +import { + ExtractedIntegrationFields, + IntegrationFieldsExtractor, + IntegrationFieldsSearchParams, +} from './types'; import { PackageNotFoundError } from '../errors'; interface IntegrationFieldsRepositoryDeps { integrationFieldsExtractor: IntegrationFieldsExtractor; } +type DatasetFieldsMetadata = Record; +type IntegrationFieldsMetadataTree = Record; + export class IntegrationFieldsRepository { - private cache: HashedCache< - IntegrationFieldsSearchParams, - Record> - >; + private cache: HashedCache; private constructor(private readonly fieldsExtractor: IntegrationFieldsExtractor) { this.cache = new HashedCache(); @@ -95,7 +99,7 @@ export class IntegrationFieldsRepository { private storeFieldsInCache = ( cacheKey: IntegrationFieldsSearchParams, - extractedFieldsMetadata: Record> + extractedFieldsMetadata: IntegrationFieldsMetadataTree ): void => { const cachedIntegration = this.cache.get(cacheKey); @@ -108,25 +112,21 @@ export class IntegrationFieldsRepository { private getCacheKey = (params: IntegrationFieldsSearchParams) => params; - private mapExtractedFieldsToFieldMetadataTree = ( - extractedFields: Record> - ) => { - return Object.entries(extractedFields).reduce( - (integrationGroup, [datasetName, datasetGroup]) => { - integrationGroup[datasetName] = Object.entries(datasetGroup).reduce( - (datasetGroupResult, [fieldName, field]) => { - datasetGroupResult[fieldName] = FieldMetadata.create({ - ...field, - source: 'integration', - }); - return datasetGroupResult; - }, - {} as Record - ); - - return integrationGroup; - }, - {} as Record> - ); + private mapExtractedFieldsToFieldMetadataTree = (extractedFields: ExtractedIntegrationFields) => { + const datasetGroups = Object.entries(extractedFields); + + return datasetGroups.reduce((integrationGroup, [datasetName, datasetGroup]) => { + const datasetFieldsEntries = Object.entries(datasetGroup); + + integrationGroup[datasetName] = datasetFieldsEntries.reduce( + (datasetFields, [fieldName, field]) => { + datasetFields[fieldName] = FieldMetadata.create({ ...field, source: 'integration' }); + return datasetFields; + }, + {} as DatasetFieldsMetadata + ); + + return integrationGroup; + }, {} as IntegrationFieldsMetadataTree); }; } From abb0477845683dbe91c94fbfd1b01aca5767e5d2 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 17:42:05 +0200 Subject: [PATCH 36/50] refactor(fields-metadata): update type --- .../server/services/fields_metadata/repositories/types.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index e2f38e7acba81..92cc77e2b7c2a 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -14,7 +14,6 @@ export interface IntegrationFieldsSearchParams { export type ExtractedIntegrationFields = Record>; -export type IntegrationFieldsExtractor = ({ - integration, - dataset, -}: IntegrationFieldsSearchParams) => Promise; +export type IntegrationFieldsExtractor = ( + params: IntegrationFieldsSearchParams +) => Promise; From 57aa072c873dc971d3c9a421a1f5769f9a2fb4a8 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 18:02:56 +0200 Subject: [PATCH 37/50] refactor(fields-metadata): improve typing readability for fleet --- x-pack/plugins/fields_metadata/server/index.ts | 6 ++++++ .../repositories/integration_fields_repository.ts | 3 ++- .../server/services/fields_metadata/repositories/types.ts | 5 ++++- .../server/services/fields_metadata/types.ts | 2 ++ .../plugins/fleet/server/services/epm/packages/utils.ts | 7 ++++--- .../plugins/fleet/server/services/epm/registry/index.ts | 8 ++------ 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fields_metadata/server/index.ts b/x-pack/plugins/fields_metadata/server/index.ts index a0673d7f58217..1c55673e44e92 100644 --- a/x-pack/plugins/fields_metadata/server/index.ts +++ b/x-pack/plugins/fields_metadata/server/index.ts @@ -8,6 +8,12 @@ import { PluginInitializerContext } from '@kbn/core/server'; export type { FieldsMetadataServerSetup, FieldsMetadataServerStart } from './types'; +export type { + IntegrationName, + DatasetName, + ExtractedIntegrationFields, + ExtractedDatasetFields, +} from './services/fields_metadata/types'; export async function plugin(context: PluginInitializerContext) { const { FieldsMetadataPlugin } = await import('./plugin'); diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 721ae74c060c4..5265c55c66bb3 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -11,6 +11,7 @@ import { ExtractedIntegrationFields, IntegrationFieldsExtractor, IntegrationFieldsSearchParams, + IntegrationName, } from './types'; import { PackageNotFoundError } from '../errors'; interface IntegrationFieldsRepositoryDeps { @@ -18,7 +19,7 @@ interface IntegrationFieldsRepositoryDeps { } type DatasetFieldsMetadata = Record; -type IntegrationFieldsMetadataTree = Record; +type IntegrationFieldsMetadataTree = Record; export class IntegrationFieldsRepository { private cache: HashedCache; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts index 92cc77e2b7c2a..e258c46b569b4 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/types.ts @@ -12,7 +12,10 @@ export interface IntegrationFieldsSearchParams { dataset?: string; } -export type ExtractedIntegrationFields = Record>; +export type IntegrationName = string; +export type DatasetName = string; +export type ExtractedIntegrationFields = Record; +export type ExtractedDatasetFields = Record; export type IntegrationFieldsExtractor = ( params: IntegrationFieldsSearchParams diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts index ef092b3612aee..5b87f3299d61b 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/types.ts @@ -8,6 +8,8 @@ import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; import { IntegrationFieldsExtractor, IntegrationFieldsSearchParams } from './repositories/types'; +export * from './repositories/types'; + // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldsMetadataServiceStartDeps {} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts index 084dbbad6534b..5cd83fb5df32e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts @@ -7,6 +7,7 @@ import { withSpan } from '@kbn/apm-utils'; import type { FieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; +import type { ExtractedDatasetFields } from '@kbn/fields-metadata-plugin/server'; import { load } from 'js-yaml'; @@ -24,7 +25,7 @@ type InputField = export const withPackageSpan = (stepName: string, func: () => Promise) => withSpan({ name: stepName, type: 'package' }, func); -const normalizeFields = (fields: InputField[], prefix = ''): Record => { +const normalizeFields = (fields: InputField[], prefix = ''): ExtractedDatasetFields => { return fields.reduce((normalizedFields, field) => { const flatName = prefix ? `${prefix}.${field.name}` : field.name; // Recursively resolve field groups @@ -35,7 +36,7 @@ const normalizeFields = (fields: InputField[], prefix = ''): Record); + }, {} as ExtractedDatasetFields); }; const createIntegrationField = ( @@ -87,7 +88,7 @@ export const resolveDataStreamFields = ({ } return dataStreamFields; - }, {} as Record); + }, {} as ExtractedDatasetFields); return { [dataset]: fields, diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 84e10f6bc08f7..bb4d612aa7de3 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -13,7 +13,7 @@ import semverGte from 'semver/functions/gte'; import type { Response } from 'node-fetch'; import type { Logger } from '@kbn/logging'; -import type { FieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; +import type { ExtractedIntegrationFields } from '@kbn/fields-metadata-plugin/server'; import { splitPkgKey as split } from '../../../../common/services'; @@ -357,7 +357,7 @@ export async function getPackage( export async function getPackageFieldsMetadata( params: { packageName: string; datasetName?: string }, options: { excludedFieldsAssets?: string[] } = {} -): Promise>> { +): Promise { const { packageName, datasetName } = params; const { excludedFieldsAssets = ['ecs.yml'] } = options; @@ -368,10 +368,6 @@ export async function getPackageFieldsMetadata( // Attempt retrieving latest package const resolvedPackage = await getPackage(name, version); - if (!resolvedPackage) { - throw new Error('The package you are looking for cannot be retrieved.'); - } - // We need to collect all the available data streams for the package. // In case a dataset is specified from the parameter, it will load the fields only for that specific dataset. // As a fallback case, we'll try to read the fields for all the data streams in the package. From 93754ca55aa77e45ec96d72032bea752b4dd4973 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 18:08:04 +0200 Subject: [PATCH 38/50] refactor(fleet): improve comments --- x-pack/plugins/fleet/server/services/epm/packages/utils.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts index 5cd83fb5df32e..5be4833d290bd 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/utils.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/utils.ts @@ -78,6 +78,13 @@ export const resolveDataStreamFields = ({ excludedFieldsAssets ); + /** + * We want to create a single dictionary with fields taken from all the dataset /fields assets. + * This step + * - reads the files buffer + * - normalizes the fields data structure for each file + * - finally merge the fields from each file into a single dictionary + */ const fields = dataStreamFieldsAssetPaths.reduce((dataStreamFields, fieldsAssetPath) => { const fieldsAssetBuffer = assetsMap.get(fieldsAssetPath); From f86c2feff116b9c4c586ecb4700ca734c4c0d912 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 23 May 2024 23:19:28 +0200 Subject: [PATCH 39/50] refactor(fields-metadata): fix useFieldsMetadata mocks --- .../use_fields_metadata.mock.ts | 16 ++++++++++++++++ x-pack/plugins/fields_metadata/public/mocks.tsx | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.mock.ts diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.mock.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.mock.ts new file mode 100644 index 0000000000000..3928b350aaae8 --- /dev/null +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.mock.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UseFieldsMetadataHook } from './use_fields_metadata'; + +export const createUseFieldsMetadataHookMock = (): jest.Mocked => + jest.fn(() => ({ + fieldsMetadata: undefined, + loading: false, + error: undefined, + reload: jest.fn(), + })); diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index d1fb85b8351be..338aa7be3b7d4 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -5,12 +5,12 @@ * 2.0. */ +import { createUseFieldsMetadataHookMock } from './hooks/use_fields_metadata/use_fields_metadata.mock'; import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; -import { FieldsMetadataPublicStart } from './types'; -const createFieldsMetadataPublicStartMock = (): jest.Mocked => ({ +const createFieldsMetadataPublicStartMock = () => ({ client: createFieldsMetadataServiceStartMock().client, - useFieldsMetadata: jest.fn(), + useFieldsMetadata: createUseFieldsMetadataHookMock(), }); export const fieldsMetadataPluginPublicMock = { From 3a738de96aaefb08d57559563078c20c52978864 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Fri, 24 May 2024 15:31:19 +0200 Subject: [PATCH 40/50] docs(fields-metadata): add readme --- x-pack/plugins/fields_metadata/README.md | 167 ++++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md index 4940151d84281..1c151fdf71017 100755 --- a/x-pack/plugins/fields_metadata/README.md +++ b/x-pack/plugins/fields_metadata/README.md @@ -1,3 +1,166 @@ -# Fields metadata plugin +# Fields Metadata Plugin -Exposes services for async usage and search of field metadata. +The `@kbn/fields-metadata-plugin` is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. + +## Components and Mechanisms + +### FieldsMetadataService (Server-side) + +The `FieldsMetadataService` is instantiated during the plugin setup/start lifecycle on the server side. It exposes a client that can be used to consume field metadata and provides tools for registering external dependencies. + +#### Start Contract + +The start contract exposes a `FieldsMetadataClient` instance, which offers the following methods: +- `getByName(name: string, params? {integration: string, dataset?: string})`: Retrieves a single `FieldMetadata` instance by name. + +```ts +const timestampField = await client.getByName('@timestamp') +/* +{ + dashed_name: 'timestamp', + type: 'date', + ... +} +*/ +``` + +- `find(params?: {fieldNames: string[], integration: string, dataset?: string})`: Retrieves a record of matching `FieldMetadata` instances based on the query parameters. + +**Parameters** +| Name | Type | Example | Optional | +|---|---|---|---| +| fieldNames | [] | ['@timestamp', 'onepassword.client.platform_version'] | ✅ | +| integration | string | 1password | ✅ | +| dataset | string | 1password.item_usages | ✅ | + +```ts +const fields = await client.find({ + fieldNames: ['@timestamp', 'onepassword.client.platform_version'], + integration: '1password' +}) +/* +{ + '@timestamp': { + dashed_name: 'timestamp', + type: 'date', + ... + }, + 'onepassword.client.platform_version': { + name: 'platform_version', + type: 'keyword', + ... + }, +} +*/ +``` + +> N.B. Passing the `dataset` name parameter to `.find` helps narrowing the scope of the integration assets that need to be fetched, increasing the performance of the request. Using only the `integration` parameter should achieve the same result, but is recommended always passing the `dataset` as well if known or unless the required fields come from different datasets of the same integration. + +> N.B. In case the `fieldNames` parameter is not passed to `.find`, the result will give the whole list of ECS fields by default. This should be avoided as much as possible, although it helps covering cases where we might need the whole ECS fields list. + +#### Source Repositories + +The `FieldsMetadataClient` relies on source repositories to fetch field metadata. Currently, there are two repository sources: +- `EcsFieldsRepository`: Fetches static ECS field metadata. +- `IntegrationFieldsRepository`: Fetches fields from an integration package from the Elastic Package Registry (EPR). This requires the `fleet` plugin to be enabled to access the registered fields extractor. + +To improve performance, a caching layer is applied to the results retrieved from external sources, minimizing latency and enhancing efficiency. + +### Fields Metadata API + +A REST API endpoint is exposed to facilitate the retrieval of field metadata: + +- `GET /internal/fields_metadata/find`: Supports query parameters to filter and find field metadata, optimizing the payload served to the client. + +**Parameters** +| Name | Type | Example | Optional | +|---|---|---|---| +| fieldNames | [] | ['@timestamp', 'onepassword.client.platform_version'] | ✅ | +| attributes | FieldAttribute[] | ['type', 'description', 'name'] | ✅ | +| integration | string | 1password | ✅ | +| dataset | string | 1password.item_usages | ✅ | + +### FieldsMetadataService (Client-side) + +The client-side counterpart of the `FieldsMetadataService` ensures safe consumption of the exposed API and performs necessary validation steps. The client is returned by the public start contract of the plugin, allowing other parts of Kibana to use fields metadata directly. + +With this client request/response validation, error handling and client-side caching are all handled out of the box. + +Typical use cases for this client are integrating fields metadata on existing state management solutions or early metadata retrieval on initialization. + +```ts +export class FieldsMetadataPlugin implements Plugin { + ... + + public start(core: CoreStart, plugins) { + const myFieldsMetadata = plugins.fieldsMetadata.client.find(/* ... */); + ... + } +} +``` + +### useFieldsMetadata (React Hook) + +For simpler use cases, the `useFieldsMetadata` React custom hook is provided. This hook is pre-configured with the required dependencies and allows quick access to field metadata client-side. It is essential to retrieve this hook from the start contract of the plugin to ensure proper dependency injection. + +**Parameters** +| Name | Type | Example | Optional | +|---|---|---|---| +| fieldNames | [] | ['@timestamp', 'onepassword.client.platform_version'] | ✅ | +| attributes | FieldAttribute[] | ['type', 'description', 'name'] | ✅ | +| integration | string | 1password | ✅ | +| dataset | string | 1password.item_usages | ✅ | + +```ts +const FieldsComponent = () => { + const { + fieldsMetadata: { useFieldsMetadata }, + } = useServices(); // Or useKibana and any other utility to get the plugin deps + + const { fieldsMetadata, error, loading } = useFieldsMetadata({ + fieldsName: ['@timestamp', 'agent.name'], + attributes: ['name', 'type'] + }); + + if (loading) { + return
Loading...
; + } + + if (error) { + return
Error: {error.message}
; + } + + return ( +
+ {fieldsMetadata.map(field => ( +
{field.name}: {field.type}
+ ))} +
+ ); +}; +``` + +### registerIntegrationFieldsExtractor + +To handle the complexity of fetching fields from an integration dataset, the `PackageService.prototype.getPackageFieldsMetadata()` method is implemented. This method maintains the separation of concerns and avoids direct dependency on the fleet plugin. During the fleet plugin setup, a `registerIntegrationFieldsExtractor` service is created to register a callback that retrieves fields by given parameters. + +```ts +import { registerIntegrationFieldsExtractor } from '@kbn/fields-metadata-plugin/server'; + +registerIntegrationFieldsExtractor((params) => { + // Custom logic to retrieve fields from an integration + const fields = getFieldsFromIntegration(params); + return fields; +}); +``` +```ts +export class FleetPluginServer implements Plugin { + public setup(core: CoreStart, plugins) { + plugins.fieldsMetadata.registerIntegrationFieldsExtractor((params) => { + // Custom logic to retrieve fields from an integration + const fields = getFieldsFromIntegration(params); + return fields; + }); + } +} +``` \ No newline at end of file From 0a3356f8e1204c4b3e795ff9b6fef836bf888007 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 24 May 2024 13:52:19 +0000 Subject: [PATCH 41/50] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- docs/developer/plugin-list.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 9f2aa1c6860b3..b378117ecb94f 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -579,7 +579,7 @@ activities. |{kib-repo}blob/{branch}/x-pack/plugins/fields_metadata/README.md[fieldsMetadata] -|Exposes services for async usage and search of field metadata. +|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. |{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload] From 65f00eac0793dc5afc8efe0ef3ca24eff3172ef0 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Mon, 27 May 2024 11:01:06 +0200 Subject: [PATCH 42/50] refactor(io-ts-utils): consistently use isRight for assertions --- .../kbn-io-ts-utils/src/array_to_string_rt/index.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts index 4d4603fe269f2..8ac8748118b2c 100644 --- a/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts +++ b/packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import { arrayToStringRt } from '.'; -import { isRight, Either, isLeft, fold } from 'fp-ts/lib/Either'; +import { isRight, Either, fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; function getValueOrThrow(either: Either) { @@ -22,8 +22,8 @@ describe('arrayToStringRt', () => { expect(isRight(arrayToStringRt.decode(''))).toBe(true); expect(isRight(arrayToStringRt.decode('message'))).toBe(true); expect(isRight(arrayToStringRt.decode('message,event.original'))).toBe(true); - expect(isLeft(arrayToStringRt.decode({}))).toBe(true); - expect(isLeft(arrayToStringRt.decode(true))).toBe(true); + expect(isRight(arrayToStringRt.decode({}))).toBe(false); + expect(isRight(arrayToStringRt.decode(true))).toBe(false); }); it('should return array of strings when decoding', () => { @@ -47,6 +47,6 @@ describe('arrayToStringRt', () => { expect(isRight(valid)).toBe(true); expect(getValueOrThrow(valid)).toEqual(validInput); - expect(isLeft(invalid)).toBe(true); + expect(isRight(invalid)).toBe(false); }); }); From 169b68c04e85ce254ef77ca169dad94a2398115a Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 28 May 2024 17:11:35 +0200 Subject: [PATCH 43/50] refactor(fields-metadata): update serialization for unordered arrays --- x-pack/plugins/fields_metadata/common/hashed_cache.ts | 10 +++++++--- .../hooks/use_fields_metadata/use_fields_metadata.ts | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fields_metadata/common/hashed_cache.ts b/x-pack/plugins/fields_metadata/common/hashed_cache.ts index dfd5afe92c61a..c46e34224d4d6 100644 --- a/x-pack/plugins/fields_metadata/common/hashed_cache.ts +++ b/x-pack/plugins/fields_metadata/common/hashed_cache.ts @@ -22,21 +22,25 @@ export class HashedCache { } public get(key: KeyType): ValueType | undefined { - const serializedKey = hash(key); + const serializedKey = this.getHashedKey(key); return this.cache.get(serializedKey); } public set(key: KeyType, value: ValueType) { - const serializedKey = hash(key); + const serializedKey = this.getHashedKey(key); return this.cache.set(serializedKey, value); } public has(key: KeyType): boolean { - const serializedKey = hash(key); + const serializedKey = this.getHashedKey(key); return this.cache.has(serializedKey); } public reset() { return this.cache.reset(); } + + private getHashedKey(key: KeyType) { + return hash(key, { unorderedArrays: true }); + } } diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 698e5e4da83d7..7f43272896005 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -38,7 +38,7 @@ export const createUseFieldsMetadataHook = ({ fieldsMetadataClient, }: UseFieldsMetadataFactoryDeps): UseFieldsMetadataHook => { return (params = {}) => { - const serializedParams = hash(params); + const serializedParams = hash(params, { unorderedArrays: true }); const [{ error, loading, value }, load] = useAsyncFn( () => fieldsMetadataClient.find(params), From 9a28766f77a21c7d8c65403c4a19a2397150768f Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 28 May 2024 18:25:58 +0200 Subject: [PATCH 44/50] refactor(fields-metadata): optimize first bundle load --- packages/kbn-optimizer/limits.yml | 2 +- x-pack/plugins/fields_metadata/README.md | 4 +- .../use_fields_metadata.ts | 10 ++--- .../fields_metadata/fields_metadata_client.ts | 37 +++++++++++++------ 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 954cb099d608a..cd8399815bd72 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -61,7 +61,7 @@ pageLoadAssetSize: expressionXY: 45000 features: 21723 fieldFormats: 65209 - fieldsMetadata: 82448 + fieldsMetadata: 21885 files: 22673 filesManagement: 18683 fileUpload: 25664 diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md index 1c151fdf71017..512e06c87c09a 100755 --- a/x-pack/plugins/fields_metadata/README.md +++ b/x-pack/plugins/fields_metadata/README.md @@ -111,6 +111,8 @@ For simpler use cases, the `useFieldsMetadata` React custom hook is provided. Th | integration | string | 1password | ✅ | | dataset | string | 1password.item_usages | ✅ | +It also accepts a second argument, an array of dependencies to determine when the hook should update the retrieved data. + ```ts const FieldsComponent = () => { const { @@ -120,7 +122,7 @@ const FieldsComponent = () => { const { fieldsMetadata, error, loading } = useFieldsMetadata({ fieldsName: ['@timestamp', 'agent.name'], attributes: ['name', 'type'] - }); + }, []); if (loading) { return
Loading...
; diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 7f43272896005..98ce704010bb9 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -7,7 +7,6 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; -import hash from 'object-hash'; import { FieldAttribute, FieldName } from '../../../common'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; import { IFieldsMetadataClient } from '../../services/fields_metadata'; @@ -31,18 +30,17 @@ export interface UseFieldsMetadataReturnType { } export type UseFieldsMetadataHook = ( - params?: UseFieldsMetadataParams + params?: UseFieldsMetadataParams, + deps?: Parameters[1] ) => UseFieldsMetadataReturnType; export const createUseFieldsMetadataHook = ({ fieldsMetadataClient, }: UseFieldsMetadataFactoryDeps): UseFieldsMetadataHook => { - return (params = {}) => { - const serializedParams = hash(params, { unorderedArrays: true }); - + return (params = {}, deps) => { const [{ error, loading, value }, load] = useAsyncFn( () => fieldsMetadataClient.find(params), - [serializedParams] + deps ); useEffect(() => { diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts index 176721c704126..d7113da202b1a 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -6,12 +6,10 @@ */ import { HttpStart } from '@kbn/core/public'; -import { HashedCache } from '../../../common/hashed_cache'; +import type { HashedCache } from '../../../common/hashed_cache'; import { FindFieldsMetadataRequestQuery, - findFieldsMetadataRequestQueryRT, FindFieldsMetadataResponsePayload, - findFieldsMetadataResponsePayloadRT, } from '../../../common/latest'; import { DecodeFieldsMetadataError, @@ -19,24 +17,32 @@ import { FieldName, FIND_FIELDS_METADATA_URL, } from '../../../common/fields_metadata'; -import { decodeOrThrow } from '../../../common/runtime_types'; +// import { decodeOrThrow } from '../../../common/runtime_types'; import { IFieldsMetadataClient } from './types'; export class FieldsMetadataClient implements IFieldsMetadataClient { - private cache: HashedCache; + cache?: HashedCache; - constructor(private readonly http: HttpStart) { - this.cache = new HashedCache(); - } + constructor(private readonly http: HttpStart) {} public async find( params: FindFieldsMetadataRequestQuery ): Promise { + const cache = await this.loadLazyCache(); + // Initially lookup for existing results given request parameters - if (this.cache.has(params)) { - return this.cache.get(params) as FindFieldsMetadataResponsePayload; + if (cache.has(params)) { + return cache.get(params) as FindFieldsMetadataResponsePayload; } + const [ + { findFieldsMetadataRequestQueryRT, findFieldsMetadataResponsePayloadRT }, + { decodeOrThrow }, + ] = await Promise.all([ + import('../../../common/latest'), + import('../../../common/runtime_types'), + ]); + const query = findFieldsMetadataRequestQueryRT.encode(params); const response = await this.http @@ -56,10 +62,19 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { )(response); // Store cached results for given request parameters - this.cache.set(params, data); + cache.set(params, data); return data; } + + private async loadLazyCache() { + if (!this.cache) { + const { HashedCache } = await import('../../../common/hashed_cache'); + this.cache = new HashedCache(); + } + + return this.cache; + } } const truncateFieldNamesList = (fieldNames?: FieldName[]) => { From 4f3eeb26e75ef72a4487f9b25e1d9f04d1d113ef Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 28 May 2024 18:30:20 +0200 Subject: [PATCH 45/50] refactor(fields-metadata): remove commented import --- .../public/services/fields_metadata/fields_metadata_client.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts index d7113da202b1a..bd7af230bf1ab 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -17,7 +17,6 @@ import { FieldName, FIND_FIELDS_METADATA_URL, } from '../../../common/fields_metadata'; -// import { decodeOrThrow } from '../../../common/runtime_types'; import { IFieldsMetadataClient } from './types'; export class FieldsMetadataClient implements IFieldsMetadataClient { From e55bf021a07600306945933586a76ed46e7c692c Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 28 May 2024 20:18:26 +0200 Subject: [PATCH 46/50] refactor(fields-metadata): further bundle optimization --- .../use_fields_metadata.test.ts | 11 +++--- .../use_fields_metadata.ts | 14 ++++---- .../plugins/fields_metadata/public/mocks.tsx | 2 +- .../plugins/fields_metadata/public/plugin.ts | 6 ++-- .../fields_metadata/fields_metadata_client.ts | 36 ++++++------------- .../fields_metadata_service.mock.ts | 10 ++++-- .../fields_metadata_service.ts | 20 ++++++++--- .../public/services/fields_metadata/index.ts | 1 - .../public/services/fields_metadata/types.ts | 2 +- .../plugins/fields_metadata/public/types.ts | 4 +-- 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts index 3321ec1737076..35bbc0df8c0af 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts @@ -8,8 +8,9 @@ import { renderHook } from '@testing-library/react-hooks'; import { createUseFieldsMetadataHook, UseFieldsMetadataParams } from './use_fields_metadata'; -import { createFieldsMetadataClientMock } from '../../services/fields_metadata/fields_metadata_client.mock'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; +import { createFieldsMetadataServiceStartMock } from '../../services/fields_metadata/fields_metadata_service.mock'; +import { IFieldsMetadataClient } from '../../services/fields_metadata'; const fields: FindFieldsMetadataResponsePayload['fields'] = { '@timestamp': { @@ -29,13 +30,15 @@ const fields: FindFieldsMetadataResponsePayload['fields'] = { const mockedFieldsMetadataResponse = { fields }; -const fieldsMetadataClient = createFieldsMetadataClientMock(); +const fieldsMetadataService = createFieldsMetadataServiceStartMock(); -const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataClient }); +const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataService }); describe('useFieldsMetadata', () => { - beforeEach(() => { + let fieldsMetadataClient: jest.Mocked; + beforeEach(async () => { jest.clearAllMocks(); + fieldsMetadataClient = await fieldsMetadataService.getClient(); }); it('should return the fieldsMetadata value from the API', async () => { diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 98ce704010bb9..622cb7d066405 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -9,10 +9,10 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; import { FieldAttribute, FieldName } from '../../../common'; import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; -import { IFieldsMetadataClient } from '../../services/fields_metadata'; +import { FieldsMetadataServiceStart } from '../../services/fields_metadata'; interface UseFieldsMetadataFactoryDeps { - fieldsMetadataClient: IFieldsMetadataClient; + fieldsMetadataService: FieldsMetadataServiceStart; } export interface UseFieldsMetadataParams { @@ -35,13 +35,13 @@ export type UseFieldsMetadataHook = ( ) => UseFieldsMetadataReturnType; export const createUseFieldsMetadataHook = ({ - fieldsMetadataClient, + fieldsMetadataService, }: UseFieldsMetadataFactoryDeps): UseFieldsMetadataHook => { return (params = {}, deps) => { - const [{ error, loading, value }, load] = useAsyncFn( - () => fieldsMetadataClient.find(params), - deps - ); + const [{ error, loading, value }, load] = useAsyncFn(async () => { + const client = await fieldsMetadataService.getClient(); + return client.find(params); + }, deps); useEffect(() => { load(); diff --git a/x-pack/plugins/fields_metadata/public/mocks.tsx b/x-pack/plugins/fields_metadata/public/mocks.tsx index 338aa7be3b7d4..400ede3aa0fc7 100644 --- a/x-pack/plugins/fields_metadata/public/mocks.tsx +++ b/x-pack/plugins/fields_metadata/public/mocks.tsx @@ -9,7 +9,7 @@ import { createUseFieldsMetadataHookMock } from './hooks/use_fields_metadata/use import { createFieldsMetadataServiceStartMock } from './services/fields_metadata/fields_metadata_service.mock'; const createFieldsMetadataPublicStartMock = () => ({ - client: createFieldsMetadataServiceStartMock().client, + getClient: createFieldsMetadataServiceStartMock().getClient, useFieldsMetadata: createUseFieldsMetadataHookMock(), }); diff --git a/x-pack/plugins/fields_metadata/public/plugin.ts b/x-pack/plugins/fields_metadata/public/plugin.ts index d258368bad57c..d385b3d2ae232 100644 --- a/x-pack/plugins/fields_metadata/public/plugin.ts +++ b/x-pack/plugins/fields_metadata/public/plugin.ts @@ -26,12 +26,12 @@ export class FieldsMetadataPlugin implements FieldsMetadataClientPluginClass { public start(core: CoreStart) { const { http } = core; - const { client } = this.fieldsMetadata.start({ http }); + const fieldsMetadataService = this.fieldsMetadata.start({ http }); - const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataClient: client }); + const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataService }); return { - client, + getClient: fieldsMetadataService.getClient, useFieldsMetadata, }; } diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts index bd7af230bf1ab..176721c704126 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_client.ts @@ -6,10 +6,12 @@ */ import { HttpStart } from '@kbn/core/public'; -import type { HashedCache } from '../../../common/hashed_cache'; +import { HashedCache } from '../../../common/hashed_cache'; import { FindFieldsMetadataRequestQuery, + findFieldsMetadataRequestQueryRT, FindFieldsMetadataResponsePayload, + findFieldsMetadataResponsePayloadRT, } from '../../../common/latest'; import { DecodeFieldsMetadataError, @@ -17,31 +19,24 @@ import { FieldName, FIND_FIELDS_METADATA_URL, } from '../../../common/fields_metadata'; +import { decodeOrThrow } from '../../../common/runtime_types'; import { IFieldsMetadataClient } from './types'; export class FieldsMetadataClient implements IFieldsMetadataClient { - cache?: HashedCache; + private cache: HashedCache; - constructor(private readonly http: HttpStart) {} + constructor(private readonly http: HttpStart) { + this.cache = new HashedCache(); + } public async find( params: FindFieldsMetadataRequestQuery ): Promise { - const cache = await this.loadLazyCache(); - // Initially lookup for existing results given request parameters - if (cache.has(params)) { - return cache.get(params) as FindFieldsMetadataResponsePayload; + if (this.cache.has(params)) { + return this.cache.get(params) as FindFieldsMetadataResponsePayload; } - const [ - { findFieldsMetadataRequestQueryRT, findFieldsMetadataResponsePayloadRT }, - { decodeOrThrow }, - ] = await Promise.all([ - import('../../../common/latest'), - import('../../../common/runtime_types'), - ]); - const query = findFieldsMetadataRequestQueryRT.encode(params); const response = await this.http @@ -61,19 +56,10 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { )(response); // Store cached results for given request parameters - cache.set(params, data); + this.cache.set(params, data); return data; } - - private async loadLazyCache() { - if (!this.cache) { - const { HashedCache } = await import('../../../common/hashed_cache'); - this.cache = new HashedCache(); - } - - return this.cache; - } } const truncateFieldNamesList = (fieldNames?: FieldName[]) => { diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts index 76f892f63e80c..f5f21c1eeb4c0 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.mock.ts @@ -6,9 +6,13 @@ */ import { createFieldsMetadataClientMock } from './fields_metadata_client.mock'; -import { FieldsMetadataServiceStart } from './types'; +import { IFieldsMetadataClient } from './types'; + +interface FieldsMetadataServiceStartMock { + getClient: () => Promise>; +} export const createFieldsMetadataServiceStartMock = - (): jest.Mocked => ({ - client: createFieldsMetadataClientMock(), + (): jest.Mocked => ({ + getClient: jest.fn().mockResolvedValue(createFieldsMetadataClientMock()), }); diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts index 96b5e9464eac5..c5d363fbf9ffe 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/fields_metadata_service.ts @@ -5,23 +5,33 @@ * 2.0. */ -import { FieldsMetadataClient } from './fields_metadata_client'; -import { +import type { FieldsMetadataServiceStartDeps, FieldsMetadataServiceSetup, FieldsMetadataServiceStart, + IFieldsMetadataClient, } from './types'; export class FieldsMetadataService { + private client?: IFieldsMetadataClient; + public setup(): FieldsMetadataServiceSetup { return {}; } public start({ http }: FieldsMetadataServiceStartDeps): FieldsMetadataServiceStart { - const client = new FieldsMetadataClient(http); - return { - client, + getClient: () => this.getClient({ http }), }; } + + private async getClient({ http }: FieldsMetadataServiceStartDeps) { + if (!this.client) { + const { FieldsMetadataClient } = await import('./fields_metadata_client'); + const client = new FieldsMetadataClient(http); + this.client = client; + } + + return this.client; + } } diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts index b0253f9d2881e..76628104a8198 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/index.ts @@ -5,6 +5,5 @@ * 2.0. */ -export * from './fields_metadata_client'; export * from './fields_metadata_service'; export * from './types'; diff --git a/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts index 0f103f3a88fb4..7ab70cc8e8500 100644 --- a/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/public/services/fields_metadata/types.ts @@ -15,7 +15,7 @@ import { export interface FieldsMetadataServiceSetup {} export interface FieldsMetadataServiceStart { - client: IFieldsMetadataClient; + getClient: () => Promise; } export interface FieldsMetadataServiceStartDeps { diff --git a/x-pack/plugins/fields_metadata/public/types.ts b/x-pack/plugins/fields_metadata/public/types.ts index 34a8ee0dda61f..d7b6ff83a3164 100644 --- a/x-pack/plugins/fields_metadata/public/types.ts +++ b/x-pack/plugins/fields_metadata/public/types.ts @@ -7,13 +7,13 @@ import type { CoreSetup, CoreStart, Plugin as PluginClass } from '@kbn/core/public'; import type { UseFieldsMetadataHook } from './hooks/use_fields_metadata/use_fields_metadata'; -import type { IFieldsMetadataClient } from './services/fields_metadata'; +import type { FieldsMetadataServiceStart } from './services/fields_metadata'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldsMetadataPublicSetup {} export interface FieldsMetadataPublicStart { - client: IFieldsMetadataClient; + getClient: FieldsMetadataServiceStart['getClient']; useFieldsMetadata: UseFieldsMetadataHook; } From 0817162997133328a9dcc95435a28b9e4413cb70 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 29 May 2024 07:59:26 +0200 Subject: [PATCH 47/50] refactor(fields-metadata): revert change to dataViewFieldEditor bundle limits --- packages/kbn-optimizer/limits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index cd8399815bd72..3a535d408a910 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -30,7 +30,7 @@ pageLoadAssetSize: data: 454087 datasetQuality: 50624 dataViewEditor: 28082 - dataViewFieldEditor: 42017 + dataViewFieldEditor: 27000 dataViewManagement: 5300 dataViews: 65000 dataVisualizer: 27530 From 9b132213d1f9cf79e25275feb14e4a1c0106d908 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Wed, 29 May 2024 11:32:33 +0200 Subject: [PATCH 48/50] refactor(fields-metadata): make dataset parameter required if integration is passed --- x-pack/plugins/fields_metadata/README.md | 7 ++- .../common/fields_metadata/common.ts | 1 + .../v1/find_fields_metadata.ts | 43 +++++++++++++++++-- .../use_fields_metadata.ts | 13 +++--- .../integration_fields_repository.ts | 10 +++-- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md index 512e06c87c09a..3625425499abd 100755 --- a/x-pack/plugins/fields_metadata/README.md +++ b/x-pack/plugins/fields_metadata/README.md @@ -24,7 +24,7 @@ const timestampField = await client.getByName('@timestamp') */ ``` -- `find(params?: {fieldNames: string[], integration: string, dataset?: string})`: Retrieves a record of matching `FieldMetadata` instances based on the query parameters. +- `find(params?: {fieldNames?: string[], integration?: string, dataset?: string})`: Retrieves a record of matching `FieldMetadata` instances based on the query parameters. **Parameters** | Name | Type | Example | Optional | @@ -37,6 +37,7 @@ const timestampField = await client.getByName('@timestamp') const fields = await client.find({ fieldNames: ['@timestamp', 'onepassword.client.platform_version'], integration: '1password' + dataset: '*' }) /* { @@ -54,7 +55,9 @@ const fields = await client.find({ */ ``` -> N.B. Passing the `dataset` name parameter to `.find` helps narrowing the scope of the integration assets that need to be fetched, increasing the performance of the request. Using only the `integration` parameter should achieve the same result, but is recommended always passing the `dataset` as well if known or unless the required fields come from different datasets of the same integration. +> N.B. Passing the `dataset` name parameter to `.find` helps narrowing the scope of the integration assets that need to be fetched, increasing the performance of the request. +In case the exact dataset for a field is unknown, is it still possible to pass a `*` value as `dataset` parameter to access all the integration datasets' fields. +Still, is recommended always passing the `dataset` as well if known or unless the required fields come from different datasets of the same integration. > N.B. In case the `fieldNames` parameter is not passed to `.find`, the result will give the whole list of ECS fields by default. This should be avoided as much as possible, although it helps covering cases where we might need the whole ECS fields list. diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts index 4b5c58958d08b..dbc7857abd1b8 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/common.ts @@ -6,3 +6,4 @@ */ export const FIND_FIELDS_METADATA_URL = '/internal/fields_metadata'; +export const ANY_DATASET = '*'; diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts index fd06a941109dc..d293f9232a6f7 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/v1/find_fields_metadata.ts @@ -6,10 +6,13 @@ */ import { arrayToStringRt } from '@kbn/io-ts-utils'; +import { either } from 'fp-ts/lib/Either'; import * as rt from 'io-ts'; -import { fieldAttributeRT, partialFieldMetadataPlainRT } from '../types'; +import { ANY_DATASET } from '../common'; +import { FetchFieldsMetadataError } from '../errors'; +import { FieldAttribute, fieldAttributeRT, FieldName, partialFieldMetadataPlainRT } from '../types'; -export const findFieldsMetadataRequestQueryRT = rt.exact( +const baseFindFieldsMetadataRequestQueryRT = rt.exact( rt.partial({ attributes: arrayToStringRt.pipe(rt.array(fieldAttributeRT)), fieldNames: arrayToStringRt.pipe(rt.array(rt.string)), @@ -18,11 +21,45 @@ export const findFieldsMetadataRequestQueryRT = rt.exact( }) ); +// Define a refinement that enforces the constraint +export const findFieldsMetadataRequestQueryRT = new rt.Type( + 'FindFieldsMetadataRequestQuery', + (query): query is rt.TypeOf => + baseFindFieldsMetadataRequestQueryRT.is(query) && + (query.integration ? query.dataset !== undefined : true), + (input, context) => + either.chain(baseFindFieldsMetadataRequestQueryRT.validate(input, context), (query) => { + try { + if (query.integration && !query.dataset) { + throw new FetchFieldsMetadataError('dataset is required if integration is provided'); + } + + return rt.success(query); + } catch (error) { + return rt.failure(query, context, error.message); + } + }), + baseFindFieldsMetadataRequestQueryRT.encode +); + export const findFieldsMetadataResponsePayloadRT = rt.type({ fields: rt.record(rt.string, partialFieldMetadataPlainRT), }); -export type FindFieldsMetadataRequestQuery = rt.TypeOf; +export type FindFieldsMetadataRequestQuery = + | { + attributes?: FieldAttribute[]; + fieldNames?: FieldName[]; + integration?: undefined; + dataset?: undefined; + } + | { + attributes?: FieldAttribute[]; + fieldNames?: FieldName[]; + integration: string; + dataset: typeof ANY_DATASET | (string & {}); + }; + export type FindFieldsMetadataResponsePayload = rt.TypeOf< typeof findFieldsMetadataResponsePayloadRT >; diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts index 622cb7d066405..ff8a07ebba338 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.ts @@ -7,20 +7,17 @@ import { useEffect } from 'react'; import useAsyncFn from 'react-use/lib/useAsyncFn'; -import { FieldAttribute, FieldName } from '../../../common'; -import { FindFieldsMetadataResponsePayload } from '../../../common/latest'; +import { + FindFieldsMetadataRequestQuery, + FindFieldsMetadataResponsePayload, +} from '../../../common/latest'; import { FieldsMetadataServiceStart } from '../../services/fields_metadata'; interface UseFieldsMetadataFactoryDeps { fieldsMetadataService: FieldsMetadataServiceStart; } -export interface UseFieldsMetadataParams { - attributes?: FieldAttribute[]; - fieldNames?: FieldName[]; - integration?: string; - dataset?: string; -} +export type UseFieldsMetadataParams = FindFieldsMetadataRequestQuery; export interface UseFieldsMetadataReturnType { fieldsMetadata: FindFieldsMetadataResponsePayload['fields'] | undefined; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts index 5265c55c66bb3..0f25d9357855e 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/integration_fields_repository.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { ANY_DATASET } from '../../../../common/fields_metadata'; import { HashedCache } from '../../../../common/hashed_cache'; import { FieldMetadata, IntegrationFieldName } from '../../../../common'; import { @@ -73,6 +74,7 @@ export class IntegrationFieldsRepository { ): FieldMetadata | undefined { const cacheKey = this.getCacheKey({ integration, dataset }); const cachedIntegration = this.cache.get(cacheKey); + const datasetName = dataset === ANY_DATASET ? null : dataset; // 1. Integration fields were never fetched if (!cachedIntegration) { @@ -80,18 +82,18 @@ export class IntegrationFieldsRepository { } // 2. Dataset is passed but was never fetched before - if (dataset && !cachedIntegration.hasOwnProperty(dataset)) { + if (datasetName && !cachedIntegration.hasOwnProperty(datasetName)) { return undefined; } // 3. Dataset is passed and it was previously fetched, should return the field - if (dataset && cachedIntegration.hasOwnProperty(dataset)) { - const targetDataset = cachedIntegration[dataset]; + if (datasetName && cachedIntegration.hasOwnProperty(datasetName)) { + const targetDataset = cachedIntegration[datasetName]; return targetDataset[fieldName]; } // 4. Dataset is not passed, we attempt search on all stored datasets - if (!dataset) { + if (!datasetName) { // Merge all the available datasets into a unique field list. Overriding fields might occur in the process. const cachedDatasetsFields = Object.assign({}, ...Object.values(cachedIntegration)); return cachedDatasetsFields[fieldName]; From b065506944ed7f48e50236ddb78f6badc9535a9d Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Thu, 30 May 2024 12:25:19 +0200 Subject: [PATCH 49/50] tests(fleet): add tests for data stream fields resolution --- .../services/epm/packages/utils.test.ts | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/utils.test.ts diff --git a/x-pack/plugins/fleet/server/services/epm/packages/utils.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/utils.test.ts new file mode 100644 index 0000000000000..166687a836fb1 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/utils.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dump } from 'js-yaml'; + +import type { AssetsMap } from '../../../../common/types'; + +import type { RegistryDataStream } from '../../../../common'; + +import { resolveDataStreamFields } from './utils'; + +describe('resolveDataStreamFields', () => { + const statusAssetYml = dump([ + { + name: 'apache.status', + type: 'group', + fields: [ + { + name: 'total_accesses', + type: 'long', + description: 'Total number of access requests.\n', + metric_type: 'counter', + }, + { + name: 'uptime', + type: 'group', + fields: [ + { + name: 'server_uptime', + type: 'long', + description: 'Server uptime in seconds.\n', + metric_type: 'counter', + }, + { + name: 'uptime', + type: 'long', + description: 'Server uptime.\n', + metric_type: 'counter', + }, + ], + }, + ], + }, + ]); + + const dataStream = { + dataset: 'apache.status', + path: 'status', + } as RegistryDataStream; + + const assetsMap = new Map([ + ['apache-1.18.0/data_stream/status/fields/fields.yml', Buffer.from(statusAssetYml)], + ]) as AssetsMap; + + const expectedResult = { + 'apache.status': { + 'apache.status.total_accesses': { + name: 'total_accesses', + type: 'long', + description: 'Total number of access requests.\n', + metric_type: 'counter', + flat_name: 'apache.status.total_accesses', + }, + 'apache.status.uptime.server_uptime': { + name: 'server_uptime', + type: 'long', + description: 'Server uptime in seconds.\n', + metric_type: 'counter', + flat_name: 'apache.status.uptime.server_uptime', + }, + 'apache.status.uptime.uptime': { + name: 'uptime', + type: 'long', + description: 'Server uptime.\n', + metric_type: 'counter', + flat_name: 'apache.status.uptime.uptime', + }, + }, + }; + + it('should load and resolve fields for the passed data stream', () => { + expect(resolveDataStreamFields({ dataStream, assetsMap })).toEqual(expectedResult); + }); +}); From b2b7118ce4246dfce5a0610b10415914c9114912 Mon Sep 17 00:00:00 2001 From: Marco Antonio Ghiani Date: Tue, 4 Jun 2024 17:27:43 +0200 Subject: [PATCH 50/50] refactor(fields-metadata): address comments --- x-pack/plugins/fields_metadata/README.md | 2 +- .../common/fields_metadata/models/field_metadata.ts | 1 + .../hooks/use_fields_metadata/use_fields_metadata.test.ts | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md index 3625425499abd..d08b18a029345 100755 --- a/x-pack/plugins/fields_metadata/README.md +++ b/x-pack/plugins/fields_metadata/README.md @@ -36,7 +36,7 @@ const timestampField = await client.getByName('@timestamp') ```ts const fields = await client.find({ fieldNames: ['@timestamp', 'onepassword.client.platform_version'], - integration: '1password' + integration: '1password', dataset: '*' }) /* diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts index d12cbbaf15bc0..fcde43b3c4e11 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/field_metadata.ts @@ -10,6 +10,7 @@ import pick from 'lodash/pick'; import { FieldAttribute, FieldMetadataPlain, PartialFieldMetadataPlain } from '../types'; +// Use class/interface merging to define instance properties from FieldMetadataPlain. // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface FieldMetadata extends FieldMetadataPlain {} export class FieldMetadata { diff --git a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts index 35bbc0df8c0af..eaae7c5542c0b 100644 --- a/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts +++ b/x-pack/plugins/fields_metadata/public/hooks/use_fields_metadata/use_fields_metadata.test.ts @@ -37,10 +37,13 @@ const useFieldsMetadata = createUseFieldsMetadataHook({ fieldsMetadataService }) describe('useFieldsMetadata', () => { let fieldsMetadataClient: jest.Mocked; beforeEach(async () => { - jest.clearAllMocks(); fieldsMetadataClient = await fieldsMetadataService.getClient(); }); + afterEach(() => { + jest.clearAllMocks(); + }); + it('should return the fieldsMetadata value from the API', async () => { fieldsMetadataClient.find.mockResolvedValue(mockedFieldsMetadataResponse); const { result, waitForNextUpdate } = renderHook(() => useFieldsMetadata());