[Spacetime] Fields metadata services

.github/CODEOWNERS

@@ -448,6 +448,7 @@ examples/field_formats_example @elastic/kibana-data-discovery
 src/plugins/field_formats @elastic/kibana-data-discovery
 packages/kbn-field-types @elastic/kibana-data-discovery
 packages/kbn-field-utils @elastic/kibana-data-discovery
+x-pack/plugins/fields_metadata @elastic/obs-ux-logs-team
 x-pack/plugins/file_upload @elastic/kibana-gis
 examples/files_example @elastic/appex-sharedux
 src/plugins/files_management @elastic/appex-sharedux

docs/developer/plugin-list.asciidoc

@@ -582,6 +582,10 @@ activities.
 |The features plugin enhance Kibana with a per-feature privilege system.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/fields_metadata/README.md[fieldsMetadata]
+|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload]
 |WARNING: Missing README.
 

package.json

@@ -492,6 +492,7 @@
     "@kbn/field-formats-plugin": "link:src/plugins/field_formats",
     "@kbn/field-types": "link:packages/kbn-field-types",
     "@kbn/field-utils": "link:packages/kbn-field-utils",
+    "@kbn/fields-metadata-plugin": "link:x-pack/plugins/fields_metadata",
     "@kbn/file-upload-plugin": "link:x-pack/plugins/file_upload",
     "@kbn/files-example-plugin": "link:examples/files_example",
     "@kbn/files-management-plugin": "link:src/plugins/files_management",

packages/kbn-io-ts-utils/index.ts

@@ -9,6 +9,7 @@
 export type { IndexPattern } from './src/index_pattern_rt';
 export type { NonEmptyString, NonEmptyStringBrand } from './src/non_empty_string_rt';
 
+export { arrayToStringRt } from './src/array_to_string_rt';
 export { deepExactRt } from './src/deep_exact_rt';
 export { indexPatternRt } from './src/index_pattern_rt';
 export { jsonRt } from './src/json_rt';

packages/kbn-io-ts-utils/src/array_to_string_rt/index.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as rt from 'io-ts';
+import { arrayToStringRt } from '.';
+import { isRight, Either, fold } from 'fp-ts/lib/Either';
+import { identity } from 'fp-ts/lib/function';
+
+function getValueOrThrow(either: Either<unknown, any>) {
+  return fold(() => {
+    throw new Error('Cannot get right value of left');
+  }, identity)(either);
+}
+
+describe('arrayToStringRt', () => {
+  it('should validate strings', () => {
+    expect(isRight(arrayToStringRt.decode(''))).toBe(true);
+    expect(isRight(arrayToStringRt.decode('message'))).toBe(true);
+    expect(isRight(arrayToStringRt.decode('message,event.original'))).toBe(true);
+    expect(isRight(arrayToStringRt.decode({}))).toBe(false);
+    expect(isRight(arrayToStringRt.decode(true))).toBe(false);
+  });
+
+  it('should return array of strings when decoding', () => {
+    expect(getValueOrThrow(arrayToStringRt.decode(''))).toEqual(['']);
+    expect(getValueOrThrow(arrayToStringRt.decode('message'))).toEqual(['message']);
+    expect(getValueOrThrow(arrayToStringRt.decode('message,event.original'))).toEqual([
+      'message',
+      'event.original',
+    ]);
+  });
+
+  it('should be pipable', () => {
+    const piped = arrayToStringRt.pipe(rt.array(rt.string));
+
+    const validInput = ['message', 'event.original'];
+    const invalidInput = {};
+
+    const valid = piped.decode(validInput.join(','));
+    const invalid = piped.decode(invalidInput);
+
+    expect(isRight(valid)).toBe(true);
+    expect(getValueOrThrow(valid)).toEqual(validInput);
+
+    expect(isRight(invalid)).toBe(false);
+  });
+});

packages/kbn-io-ts-utils/src/array_to_string_rt/index.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { either } from 'fp-ts/lib/Either';
+import * as rt from 'io-ts';
+
+export const arrayToStringRt = new rt.Type<unknown[], string, unknown>(
+  'arrayToString',
+  rt.array(rt.unknown).is,
+  (input, context) =>
+    either.chain(rt.string.validate(input, context), (str) => {
+      try {
+        return rt.success(str.split(','));
+      } catch (e) {
+        return rt.failure(input, context);
+      }
+    }),
+  (arr) => arr.join(',')
+);

packages/kbn-optimizer/limits.yml

@@ -62,6 +62,7 @@ pageLoadAssetSize:
   expressionXY: 45000
   features: 21723
   fieldFormats: 65209
+  fieldsMetadata: 21885
   files: 22673
   filesManagement: 18683
   fileUpload: 25664

src/plugins/unified_doc_viewer/kibana.jsonc

@@ -9,5 +9,6 @@
     "browser": true,
     "requiredBundles": ["kibanaUtils"],
     "requiredPlugins": ["data", "discoverShared", "fieldFormats"],
+    "optionalPlugins": ["fieldsMetadata"]
   }
 }

src/plugins/unified_doc_viewer/public/__mocks__/services.ts

@@ -10,6 +10,7 @@ import { analyticsServiceMock } from '@kbn/core-analytics-browser-mocks';
 import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
 import { discoverSharedPluginMock } from '@kbn/discover-shared-plugin/public/mocks';
 import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks';
+import { fieldsMetadataPluginPublicMock } from '@kbn/fields-metadata-plugin/public/mocks';
 import { uiSettingsServiceMock } from '@kbn/core-ui-settings-browser-mocks';
 import type { UnifiedDocViewerServices, UnifiedDocViewerStart } from '../types';
 import { Storage } from '@kbn/kibana-utils-plugin/public';
@@ -24,6 +25,7 @@ export const mockUnifiedDocViewerServices: jest.Mocked<UnifiedDocViewerServices>
   data: dataPluginMock.createStartContract(),
   discoverShared: discoverSharedPluginMock.createStartContract(),
   fieldFormats: fieldFormatsMock,
+  fieldsMetadata: fieldsMetadataPluginPublicMock.createStartContract(),
   storage: new Storage(localStorage),
   uiSettings: uiSettingsServiceMock.createStartContract(),
   unifiedDocViewer: mockUnifiedDocViewer,

src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx

@@ -13,6 +13,7 @@ import { i18n } from '@kbn/i18n';
 import { DataTableRecord, LogDocumentOverview, fieldConstants } from '@kbn/discover-utils';
 import { HighlightField } from './sub_components/highlight_field';
 import { HighlightSection } from './sub_components/highlight_section';
+import { getUnifiedDocViewerServices } from '../../plugin';
 
 export function LogsOverviewHighlights({
   formattedDoc,
@@ -21,6 +22,30 @@ export function LogsOverviewHighlights({
   formattedDoc: LogDocumentOverview;
   flattenedDoc: DataTableRecord['flattened'];
 }) {
+  const {
+    fieldsMetadata: { useFieldsMetadata },
+  } = getUnifiedDocViewerServices();
+
+  const { fieldsMetadata = {} } = useFieldsMetadata({
+    attributes: ['flat_name', 'short', 'type'],
+    fieldNames: [
+      fieldConstants.SERVICE_NAME_FIELD,
+      fieldConstants.HOST_NAME_FIELD,
+      fieldConstants.TRACE_ID_FIELD,
+      fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD,
+      fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD,
+      fieldConstants.CLOUD_PROVIDER_FIELD,
+      fieldConstants.CLOUD_REGION_FIELD,
+      fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD,
+      fieldConstants.CLOUD_PROJECT_ID_FIELD,
+      fieldConstants.CLOUD_INSTANCE_ID_FIELD,
+      fieldConstants.LOG_FILE_PATH_FIELD,
+      fieldConstants.DATASTREAM_DATASET_FIELD,
+      fieldConstants.DATASTREAM_NAMESPACE_FIELD,
+      fieldConstants.AGENT_NAME_FIELD,
+    ],
+  });
+
   const getHighlightProps = (field: keyof LogDocumentOverview) => ({
     field,
     formattedValue: formattedDoc[field],
@@ -38,34 +63,39 @@ export function LogsOverviewHighlights({
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewService"
             label={serviceLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.SERVICE_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.SERVICE_NAME_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.HOST_NAME_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewHostName"
             label={hostNameLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.HOST_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.HOST_NAME_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.TRACE_ID_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewTrace"
             label={traceLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.TRACE_ID_FIELD]}
             {...getHighlightProps(fieldConstants.TRACE_ID_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewClusterName"
             label={orchestratorClusterNameLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewResourceId"
             label={orchestratorResourceIdLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD]}
             {...getHighlightProps(fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD)}
           />
         )}
@@ -79,6 +109,7 @@ export function LogsOverviewHighlights({
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudProvider"
             label={cloudProviderLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_PROVIDER_FIELD]}
             icon={
               <CloudProviderIcon
                 cloudProvider={first(
@@ -93,27 +124,31 @@ export function LogsOverviewHighlights({
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudRegion"
             label={cloudRegionLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_REGION_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_REGION_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudAz"
             label={cloudAvailabilityZoneLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.CLOUD_PROJECT_ID_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudProjectId"
             label={cloudProjectIdLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_PROJECT_ID_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_PROJECT_ID_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.CLOUD_INSTANCE_ID_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudInstanceId"
             label={cloudInstanceIdLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_INSTANCE_ID_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_INSTANCE_ID_FIELD)}
           />
         )}
@@ -127,20 +162,23 @@ export function LogsOverviewHighlights({
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewLogPathFile"
             label={logPathFileLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.LOG_FILE_PATH_FIELD]}
             {...getHighlightProps(fieldConstants.LOG_FILE_PATH_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.DATASTREAM_DATASET_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewDataset"
             label={datasetLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.DATASTREAM_DATASET_FIELD]}
             {...getHighlightProps(fieldConstants.DATASTREAM_DATASET_FIELD)}
           />
         )}
         {formattedDoc[fieldConstants.DATASTREAM_NAMESPACE_FIELD] && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewNamespace"
             label={namespaceLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.DATASTREAM_NAMESPACE_FIELD]}
             useBadge
             {...getHighlightProps(fieldConstants.DATASTREAM_NAMESPACE_FIELD)}
           />
@@ -149,6 +187,7 @@ export function LogsOverviewHighlights({
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewLogShipper"
             label={shipperLabel}
+            fieldMetadata={fieldsMetadata[fieldConstants.AGENT_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.AGENT_NAME_FIELD)}
           />
         )}

src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field.tsx

@@ -11,12 +11,14 @@ import { css } from '@emotion/react';
 import React, { ReactNode } from 'react';
 import { dynamic } from '@kbn/shared-ux-utility';
 import { euiThemeVars } from '@kbn/ui-theme';
+import { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common';
 import { HoverActionPopover } from './hover_popover_action';
 
 const HighlightFieldDescription = dynamic(() => import('./highlight_field_description'));
 
 interface HighlightFieldProps {
   field: string;
+  fieldMetadata?: PartialFieldMetadataPlain;
   formattedValue?: string;
   icon?: ReactNode;
   label: string;
@@ -26,20 +28,23 @@ interface HighlightFieldProps {
 
 export function HighlightField({
   field,
+  fieldMetadata,
   formattedValue,
   icon,
   label,
   useBadge = false,
   value,
   ...props
 }: HighlightFieldProps) {
+  const hasFieldDescription = !!fieldMetadata?.short;
+
   return formattedValue && value ? (
     <div {...props}>
       <EuiFlexGroup responsive={false} alignItems="center" gutterSize="xs">
         <EuiTitle css={fieldNameStyle} size="xxxs">
           <span>{label}</span>
         </EuiTitle>
-        <HighlightFieldDescription fieldName={field} />
+        {hasFieldDescription ? <HighlightFieldDescription fieldMetadata={fieldMetadata} /> : null}
       </EuiFlexGroup>
       <HoverActionPopover title={value} value={value} field={field}>
         <EuiFlexGroup

src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview/sub_components/highlight_field_description.tsx

@@ -7,14 +7,16 @@
  */
 
 import { EuiFlexGroup, EuiIconTip } from '@elastic/eui';
-import { EcsFlat } from '@elastic/ecs';
+import { PartialFieldMetadataPlain } from '@kbn/fields-metadata-plugin/common';
 import { FieldIcon } from '@kbn/react-field';
 import React from 'react';
 
-export function HighlightFieldDescription({ fieldName }: { fieldName: string }) {
-  const { short, type } = EcsFlat[fieldName as keyof typeof EcsFlat] ?? {};
-
-  if (!short) return null;
+export function HighlightFieldDescription({
+  fieldMetadata,
+}: {
+  fieldMetadata: PartialFieldMetadataPlain;
+}) {
+  const { flat_name: fieldName, short, type } = fieldMetadata;
 
   const title = (
     <EuiFlexGroup alignItems="center" gutterSize="s">

src/plugins/unified_doc_viewer/public/plugin.tsx

@@ -18,6 +18,7 @@ import { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { CoreStart } from '@kbn/core/public';
 import { dynamic } from '@kbn/shared-ux-utility';
 import { DiscoverSharedPublicStart } from '@kbn/discover-shared-plugin/public';
+import { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { UnifiedDocViewerServices } from './types';
 
 export const [getUnifiedDocViewerServices, setUnifiedDocViewerServices] =
@@ -50,6 +51,7 @@ export interface UnifiedDocViewerStartDeps {
   data: DataPublicPluginStart;
   discoverShared: DiscoverSharedPublicStart;
   fieldFormats: FieldFormatsStart;
+  fieldsMetadata: FieldsMetadataPublicStart;
 }
 
 export class UnifiedDocViewerPublicPlugin
@@ -118,7 +120,7 @@ export class UnifiedDocViewerPublicPlugin
 
   public start(core: CoreStart, deps: UnifiedDocViewerStartDeps) {
     const { analytics, uiSettings } = core;
-    const { data, discoverShared, fieldFormats } = deps;
+    const { data, discoverShared, fieldFormats, fieldsMetadata } = deps;
     const storage = new Storage(localStorage);
     const unifiedDocViewer = {
       registry: this.docViewsRegistry,
@@ -128,6 +130,7 @@ export class UnifiedDocViewerPublicPlugin
       data,
       discoverShared,
       fieldFormats,
+      fieldsMetadata,
       storage,
       uiSettings,
       unifiedDocViewer,