[Inventory][ECO] Replace Entity with InventoryEntityLatest type

packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts

@@ -58,6 +58,8 @@ export class K8sEntity extends Serializable<EntityFields> {
       'entity.definition_id': `builtin_${entityTypeWithSchema}`,
       'entity.identity_fields': identityFields,
       'entity.display_name': getDisplayName({ identityFields, fields }),
+      'entity.definition_version': '1.0.0',
+      'entity.schema_version': '1.0',
     });
   }
 }

src/plugins/home/public/application/components/tutorial/replace_template_strings.js

@@ -38,7 +38,6 @@ export function replaceTemplateStrings(text, params = {}) {
           filebeat: docLinks.links.filebeat.base,
           metricbeat: docLinks.links.metricbeat.base,
           heartbeat: docLinks.links.heartbeat.base,
-          functionbeat: docLinks.links.functionbeat.base,
           winlogbeat: docLinks.links.winlogbeat.base,
           auditbeat: docLinks.links.auditbeat.base,
         },

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -11,9 +11,9 @@ import { arrayOfStringsSchema } from './common';
 export const entityBaseSchema = z.object({
   id: z.string(),
   type: z.string(),
-  identity_fields: arrayOfStringsSchema,
+  identity_fields: z.union([arrayOfStringsSchema, z.string()]),
   display_name: z.string(),
-  metrics: z.record(z.string(), z.number()),
+  metrics: z.optional(z.record(z.string(), z.number())),
   definition_version: z.string(),
   schema_version: z.string(),
   definition_id: z.string(),
@@ -24,10 +24,13 @@ export interface MetadataRecord {
 }
 
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
+
 type Literal = z.infer<typeof literalSchema>;
-type Metadata = Literal | { [key: string]: Metadata } | Metadata[];
+interface Metadata {
+  [key: string]: Metadata | Literal | Literal[];
+}
 export const entityMetadataSchema: z.ZodType<Metadata> = z.lazy(() =>
-  z.union([literalSchema, z.array(entityMetadataSchema), z.record(entityMetadataSchema)])
+  z.record(z.string(), z.union([literalSchema, z.array(literalSchema), entityMetadataSchema]))
 );
 
 export const entityLatestSchema = z
@@ -39,3 +42,6 @@ export const entityLatestSchema = z
     ),
   })
   .and(entityMetadataSchema);
+
+export type EntityInstance = z.infer<typeof entityLatestSchema>;
+export type EntityMetadata = z.infer<typeof entityMetadataSchema>;

x-pack/packages/observability/observability_utils/array/join_by_key.test.ts

@@ -221,4 +221,50 @@ describe('joinByKey', () => {
       },
     });
   });
+
+  it('deeply merges by unflatten keys', () => {
+    const joined = joinByKey(
+      [
+        {
+          service: {
+            name: 'opbeans-node',
+            metrics: {
+              cpu: 0.1,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+        {
+          service: {
+            environment: 'prod',
+            metrics: {
+              memory: 0.5,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+      ],
+      'properties.foo'
+    );
+
+    expect(joined).toEqual([
+      {
+        service: {
+          name: 'opbeans-node',
+          environment: 'prod',
+          metrics: {
+            cpu: 0.1,
+            memory: 0.5,
+          },
+        },
+        properties: {
+          foo: 'bar',
+        },
+      },
+    ]);
+  });
 });

x-pack/packages/observability/observability_utils/array/join_by_key.ts

@@ -18,18 +18,29 @@ export type JoinedReturnType<
   }
 >;
 
-type ArrayOrSingle<T> = T | T[];
+function getValueByPath(obj: any, path: string): any {
+  return path.split('.').reduce((acc, keyPart) => {
+    // Check if acc is a valid object and has the key
+    return acc && Object.prototype.hasOwnProperty.call(acc, keyPart) ? acc[keyPart] : undefined;
+  }, obj);
+}
 
+type NestedKeys<T> = T extends object
+  ? { [K in keyof T]: K extends string ? `${K}` | `${K}.${NestedKeys<T[K]>}` : never }[keyof T]
+  : never;
+
+type ArrayOrSingle<T> = T | T[];
+type CombinedNestedKeys<T, U> = (NestedKeys<T> & NestedKeys<U>) | (keyof T & keyof U);
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>
 >(items: T[], key: V): JoinedReturnType<T, U>;
 
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>,
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>,
   W extends JoinedReturnType<T, U>,
   X extends (a: T, b: T) => ValuesType<W>
 >(items: T[], key: V, mergeFn: X): W;
@@ -45,7 +56,7 @@ export function joinByKey(
   items.forEach((current) => {
     // The key of the map is a stable JSON string of the values from given keys.
     // We need stable JSON string to support plain object values.
-    const stableKey = stableStringify(keys.map((k) => current[k]));
+    const stableKey = stableStringify(keys.map((k) => current[k] ?? getValueByPath(current, k)));
 
     if (map.has(stableKey)) {
       const item = map.get(stableKey);

x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts

@@ -9,13 +9,28 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import { withSpan } from '@kbn/apm-utils';
 import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types';
+import { esqlResultToPlainObjects } from '../utils/esql_result_to_plain_objects';
 
 type SearchRequest = ESSearchRequest & {
   index: string | string[];
   track_total_hits: number | boolean;
   size: number | boolean;
 };
 
+type EsqlQueryParameters = EsqlQueryRequest & { parseOutput?: boolean };
+type EsqlOutputParameters = Omit<EsqlQueryRequest, 'format' | 'columnar'> & {
+  parseOutput?: true;
+  format?: 'json';
+  columnar?: false;
+};
+
+type EsqlParameters = EsqlOutputParameters | EsqlQueryParameters;
+
+export type InferEsqlResponseOf<
+  TOutput = unknown,
+  TParameters extends EsqlParameters = EsqlParameters
+> = TParameters extends EsqlOutputParameters ? TOutput[] : ESQLSearchResponse;
+
 /**
  * An Elasticsearch Client with a fully typed `search` method and built-in
  * APM instrumentation.
@@ -25,7 +40,14 @@ export interface ObservabilityElasticsearchClient {
     operationName: string,
     parameters: TSearchRequest
   ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
-  esql(operationName: string, parameters: EsqlQueryRequest): Promise<ESQLSearchResponse>;
+  esql<TOutput = unknown, TQueryParams extends EsqlOutputParameters = EsqlOutputParameters>(
+    operationName: string,
+    parameters: TQueryParams
+  ): Promise<InferEsqlResponseOf<TOutput, TQueryParams>>;
+  esql<TOutput = unknown, TQueryParams extends EsqlQueryParameters = EsqlQueryParameters>(
+    operationName: string,
+    parameters: TQueryParams
+  ): Promise<InferEsqlResponseOf<TOutput, TQueryParams>>;
   client: ElasticsearchClient;
 }
 
@@ -40,11 +62,14 @@ export function createObservabilityEsClient({
 }): ObservabilityElasticsearchClient {
   return {
     client,
-    esql(operationName: string, parameters: EsqlQueryRequest) {
+    esql<TOutput = unknown, TSearchRequest extends EsqlParameters = EsqlParameters>(
+      operationName: string,
+      { parseOutput = true, format = 'json', columnar = false, ...parameters }: TSearchRequest
+    ) {
       logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`);
       return withSpan({ name: operationName, labels: { plugin } }, () => {
         return client.esql.query(
-          { ...parameters },
+          { ...parameters, format, columnar },
           {
             querystring: {
               drop_null_columns: true,
@@ -54,7 +79,11 @@ export function createObservabilityEsClient({
       })
         .then((response) => {
           logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`);
-          return response as unknown as ESQLSearchResponse;
+
+          const esqlResponse = response as unknown as ESQLSearchResponse;
+
+          const shouldParseOutput = parseOutput && !columnar && format === 'json';
+          return shouldParseOutput ? esqlResultToPlainObjects<TOutput>(esqlResponse) : esqlResponse;
         })
         .catch((error) => {
           throw error;

x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts

@@ -6,25 +6,28 @@
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';
+import { unflattenObject } from '../../object/unflatten_object';
 
-export function esqlResultToPlainObjects<T extends Record<string, any>>(
+export function esqlResultToPlainObjects<TDocument = unknown>(
   result: ESQLSearchResponse
-): T[] {
+): TDocument[] {
   return result.values.map((row) => {
-    return row.reduce<Record<string, unknown>>((acc, value, index) => {
-      const column = result.columns[index];
+    return unflattenObject(
+      row.reduce<Record<string, any>>((acc, value, index) => {
+        const column = result.columns[index];
 
-      if (!column) {
-        return acc;
-      }
+        if (!column) {
+          return acc;
+        }
 
-      // Removes the type suffix from the column name
-      const name = column.name.replace(/\.(text|keyword)$/, '');
-      if (!acc[name]) {
-        acc[name] = value;
-      }
+        // Removes the type suffix from the column name
+        const name = column.name.replace(/\.(text|keyword)$/, '');
+        if (!acc[name]) {
+          acc[name] = value;
+        }
 
-      return acc;
-    }, {});
-  }) as T[];
+        return acc;
+      }, {})
+    ) as TDocument;
+  });
 }

x-pack/plugins/entity_manager/public/lib/entity_client.test.ts

@@ -5,16 +5,17 @@
  * 2.0.
  */
 
-import { EntityClient, EnitityInstance } from './entity_client';
+import { EntityClient } from './entity_client';
 import { coreMock } from '@kbn/core/public/mocks';
+import type { EntityInstance } from '@kbn/entities-schema';
 
-const commonEntityFields: EnitityInstance = {
+const commonEntityFields: EntityInstance = {
   entity: {
     last_seen_timestamp: '2023-10-09T00:00:00Z',
     id: '1',
     display_name: 'entity_name',
     definition_id: 'entity_definition_id',
-  } as EnitityInstance['entity'],
+  } as EntityInstance['entity'],
 };
 
 describe('EntityClient', () => {
@@ -26,7 +27,7 @@ describe('EntityClient', () => {
 
   describe('asKqlFilter', () => {
     it('should return the kql filter', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['service.name', 'service.environment'],
@@ -42,7 +43,7 @@ describe('EntityClient', () => {
     });
 
     it('should return the kql filter when indentity_fields is composed by multiple fields', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['service.name', 'service.environment'],
@@ -59,7 +60,7 @@ describe('EntityClient', () => {
     });
 
     it('should ignore fields that are not present in the entity', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['host.name', 'foo.bar'],
@@ -76,7 +77,7 @@ describe('EntityClient', () => {
 
   describe('getIdentityFieldsValue', () => {
     it('should return identity fields values', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['service.name', 'service.environment'],
@@ -93,7 +94,7 @@ describe('EntityClient', () => {
     });
 
     it('should return identity fields values when indentity_fields is composed by multiple fields', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['service.name', 'service.environment'],
@@ -112,7 +113,7 @@ describe('EntityClient', () => {
     });
 
     it('should return identity fields when field is in the root', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         entity: {
           ...commonEntityFields.entity,
           identity_fields: ['name'],
@@ -127,7 +128,7 @@ describe('EntityClient', () => {
     });
 
     it('should throw an error when identity fields are missing', () => {
-      const entityLatest: EnitityInstance = {
+      const entityLatest: EntityInstance = {
         ...commonEntityFields,
       };