[Inventory][ECO] Replace Entity with InventoryEntityLatest type

packages/kbn-apm-synthtrace-client/src/lib/entities/kubernetes/index.ts

@@ -58,6 +58,8 @@ export class K8sEntity extends Serializable<EntityFields> {
       'entity.definition_id': `builtin_${entityTypeWithSchema}`,
       'entity.identity_fields': identityFields,
       'entity.display_name': getDisplayName({ identityFields, fields }),
+      'entity.definition_version': '1.0.0',
+      'entity.schema_version': '1.0',
     });
   }
 }

src/plugins/home/public/application/components/tutorial/replace_template_strings.js

@@ -38,7 +38,6 @@ export function replaceTemplateStrings(text, params = {}) {
           filebeat: docLinks.links.filebeat.base,
           metricbeat: docLinks.links.metricbeat.base,
           heartbeat: docLinks.links.heartbeat.base,
-          functionbeat: docLinks.links.functionbeat.base,
           winlogbeat: docLinks.links.winlogbeat.base,
           auditbeat: docLinks.links.auditbeat.base,
         },

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -11,9 +11,9 @@ import { arrayOfStringsSchema } from './common';
 export const entityBaseSchema = z.object({
   id: z.string(),
   type: z.string(),
-  identity_fields: arrayOfStringsSchema,
+  identity_fields: z.union([arrayOfStringsSchema, z.string()]),
   display_name: z.string(),
-  metrics: z.record(z.string(), z.number()),
+  metrics: z.optional(z.record(z.string(), z.number())),
   definition_version: z.string(),
   schema_version: z.string(),
   definition_id: z.string(),
@@ -24,10 +24,13 @@ export interface MetadataRecord {
 }
 
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
+
 type Literal = z.infer<typeof literalSchema>;
-type Metadata = Literal | { [key: string]: Metadata } | Metadata[];
+interface Metadata {
+  [key: string]: Metadata | Literal | Literal[];
+}
 export const entityMetadataSchema: z.ZodType<Metadata> = z.lazy(() =>
-  z.union([literalSchema, z.array(entityMetadataSchema), z.record(entityMetadataSchema)])
+  z.record(z.string(), z.union([literalSchema, z.array(literalSchema), entityMetadataSchema]))
 );
 
 export const entityLatestSchema = z

x-pack/packages/observability/observability_utils/array/join_by_key.test.ts

@@ -221,4 +221,50 @@ describe('joinByKey', () => {
       },
     });
   });
+
+  it('deeply merges by unflatten keys', () => {
+    const joined = joinByKey(
+      [
+        {
+          service: {
+            name: 'opbeans-node',
+            metrics: {
+              cpu: 0.1,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+        {
+          service: {
+            environment: 'prod',
+            metrics: {
+              memory: 0.5,
+            },
+          },
+          properties: {
+            foo: 'bar',
+          },
+        },
+      ],
+      'properties.foo'
+    );
+
+    expect(joined).toEqual([
+      {
+        service: {
+          name: 'opbeans-node',
+          environment: 'prod',
+          metrics: {
+            cpu: 0.1,
+            memory: 0.5,
+          },
+        },
+        properties: {
+          foo: 'bar',
+        },
+      },
+    ]);
+  });
 });

x-pack/packages/observability/observability_utils/array/join_by_key.ts

@@ -18,18 +18,29 @@ export type JoinedReturnType<
   }
 >;
 
-type ArrayOrSingle<T> = T | T[];
+function getValueByPath(obj: any, path: string): any {
+  return path.split('.').reduce((acc, keyPart) => {
+    // Check if acc is a valid object and has the key
+    return acc && acc[keyPart] !== undefined ? acc[keyPart] : undefined;
+  }, obj);
+}
 
+type NestedKeys<T> = T extends object
+  ? { [K in keyof T]: K extends string ? `${K}` | `${K}.${NestedKeys<T[K]>}` : never }[keyof T]
+  : never;
+
+type ArrayOrSingle<T> = T | T[];
+type CombinedNestedKeys<T, U> = (NestedKeys<T> & NestedKeys<U>) | (keyof T & keyof U);
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>
 >(items: T[], key: V): JoinedReturnType<T, U>;
 
 export function joinByKey<
   T extends Record<string, any>,
   U extends UnionToIntersection<T>,
-  V extends ArrayOrSingle<keyof T & keyof U>,
+  V extends ArrayOrSingle<CombinedNestedKeys<T, U>>,
   W extends JoinedReturnType<T, U>,
   X extends (a: T, b: T) => ValuesType<W>
 >(items: T[], key: V, mergeFn: X): W;
@@ -45,7 +56,7 @@ export function joinByKey(
   items.forEach((current) => {
     // The key of the map is a stable JSON string of the values from given keys.
     // We need stable JSON string to support plain object values.
-    const stableKey = stableStringify(keys.map((k) => current[k]));
+    const stableKey = stableStringify(keys.map((k) => current[k] ?? getValueByPath(current, k)));
 
     if (map.has(stableKey)) {
       const item = map.get(stableKey);

x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts

@@ -9,6 +9,7 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import { withSpan } from '@kbn/apm-utils';
 import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types';
+import { esqlResultToPlainObjects } from '../utils/esql_result_to_plain_objects';
 
 type SearchRequest = ESSearchRequest & {
   index: string | string[];
@@ -25,7 +26,10 @@ export interface ObservabilityElasticsearchClient {
     operationName: string,
     parameters: TSearchRequest
   ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
-  esql(operationName: string, parameters: EsqlQueryRequest): Promise<ESQLSearchResponse>;
+  esql<TDocument = unknown>(
+    operationName: string,
+    parameters: EsqlQueryRequest
+  ): Promise<TDocument[]>;
   client: ElasticsearchClient;
 }
 
@@ -40,7 +44,7 @@ export function createObservabilityEsClient({
 }): ObservabilityElasticsearchClient {
   return {
     client,
-    esql(operationName: string, parameters: EsqlQueryRequest) {
+    esql<TDocument = unknown>(operationName: string, parameters: EsqlQueryRequest) {
       logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`);
       return withSpan({ name: operationName, labels: { plugin } }, () => {
         return client.esql.query(
@@ -54,7 +58,7 @@ export function createObservabilityEsClient({
       })
         .then((response) => {
           logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`);
-          return response as unknown as ESQLSearchResponse;
+          return esqlResultToPlainObjects<TDocument>(response as unknown as ESQLSearchResponse);
         })
         .catch((error) => {
           throw error;

x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts

@@ -6,25 +6,28 @@
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';
+import { unflattenObject } from '../../object/unflatten_object';
 
-export function esqlResultToPlainObjects<T extends Record<string, any>>(
+export function esqlResultToPlainObjects<TDocument = unknown>(
   result: ESQLSearchResponse
-): T[] {
+): TDocument[] {
   return result.values.map((row) => {
-    return row.reduce<Record<string, unknown>>((acc, value, index) => {
-      const column = result.columns[index];
+    return unflattenObject(
+      row.reduce<Record<string, any>>((acc, value, index) => {
+        const column = result.columns[index];
 
-      if (!column) {
-        return acc;
-      }
+        if (!column) {
+          return acc;
+        }
 
-      // Removes the type suffix from the column name
-      const name = column.name.replace(/\.(text|keyword)$/, '');
-      if (!acc[name]) {
-        acc[name] = value;
-      }
+        // Removes the type suffix from the column name
+        const name = column.name.replace(/\.(text|keyword)$/, '');
+        if (!acc[name]) {
+          acc[name] = value;
+        }
 
-      return acc;
-    }, {});
-  }) as T[];
+        return acc;
+      }, {})
+    ) as TDocument;
+  });
 }

x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/transaction_details/transaction_details.cy.ts

@@ -115,7 +115,10 @@ describe('Transaction details', () => {
     );
 
     cy.contains('Top 5 errors', { timeout: 30000 });
-    cy.getByTestSubj('topErrorsForTransactionTable').contains('a', '[MockError] Foo').click();
+    cy.getByTestSubj('topErrorsForTransactionTable')
+      .should('be.visible')
+      .contains('a', '[MockError] Foo', { timeout: 10000 })
+      .click();
     cy.url().should('include', 'opbeans-java/errors');
   });
 

x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts

@@ -74,7 +74,7 @@ describe('getDataStreamTypes', () => {
   it('should return metrics and entity source_data_stream types when entityCentriExperienceEnabled is true and has entity data', async () => {
     (getHasMetricsData as jest.Mock).mockResolvedValue(true);
     (getLatestEntity as jest.Mock).mockResolvedValue({
-      'source_data_stream.type': ['logs', 'metrics'],
+      sourceDataStreamType: ['logs', 'metrics'],
     });
 
     const params = {
@@ -118,7 +118,7 @@ describe('getDataStreamTypes', () => {
   it('should return entity source_data_stream types when has no metrics', async () => {
     (getHasMetricsData as jest.Mock).mockResolvedValue(false);
     (getLatestEntity as jest.Mock).mockResolvedValue({
-      'source_data_stream.type': ['logs', 'traces'],
+      sourceDataStreamType: ['logs', 'traces'],
     });
 
     const params = {

x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts

@@ -7,11 +7,9 @@
 
 import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
 import { findInventoryFields } from '@kbn/metrics-data-access-plugin/common';
-import {
-  EntityDataStreamType,
-  SOURCE_DATA_STREAM_TYPE,
-} from '@kbn/observability-shared-plugin/common';
+import { EntityDataStreamType } from '@kbn/observability-shared-plugin/common';
 import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
+import { castArray } from 'lodash';
 import { type InfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
 import { getHasMetricsData } from './get_has_metrics_data';
 import { getLatestEntity } from './get_latest_entity';
@@ -45,15 +43,15 @@ export async function getDataStreamTypes({
     return Array.from(sourceDataStreams);
   }
 
-  const entity = await getLatestEntity({
+  const latestEntity = await getLatestEntity({
     inventoryEsClient: obsEsClient,
     entityId,
     entityType,
     entityManagerClient,
   });
 
-  if (entity?.[SOURCE_DATA_STREAM_TYPE]) {
-    [entity[SOURCE_DATA_STREAM_TYPE]].flat().forEach((item) => {
+  if (latestEntity) {
+    castArray(latestEntity.sourceDataStreamType).forEach((item) => {
       sourceDataStreams.add(item as EntityDataStreamType);
     });
   }

x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts

@@ -7,20 +7,16 @@
 
 import { ENTITY_LATEST, entitiesAliasPattern } from '@kbn/entities-schema';
 import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
-import {
-  ENTITY_TYPE,
-  SOURCE_DATA_STREAM_TYPE,
-} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
+import { ENTITY_TYPE, SOURCE_DATA_STREAM_TYPE } from '@kbn/observability-shared-plugin/common';
 import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client';
-import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects';
 
 const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({
   type: '*',
   dataset: ENTITY_LATEST,
 });
 
-interface Entity {
-  [SOURCE_DATA_STREAM_TYPE]: string | string[];
+interface LatestEntityResponse {
+  sourceDataStreamType?: string | string[];
 }
 
 export async function getLatestEntity({
@@ -33,18 +29,20 @@ export async function getLatestEntity({
   entityType: 'host' | 'container';
   entityId: string;
   entityManagerClient: EntityClient;
-}): Promise<Entity | undefined> {
+}): Promise<LatestEntityResponse | undefined> {
   const { definitions } = await entityManagerClient.getEntityDefinitions({
     builtIn: true,
     type: entityType,
   });
 
   const hostOrContainerIdentityField = definitions[0]?.identityFields?.[0]?.field;
   if (hostOrContainerIdentityField === undefined) {
-    return { [SOURCE_DATA_STREAM_TYPE]: [] };
+    return undefined;
   }
 
-  const latestEntitiesEsqlResponse = await inventoryEsClient.esql('get_latest_entities', {
+  const response = await inventoryEsClient.esql<{
+    source_data_stream?: { type?: string | string[] };
+  }>('get_latest_entities', {
     query: `FROM ${ENTITIES_LATEST_ALIAS}
         | WHERE ${ENTITY_TYPE} == ?
         | WHERE ${hostOrContainerIdentityField} == ?
@@ -53,5 +51,5 @@ export async function getLatestEntity({
     params: [entityType, entityId],
   });
 
-  return esqlResultToPlainObjects<Entity>(latestEntitiesEsqlResponse)[0];
+  return { sourceDataStreamType: response[0].source_data_stream?.type };
 }

x-pack/plugins/observability_solution/inventory/.storybook/get_mock_inventory_context.tsx

@@ -24,7 +24,14 @@ export function getMockInventoryContext(): InventoryKibanaContext {
 
   return {
     ...coreStart,
-    entityManager: {} as unknown as EntityManagerPublicPluginStart,
+    entityManager: {
+      entityClient: {
+        asKqlFilter: jest.fn(),
+        getIdentityFieldsValue() {
+          return 'entity_id';
+        },
+      },
+    } as unknown as EntityManagerPublicPluginStart,
     observabilityShared: {} as unknown as ObservabilitySharedPluginStart,
     inference: {} as unknown as InferencePublicStart,
     share: {