[8.x] [APM][Otel] Add Otel client based on PoC data (#192293)

packages/kbn-apm-synthtrace-client/index.ts

@@ -37,3 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator }
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
 export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';
 export { type EntityFields, entities } from './src/lib/entities';
+export { otel, type OtelDocument } from './src/lib/otel';

packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { OtelDocument } from '../../..';
+import { Serializable } from '../serializable';
+
+export interface OtelErrorDocument extends OtelDocument {
+  'event.name'?: string;
+  attributes?: {
+    'exception.message'?: string;
+    'error.stack_trace'?: string;
+    'exception.handled'?: boolean;
+    'exception.type'?: string;
+    'processor.event'?: string;
+    'timestamp.us'?: number;
+    'event.name'?: string;
+    'error.id'?: string;
+  };
+}
+
+export class OtelError extends Serializable<OtelErrorDocument> {
+  constructor(fields: OtelErrorDocument) {
+    super({
+      ...fields,
+    });
+  }
+}

packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts

@@ -0,0 +1,213 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Fields } from '../entity';
+import { Serializable } from '../serializable';
+import { OtelError } from './error';
+import { OtelMetric } from './metric';
+import { OtelTransaction } from './transaction';
+
+interface OtelSharedResourceAttributes {
+  'service.name'?: string;
+  'agent.name'?: string;
+  'agent.version'?: string;
+  'metricset.interval'?: string;
+  'service.instance.id'?: string;
+  'telemetry.sdk.language'?: string;
+  'telemetry.sdk.name'?: string;
+  'telemetry.sdk.version'?: string;
+  'some.resource.attribute'?: string;
+}
+
+export interface OtelDocument extends Fields {
+  data_stream?: {
+    dataset: string;
+    namespace: string;
+    type: string;
+  };
+  attributes?: {
+    'timestamp.us'?: number;
+    'metricset.name'?: string;
+    [key: string]: any;
+  };
+  resource?: {
+    attributes?: OtelSharedResourceAttributes;
+    dropped_attributes_count?: number;
+    schema_url?: string;
+  };
+  scope?: {
+    attributes?: {
+      'service.framework.name'?: string;
+      'service.framework.version'?: string;
+    };
+    dropped_attributes_count?: number;
+    name?: string;
+  };
+  name?: string;
+  trace_id?: string;
+  trace?: { id: string };
+  span_id?: string;
+  span?: { id: string };
+  dropped_attributes_count?: number;
+  dropped_events_count?: number;
+  dropped_links_count?: number;
+  timestamp_us?: number;
+}
+
+class Otel extends Serializable<OtelDocument> {
+  constructor(fields: OtelDocument) {
+    super({
+      ...fields,
+    });
+  }
+
+  error(spanId: string) {
+    return new OtelError({
+      ...this.fields,
+      attributes: {
+        'exception.message': 'boom',
+        'exception.handled': false,
+        'exception.type': '*errors.errorString',
+        'error.stack_trace': 'Error: INTERNAL: Boom',
+        'processor.event': 'error',
+        'timestamp.us': 1726580752010657,
+        'event.name': 'exception',
+        'error.id': `error-${spanId}`,
+      },
+      data_stream: {
+        dataset: 'generic.otel',
+        namespace: 'default',
+        type: 'logs',
+      },
+      'event.name': 'exception',
+      dropped_attributes_count: 0,
+      resource: {
+        attributes: {
+          'agent.name': 'opentelemetry/go',
+          'agent.version': '1.28.0',
+          'service.name': 'sendotlp-synth',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+        },
+        dropped_attributes_count: 0,
+        schema_url: 'https://opentelemetry.io/schemas/1.26.0',
+      },
+      scope: {
+        attributes: {
+          'service.framework.name': 'sendotlp-synth',
+          'service.framework.version': '',
+        },
+        dropped_attributes_count: 0,
+        name: 'sendotlp-synth',
+      },
+      span_id: spanId,
+    });
+  }
+
+  metric() {
+    return new OtelMetric({
+      ...this.fields,
+      attributes: {
+        'metricset.name': 'service_destination',
+        'processor.event': 'metric',
+        'event.outcome': 'success',
+        'service.target.name': 'foo_service',
+        'service.target.type': 'http',
+        'span.name': 'child1',
+        'span.destination.service.resource': 'foo_service:8080',
+      },
+      data_stream: {
+        dataset: 'service_destination.10m.otel',
+        namespace: 'default',
+        type: 'metrics',
+      },
+      metrics: {
+        service_summary: 2,
+      },
+      resource: {
+        attributes: {
+          'agent.name': 'otlp',
+          'agent.version': '1.28.0',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+          'service.name': 'sendotlp-synth',
+          'metricset.interval': '10m',
+        },
+        dropped_attributes_count: 0,
+      },
+      scope: {
+        dropped_attributes_count: 0,
+        name: 'github.com/elastic/opentelemetry-collector-components/connector/spanmetricsconnectorv2',
+      },
+    });
+  }
+
+  // In Otel we have only spans (https://opentelemetry.io/docs/concepts/signals/traces/#spans)
+  // we call the root span a transaction to match our data model
+  transaction(id: string) {
+    return new OtelTransaction({
+      ...this.fields,
+      attributes: {
+        'event.outcome': 'success',
+        'event.success_count': 1,
+        'processor.event': 'transaction',
+        'timestamp.us': 1726580752010657,
+        'transaction.duration.us': 15202,
+        'transaction.id': id,
+        'transaction.name': 'parent-synth',
+        'transaction.representative_count': 1,
+        'transaction.result': 'HTTP 2xx',
+        'transaction.root': true,
+        'transaction.sampled': true,
+        'transaction.type': 'unknown',
+      },
+      data_stream: {
+        dataset: 'generic.otel',
+        namespace: 'default',
+        type: 'traces',
+      },
+      duration: 11742370,
+      kind: 'Internal',
+      name: 'parent-synth',
+      resource: {
+        attributes: {
+          'agent.name': 'otlp',
+          'agent.version': '1.28.0',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+          'service.name': 'sendotlp-synth',
+        },
+        dropped_attributes_count: 0,
+        schema_url: 'https://opentelemetry.io/schemas/1.26.0',
+      },
+      scope: {
+        attributes: {
+          'service.framework.name': 'sendotlp-synth',
+          'service.framework.version': '',
+        },
+        dropped_attributes_count: 0,
+        name: 'sendotlp-synth',
+      },
+      span_id: id,
+      status: {
+        code: 'Unset',
+      },
+    });
+  }
+}
+
+export function create(id: string): Otel {
+  return new Otel({
+    trace_id: id,
+    dropped_attributes_count: 0,
+    dropped_events_count: 0,
+    dropped_links_count: 0,
+  });
+}
+
+export const otel = {
+  create,
+};

packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OtelDocument } from '.';
+import { Serializable } from '../serializable';
+
+export interface OtelMetricDocument extends OtelDocument {
+  attributes?: {
+    'metricset.name'?: string;
+    'processor.event'?: string;
+    'event.outcome'?: string;
+    'service.target.name'?: string;
+    'service.target.type'?: string;
+    'span.name'?: string;
+    'span.destination.service.resource'?: string;
+  };
+  metrics?: {
+    service_summary?: number;
+  };
+}
+export class OtelMetric extends Serializable<OtelMetricDocument> {
+  constructor(fields: OtelMetricDocument) {
+    super({
+      ...fields,
+    });
+  }
+}

packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OtelDocument } from '.';
+import { Serializable } from '../serializable';
+
+export interface OtelTransactionDocument extends OtelDocument {
+  attributes?: {
+    'event.outcome'?: string;
+    'event.success_count'?: number;
+    'processor.event'?: string;
+    'timestamp.us'?: number;
+    'transaction.duration.us'?: number;
+    'transaction.id'?: string;
+    'transaction.name'?: string;
+    'transaction.representative_count'?: number;
+    'transaction.result'?: string;
+    'transaction.root'?: boolean;
+    'transaction.sampled'?: boolean;
+    'transaction.type'?: string;
+  };
+  status?: {
+    code?: string;
+  };
+  dropped_events_count?: number;
+  dropped_links_count?: number;
+  duration?: number;
+  kind?: string;
+  name?: string;
+}
+
+export class OtelTransaction extends Serializable<OtelTransactionDocument> {
+  constructor(fields: OtelTransactionDocument) {
+    super({
+      ...fields,
+    });
+  }
+}

packages/kbn-apm-synthtrace/index.ts

@@ -18,6 +18,7 @@ export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client
 export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
 export { EntitiesSynthtraceKibanaClient } from './src/lib/entities/entities_synthtrace_kibana_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
+export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client';
 export {
   addObserverVersionTransform,
   deleteSummaryFieldTransform,

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -13,6 +13,7 @@ import {
   InfraSynthtraceEsClient,
   LogsSynthtraceEsClient,
   SyntheticsSynthtraceEsClient,
+  OtelSynthtraceEsClient,
   EntitiesSynthtraceEsClient,
 } from '../..';
 import { Logger } from '../lib/utils/create_logger';
@@ -25,6 +26,7 @@ interface EsClients {
   logsEsClient: LogsSynthtraceEsClient;
   infraEsClient: InfraSynthtraceEsClient;
   syntheticsEsClient: SyntheticsSynthtraceEsClient;
+  otelEsClient: OtelSynthtraceEsClient;
   entitiesEsClient: EntitiesSynthtraceEsClient;
 }
 

packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -15,6 +15,7 @@ import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
 import { getSyntheticsEsClient } from './get_synthetics_es_client';
+import { getOtelSynthtraceEsClient } from './get_otel_es_client';
 import { getEntitiesEsClient } from './get_entities_es_client';
 import { getEntitiesKibanaClient } from './get_entites_kibana_client';
 
@@ -74,13 +75,19 @@ export async function bootstrap(runOptions: RunOptions) {
     logger,
     concurrency: runOptions.concurrency,
   });
+  const otelEsClient = getOtelSynthtraceEsClient({
+    target: esUrl,
+    logger,
+    concurrency: runOptions.concurrency,
+  });
 
   if (runOptions.clean) {
     await apmEsClient.clean();
     await logsEsClient.clean();
     await infraEsClient.clean();
     await entitiesEsClient.clean();
     await syntheticsEsClient.clean();
+    await otelEsClient.clean();
   }
 
   return {
@@ -90,6 +97,7 @@ export async function bootstrap(runOptions: RunOptions) {
     infraEsClient,
     entitiesEsClient,
     syntheticsEsClient,
+    otelEsClient,
     version,
     kibanaUrl,
     esUrl,