[8.19] Migrate from individual OTel packages to EDOT (#226609)

package.json

@@ -131,6 +131,7 @@
     "@elastic/monaco-esql": "^3.1.4",
     "@elastic/node-crypto": "^1.2.3",
     "@elastic/numeral": "^2.5.1",
+    "@elastic/opentelemetry-node": "^1.1.1",
     "@elastic/react-search-ui": "^1.20.2",
     "@elastic/react-search-ui-views": "^1.20.2",
     "@elastic/request-converter": "^8.17.0",
@@ -152,7 +153,7 @@
     "@formatjs/intl-utils": "^3.8.4",
     "@formatjs/ts-transformer": "^3.13.14",
     "@google/generative-ai": "^0.21.0",
-    "@grpc/grpc-js": "^1.8.22",
+    "@grpc/grpc-js": "^1.13.4",
     "@hapi/accept": "^6.0.3",
     "@hapi/boom": "^10.0.1",
     "@hapi/cookie": "^12.0.1",
@@ -1083,23 +1084,14 @@
     "@openfeature/server-sdk": "^1.18.0",
     "@openfeature/web-sdk": "^1.5.0",
     "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/api-metrics": "^0.31.0",
-    "@opentelemetry/context-async-hooks": "^2.0.0",
-    "@opentelemetry/core": "^2.0.0",
-    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.34.0",
-    "@opentelemetry/exporter-prometheus": "^0.31.0",
-    "@opentelemetry/exporter-trace-otlp-grpc": "^0.200.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.200.0",
-    "@opentelemetry/exporter-trace-otlp-proto": "^0.200.0",
-    "@opentelemetry/instrumentation": "^0.200.0",
-    "@opentelemetry/instrumentation-http": "^0.200.0",
-    "@opentelemetry/instrumentation-undici": "^0.11.0",
-    "@opentelemetry/otlp-exporter-base": "^0.200.0",
-    "@opentelemetry/resources": "^2.0.0",
-    "@opentelemetry/sdk-metrics-base": "^0.31.0",
-    "@opentelemetry/sdk-node": "^0.200.0",
-    "@opentelemetry/sdk-trace-base": "^2.0.0",
-    "@opentelemetry/sdk-trace-node": "^2.0.0",
+    "@opentelemetry/context-async-hooks": "^2.0.1",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.202.0",
+    "@opentelemetry/exporter-prometheus": "^0.202.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.202.0",
+    "@opentelemetry/instrumentation": "^0.202.0",
+    "@opentelemetry/instrumentation-undici": "^0.13.2",
+    "@opentelemetry/otlp-exporter-base": "^0.202.0",
     "@opentelemetry/semantic-conventions": "^1.34.0",
     "@paralleldrive/cuid2": "^2.2.2",
     "@reduxjs/toolkit": "1.9.7",

renovate.json

@@ -178,7 +178,7 @@
         "@aws-crypto/sha256-js",
         "@aws-crypto/util",
         "@langtrase/trace-attributes",
-        "@opentelemetry/sdk-trace-base",
+        "@arizeai/openinference-semantic-conventions",
         "@smithy/eventstream-serde-node",
         "@smithy/node-http-handler",
         "@smithy/types",
@@ -3370,7 +3370,7 @@
       ],
       "minimumReleaseAge": "7 days",
       "enabled": true
-    },    
+    },
     {
       "groupName": "redux-actions",
       "matchDepNames": [
@@ -4346,36 +4346,33 @@
       "groupName": "OpenTelemetry modules",
       "matchDepNames": [
         "@grpc/grpc-js",
+        "@elastic/opentelemetry-node",
         "@opentelemetry/api",
-        "@opentelemetry/api-metrics",
-        "@opentelemetry/core",
+        "@opentelemetry/context-async-hooks",
         "@opentelemetry/exporter-metrics-otlp-grpc",
         "@opentelemetry/exporter-prometheus",
-        "@opentelemetry/resources",
-        "@opentelemetry/sdk-metrics-base",
-        "@opentelemetry/semantic-conventions",
-        "@arizeai/openinference-semantic-conventions",
-        "@opentelemetry/context-async-hooks",
-        "@opentelemetry/exporter-trace-otlp-grpc",
         "@opentelemetry/exporter-trace-otlp-http",
         "@opentelemetry/exporter-trace-otlp-proto",
-        "@opentelemetry/otlp-exporter-base",
-        "@opentelemetry/sdk-node",
-        "@opentelemetry/sdk-trace-node",
         "@opentelemetry/instrumentation",
-        "@opentelemetry/instrumentation-http",
-        "@opentelemetry/instrumentation-undici"
+        "@opentelemetry/instrumentation-undici",
+        "@opentelemetry/otlp-exporter-base",
+        "@opentelemetry/semantic-conventions"
       ],
       "reviewers": [
-        "team:stack-monitoring",
-        "team:kibana-core"
+        "team:appex-ai-infra",
+        "team:kibana-core",
+        "team:kibana-security",
+        "team:stack-monitoring"
       ],
       "matchBaseBranches": [
         "main"
       ],
       "labels": [
+        "Team:AI Infra",
         "Team:Monitoring",
-        "backport:all-open",
+        "Team:Core",
+        "Team:Security",
+        "backport:prev-minor",
         "release_note:skip"
       ],
       "minimumReleaseAge": "7 days",

src/platform/packages/private/kbn-import-resolver/src/import_resolver.ts

@@ -142,6 +142,17 @@ export class ImportResolver {
       return this.adaptReq('src/core/server', dirname);
     }
 
+    if (req.startsWith('@modelcontextprotocol/sdk')) {
+      const relPath = req.split('@modelcontextprotocol/sdk')[1];
+      return Path.resolve(REPO_ROOT, `node_modules/@modelcontextprotocol/sdk/dist/esm/${relPath}`);
+    }
+
+    // We need this "hack" because our current import-resolver doesn't support "exports" in package.json.
+    // We should be able to remove this once we support cjs/esm interop.
+    if (req.startsWith('@elastic/opentelemetry-node/sdk')) {
+      return Path.resolve(REPO_ROOT, `node_modules/@elastic/opentelemetry-node/lib/sdk.js`);
+    }
+
     // turn root-relative paths into relative paths
     if (
       req.startsWith('src/') ||

src/platform/packages/shared/kbn-tracing/src/init_tracing.ts

@@ -6,25 +6,14 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+import { core, node, resources, tracing } from '@elastic/opentelemetry-node/sdk';
 import { LangfuseSpanProcessor, PhoenixSpanProcessor } from '@kbn/inference-tracing';
 import { fromExternalVariant } from '@kbn/std';
 import { TracingConfig } from '@kbn/tracing-config';
 import { context, propagation, trace } from '@opentelemetry/api';
 import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
-import {
-  CompositePropagator,
-  W3CBaggagePropagator,
-  W3CTraceContextPropagator,
-} from '@opentelemetry/core';
-import {
-  NodeTracerProvider,
-  ParentBasedSampler,
-  SpanProcessor,
-  TraceIdRatioBasedSampler,
-} from '@opentelemetry/sdk-trace-node';
 import type { AgentConfigOptions } from 'elastic-apm-node';
 import { castArray, once } from 'lodash';
-import { resourceFromAttributes } from '@opentelemetry/resources';
 import { ATTR_SERVICE_INSTANCE_ID, ATTR_SERVICE_NAMESPACE } from '@kbn/opentelemetry-attributes';
 import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
 import { LateBindingSpanProcessor } from '..';
@@ -40,7 +29,7 @@ export function initTracing({
   context.setGlobalContextManager(contextManager);
   contextManager.enable();
 
-  const resource = resourceFromAttributes({
+  const resource = resources.resourceFromAttributes({
     [ATTR_SERVICE_NAME]: apmConfig?.serviceName,
     [ATTR_SERVICE_INSTANCE_ID]: apmConfig?.serviceNodeName,
     [ATTR_SERVICE_NAMESPACE]: apmConfig?.environment,
@@ -49,20 +38,20 @@ export function initTracing({
   // this is used for late-binding of span processors
   const lateBindingProcessor = LateBindingSpanProcessor.get();
 
-  const allSpanProcessors: SpanProcessor[] = [lateBindingProcessor];
+  const allSpanProcessors: tracing.SpanProcessor[] = [lateBindingProcessor];
 
   propagation.setGlobalPropagator(
-    new CompositePropagator({
-      propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
+    new core.CompositePropagator({
+      propagators: [new core.W3CTraceContextPropagator(), new core.W3CBaggagePropagator()],
     })
   );
 
-  const traceIdSampler = new TraceIdRatioBasedSampler(tracingConfig?.sample_rate ?? 1);
+  const traceIdSampler = new tracing.TraceIdRatioBasedSampler(tracingConfig?.sample_rate ?? 1);
 
-  const nodeTracerProvider = new NodeTracerProvider({
+  const nodeTracerProvider = new node.NodeTracerProvider({
     // by default, base sampling on parent context,
     // or for root spans, based on the configured sample rate
-    sampler: new ParentBasedSampler({
+    sampler: new tracing.ParentBasedSampler({
       root: traceIdSampler,
     }),
     spanProcessors: allSpanProcessors,
@@ -85,8 +74,8 @@ export function initTracing({
   trace.setGlobalTracerProvider(nodeTracerProvider);
 
   propagation.setGlobalPropagator(
-    new CompositePropagator({
-      propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
+    new core.CompositePropagator({
+      propagators: [new core.W3CTraceContextPropagator(), new core.W3CBaggagePropagator()],
     })
   );
 

src/platform/packages/shared/kbn-tracing/src/late_binding_span_processor.ts

@@ -8,8 +8,8 @@
  */
 
 import { Context } from '@opentelemetry/api';
-import { ReadableSpan, Span, SpanProcessor } from '@opentelemetry/sdk-trace-node';
 import { pull } from 'lodash';
+import { tracing } from '@elastic/opentelemetry-node/sdk';
 
 const noop = async () => {};
 
@@ -18,18 +18,18 @@ const noop = async () => {};
  * which is useful if processors should be conditionally applied based on config
  * or runtime logic.
  */
-export class LateBindingSpanProcessor implements SpanProcessor {
+export class LateBindingSpanProcessor implements tracing.SpanProcessor {
   static #instance?: LateBindingSpanProcessor;
 
-  #processors: SpanProcessor[] = [];
+  #processors: tracing.SpanProcessor[] = [];
 
   private constructor() {}
 
-  onStart(span: Span, parentContext: Context): void {
+  onStart(span: tracing.Span, parentContext: Context): void {
     this.#processors.forEach((processor) => processor.onStart(span, parentContext));
   }
 
-  onEnd(span: ReadableSpan): void {
+  onEnd(span: tracing.ReadableSpan): void {
     this.#processors.forEach((processor) => processor.onEnd(span));
   }
 
@@ -40,7 +40,7 @@ export class LateBindingSpanProcessor implements SpanProcessor {
     await Promise.all(this.#processors.map((processor) => processor.shutdown()));
   }
 
-  register(processor: SpanProcessor) {
+  register(processor: tracing.SpanProcessor) {
     this.#processors.push(processor);
 
     return async () => {
@@ -49,7 +49,7 @@ export class LateBindingSpanProcessor implements SpanProcessor {
     };
   }
 
-  static register(processor: SpanProcessor): () => Promise<void> {
+  static register(processor: tracing.SpanProcessor): () => Promise<void> {
     return this.#instance?.register(processor) ?? noop;
   }
 

src/platform/test/common/plugins/otel_metrics/server/monitoring/metrics.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Counter, Meter } from '@opentelemetry/api-metrics';
+import { Counter, Meter } from '@opentelemetry/api';
 
 export class Metrics {
   requestCounter: Counter;

src/platform/test/common/plugins/otel_metrics/server/plugin.ts

@@ -8,7 +8,7 @@
  */
 
 import { CoreSetup, Plugin } from '@kbn/core/server';
-import { metrics } from '@opentelemetry/api-metrics';
+import { metrics } from '@opentelemetry/api';
 import { generateOtelMetrics } from './routes';
 import { Metrics } from './monitoring/metrics';
 

tsconfig.base.json

@@ -2196,7 +2196,10 @@
       "@kbn/zod-helpers/*": ["src/platform/packages/shared/kbn-zod-helpers/*"],
       // END AUTOMATED PACKAGE LISTING
       // Allows for importing from `kibana` package for the exported types.
-      "@emotion/core": ["typings/@emotion"]
+      "@emotion/core": ["typings/@emotion"],
+      // We need the custom typings "proxy" because our current import-resolver doesn't support "exports" in package.json.
+      // We should be able to remove this once we support cjs/esm interop.
+      "@elastic/opentelemetry-node/sdk": ["typings/@elastic/opentelemetry-node/sdk"],
     },
     // Support .tsx files and transform JSX into calls to React.createElement
     "jsx": "react",

typings/@elastic/opentelemetry-node/sdk.d.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+declare module '@elastic/opentelemetry-node/sdk' {
+  export * from '@elastic/opentelemetry-node/types/sdk';
+}

x-pack/platform/packages/shared/kbn-inference-tracing/src/base_inference_span_processor.ts

@@ -5,29 +5,23 @@
  * 2.0.
  */
 
-import { Context } from '@opentelemetry/api';
+import { api, tracing } from '@elastic/opentelemetry-node/sdk';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
-import {
-  ReadableSpan,
-  SpanProcessor,
-  Span,
-  BatchSpanProcessor,
-} from '@opentelemetry/sdk-trace-node';
 import { isInInferenceContext } from './is_in_inference_context';
 import { IS_ROOT_INFERENCE_SPAN_ATTRIBUTE_NAME } from './root_inference_span';
 
-export abstract class BaseInferenceSpanProcessor implements SpanProcessor {
-  private delegate: SpanProcessor;
+export abstract class BaseInferenceSpanProcessor implements tracing.SpanProcessor {
+  private delegate: tracing.SpanProcessor;
 
   constructor(exporter: OTLPTraceExporter, scheduledDelayMillis: number) {
-    this.delegate = new BatchSpanProcessor(exporter, {
+    this.delegate = new tracing.BatchSpanProcessor(exporter, {
       scheduledDelayMillis,
     });
   }
 
-  abstract processInferenceSpan(span: ReadableSpan): ReadableSpan;
+  abstract processInferenceSpan(span: tracing.ReadableSpan): tracing.ReadableSpan;
 
-  onStart(span: Span, parentContext: Context): void {
+  onStart(span: tracing.Span, parentContext: api.Context): void {
     const shouldTrack =
       (isInInferenceContext(parentContext) || span.instrumentationScope.name === 'inference') &&
       span.instrumentationScope.name !== '@elastic/transport';
@@ -38,7 +32,7 @@ export abstract class BaseInferenceSpanProcessor implements SpanProcessor {
     }
   }
 
-  onEnd(span: ReadableSpan): void {
+  onEnd(span: tracing.ReadableSpan): void {
     if (span.attributes._should_track) {
       delete span.attributes._should_track;
 

x-pack/platform/packages/shared/kbn-inference-tracing/src/create_inference_active_span.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { isTracingSuppressed } from '@opentelemetry/core';
+import { core } from '@elastic/opentelemetry-node/sdk';
 import { Span, context, propagation, trace } from '@opentelemetry/api';
 import { BAGGAGE_TRACKING_BEACON_KEY, BAGGAGE_TRACKING_BEACON_VALUE } from './baggage';
 import { InferenceSpanAttributes } from './with_inference_span';
@@ -49,7 +49,7 @@ export function createActiveInferenceSpan<T>(
 
   let parentContext = context.active();
 
-  if (isTracingSuppressed(parentContext)) {
+  if (core.isTracingSuppressed(parentContext)) {
     return cb();
   }
 

x-pack/platform/packages/shared/kbn-inference-tracing/src/langfuse/langfuse_span_processor.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
+import { tracing } from '@elastic/opentelemetry-node/sdk';
 import { InferenceTracingLangfuseExportConfig } from '@kbn/inference-tracing-config';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
-import { ReadableSpan } from '@opentelemetry/sdk-trace-node';
 import { memoize, omit, partition } from 'lodash';
 import { diag } from '@opentelemetry/api';
 import { BaseInferenceSpanProcessor } from '../base_inference_span_processor';
@@ -48,7 +48,7 @@ export class LangfuseSpanProcessor extends BaseInferenceSpanProcessor {
     };
   }
 
-  override processInferenceSpan(span: ReadableSpan): ReadableSpan {
+  override processInferenceSpan(span: tracing.ReadableSpan): tracing.ReadableSpan {
     // Langfuse doesn't understand fully semconv-compliant span events
     // yet, so we translate to a format it does understand. see
     // https://github.com/langfuse/langfuse/blob/c1c22a9b9b684bd45ca9436556c2599d5a23271d/web/src/features/otel/server/index.ts#L476