[8.18] Migrate from individual OTel packages to EDOT (#226609)

package.json

@@ -126,6 +126,7 @@
     "@elastic/filesaver": "1.1.2",
     "@elastic/node-crypto": "^1.2.3",
     "@elastic/numeral": "^2.5.1",
+    "@elastic/opentelemetry-node": "^1.1.1",
     "@elastic/react-search-ui": "^1.20.2",
     "@elastic/react-search-ui-views": "^1.20.2",
     "@elastic/request-converter": "^8.18.1",
@@ -147,7 +148,7 @@
     "@formatjs/intl-utils": "^3.8.4",
     "@formatjs/ts-transformer": "^3.13.14",
     "@google/generative-ai": "^0.21.0",
-    "@grpc/grpc-js": "^1.8.22",
+    "@grpc/grpc-js": "^1.13.4",
     "@hapi/accept": "^6.0.3",
     "@hapi/boom": "^10.0.1",
     "@hapi/cookie": "^12.0.1",
@@ -1043,16 +1044,18 @@
     "@mapbox/vector-tile": "1.3.1",
     "@openfeature/core": "^1.8.1",
     "@openfeature/launchdarkly-client-provider": "^0.3.2",
-    "@openfeature/server-sdk": "^1.16.1",
-    "@openfeature/web-sdk": "^1.3.1",
-    "@opentelemetry/api": "^1.1.0",
-    "@opentelemetry/api-metrics": "^0.31.0",
-    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.34.0",
-    "@opentelemetry/exporter-prometheus": "^0.31.0",
-    "@opentelemetry/resources": "^1.4.0",
-    "@opentelemetry/sdk-metrics-base": "^0.31.0",
-    "@opentelemetry/sdk-trace-base": "^1.24.0",
-    "@opentelemetry/semantic-conventions": "^1.4.0",
+    "@openfeature/server-sdk": "^1.18.0",
+    "@openfeature/web-sdk": "^1.5.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/context-async-hooks": "^2.0.1",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.202.0",
+    "@opentelemetry/exporter-prometheus": "^0.202.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.202.0",
+    "@opentelemetry/instrumentation": "^0.202.0",
+    "@opentelemetry/instrumentation-undici": "^0.13.2",
+    "@opentelemetry/otlp-exporter-base": "^0.202.0",
+    "@opentelemetry/semantic-conventions": "^1.34.0",
     "@paralleldrive/cuid2": "^2.2.2",
     "@reduxjs/toolkit": "1.9.7",
     "@slack/webhook": "^7.0.1",

renovate.json

@@ -178,7 +178,7 @@
         "@aws-crypto/sha256-js",
         "@aws-crypto/util",
         "@langtrase/trace-attributes",
-        "@opentelemetry/sdk-trace-base",
+        "@arizeai/openinference-semantic-conventions",
         "@smithy/eventstream-serde-node",
         "@smithy/node-http-handler",
         "@smithy/types",
@@ -3370,7 +3370,7 @@
       ],
       "minimumReleaseAge": "7 days",
       "enabled": true
-    },    
+    },
     {
       "groupName": "redux-actions",
       "matchDepNames": [
@@ -4346,36 +4346,33 @@
       "groupName": "OpenTelemetry modules",
       "matchDepNames": [
         "@grpc/grpc-js",
+        "@elastic/opentelemetry-node",
         "@opentelemetry/api",
-        "@opentelemetry/api-metrics",
-        "@opentelemetry/core",
+        "@opentelemetry/context-async-hooks",
         "@opentelemetry/exporter-metrics-otlp-grpc",
         "@opentelemetry/exporter-prometheus",
-        "@opentelemetry/resources",
-        "@opentelemetry/sdk-metrics-base",
-        "@opentelemetry/semantic-conventions",
-        "@arizeai/openinference-semantic-conventions",
-        "@opentelemetry/context-async-hooks",
-        "@opentelemetry/exporter-trace-otlp-grpc",
         "@opentelemetry/exporter-trace-otlp-http",
         "@opentelemetry/exporter-trace-otlp-proto",
-        "@opentelemetry/otlp-exporter-base",
-        "@opentelemetry/sdk-node",
-        "@opentelemetry/sdk-trace-node",
         "@opentelemetry/instrumentation",
-        "@opentelemetry/instrumentation-http",
-        "@opentelemetry/instrumentation-undici"
+        "@opentelemetry/instrumentation-undici",
+        "@opentelemetry/otlp-exporter-base",
+        "@opentelemetry/semantic-conventions"
       ],
       "reviewers": [
-        "team:stack-monitoring",
-        "team:kibana-core"
+        "team:appex-ai-infra",
+        "team:kibana-core",
+        "team:kibana-security",
+        "team:stack-monitoring"
       ],
       "matchBaseBranches": [
         "main"
       ],
       "labels": [
+        "Team:AI Infra",
         "Team:Monitoring",
-        "backport:all-open",
+        "Team:Core",
+        "Team:Security",
+        "backport:prev-minor",
         "release_note:skip"
       ],
       "minimumReleaseAge": "7 days",

src/dev/precommit_hook/casing_check_config.js

@@ -132,6 +132,7 @@ export const IGNORE_DIRECTORY_GLOBS = [
   'x-pack/dev-tools',
   'packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack',
   'typings/*',
+  'typings/**/*',
 ];
 
 /**

src/platform/packages/private/kbn-import-resolver/src/import_resolver.ts

@@ -142,6 +142,17 @@ export class ImportResolver {
       return this.adaptReq('src/core/server', dirname);
     }
 
+    if (req.startsWith('@modelcontextprotocol/sdk')) {
+      const relPath = req.split('@modelcontextprotocol/sdk')[1];
+      return Path.resolve(REPO_ROOT, `node_modules/@modelcontextprotocol/sdk/dist/esm/${relPath}`);
+    }
+
+    // We need this "hack" because our current import-resolver doesn't support "exports" in package.json.
+    // We should be able to remove this once we support cjs/esm interop.
+    if (req.startsWith('@elastic/opentelemetry-node/sdk')) {
+      return Path.resolve(REPO_ROOT, `node_modules/@elastic/opentelemetry-node/lib/sdk.js`);
+    }
+
     // turn root-relative paths into relative paths
     if (
       req.startsWith('src/') ||

test/common/plugins/otel_metrics/server/monitoring/metrics.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Counter, Meter } from '@opentelemetry/api-metrics';
+import { Counter, Meter } from '@opentelemetry/api';
 
 export class Metrics {
   requestCounter: Counter;

test/common/plugins/otel_metrics/server/plugin.ts

@@ -8,7 +8,7 @@
  */
 
 import { CoreSetup, Plugin } from '@kbn/core/server';
-import { metrics } from '@opentelemetry/api-metrics';
+import { metrics } from '@opentelemetry/api';
 import { generateOtelMetrics } from './routes';
 import { Metrics } from './monitoring/metrics';
 

tsconfig.base.json

@@ -2108,7 +2108,10 @@
       "@kbn/zod-helpers/*": ["src/platform/packages/shared/kbn-zod-helpers/*"],
       // END AUTOMATED PACKAGE LISTING
       // Allows for importing from `kibana` package for the exported types.
-      "@emotion/core": ["typings/@emotion"]
+      "@emotion/core": ["typings/@emotion"],
+      // We need the custom typings "proxy" because our current import-resolver doesn't support "exports" in package.json.
+      // We should be able to remove this once we support cjs/esm interop.
+      "@elastic/opentelemetry-node/sdk": ["typings/@elastic/opentelemetry-node/sdk"],
     },
     // Support .tsx files and transform JSX into calls to React.createElement
     "jsx": "react",

typings/@elastic/opentelemetry-node/sdk.d.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+declare module '@elastic/opentelemetry-node/sdk' {
+  export * from '@elastic/opentelemetry-node/types/sdk';
+}

x-pack/platform/plugins/private/monitoring_collection/README.md

@@ -72,7 +72,7 @@ See the [grpc-node docs](https://github.com/grpc/grpc-node/blob/master/doc/envir
 * First, we need to define what metrics we want to instrument with OpenTelemetry
 
   ```ts
-  import { Counter, Meter } from '@opentelemetry/api-metrics';
+  import { Counter, Meter } from '@opentelemetry/api';
 
   export class FooApiMeters {
     requestCount: Counter;
@@ -92,7 +92,7 @@ See the [grpc-node docs](https://github.com/grpc/grpc-node/blob/master/doc/envir
   ```ts
   import { IRouter } from '@kbn/core/server';
   import { FooApiMeters } from './foo_api_meters';
-  import { metrics } from '@opentelemetry/api-metrics';
+  import { metrics } from '@opentelemetry/api';
 
   export class FooApiPlugin implements Plugin {
     private metrics: Metrics;

x-pack/platform/plugins/private/monitoring_collection/server/lib/prometheus_exporter.ts

@@ -5,15 +5,15 @@
  * 2.0.
  */
 
-import { AggregationTemporality, MetricReader } from '@opentelemetry/sdk-metrics-base';
+import { metrics } from '@elastic/opentelemetry-node/sdk';
 import {
   PrometheusExporter as OpenTelemetryPrometheusExporter,
   ExporterConfig,
   PrometheusSerializer,
 } from '@opentelemetry/exporter-prometheus';
 import { KibanaResponseFactory } from '@kbn/core/server';
 
-export class PrometheusExporter extends MetricReader {
+export class PrometheusExporter extends metrics.MetricReader {
   private readonly prefix?: string;
   private readonly appendTimestamp: boolean;
   private serializer: PrometheusSerializer;
@@ -29,8 +29,8 @@ export class PrometheusExporter extends MetricReader {
     this.serializer = new PrometheusSerializer(this.prefix, this.appendTimestamp);
   }
 
-  selectAggregationTemporality(): AggregationTemporality {
-    return AggregationTemporality.CUMULATIVE;
+  selectAggregationTemporality(): metrics.AggregationTemporality {
+    return metrics.AggregationTemporality.CUMULATIVE;
   }
 
   protected onForceFlush(): Promise<void> {

x-pack/platform/plugins/private/monitoring_collection/server/plugin.ts

@@ -14,12 +14,12 @@ import {
   ServiceStatus,
 } from '@kbn/core/server';
 import { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server';
-import { metrics } from '@opentelemetry/api-metrics';
+import { api, metrics, resources } from '@elastic/opentelemetry-node/sdk';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
-import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics-base';
-import { Resource } from '@opentelemetry/resources';
-import { diag, DiagLogger, DiagLogLevel } from '@opentelemetry/api';
-import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+// import { ATTR_SERVICE_INSTANCE_ID } from '@opentelemetry/semantic-conventions/incubating';
+// Ideally we would import. But our tooling doesn't like how subdirs in these packages are exported (via "exports" in package.json).
+const ATTR_SERVICE_INSTANCE_ID = 'service.instance.id';
 import * as grpc from '@grpc/grpc-js';
 import { PrometheusExporter } from './lib/prometheus_exporter';
 import { MonitoringCollectionConfig } from './config';
@@ -42,7 +42,7 @@ export class MonitoringCollectionPlugin implements Plugin<MonitoringCollectionSe
   private readonly initializerContext: PluginInitializerContext;
   private readonly logger: Logger;
   private readonly config: MonitoringCollectionConfig;
-  private readonly otlpLogger: DiagLogger;
+  private readonly otlpLogger: api.DiagLogger;
 
   private metrics: Record<string, Metric<any>> = {};
 
@@ -126,15 +126,7 @@ export class MonitoringCollectionPlugin implements Plugin<MonitoringCollectionSe
     serviceInstanceId?: string,
     serviceVersion?: string
   ) {
-    const meterProvider = new MeterProvider({
-      resource: new Resource({
-        [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
-        [SemanticResourceAttributes.SERVICE_INSTANCE_ID]: serviceInstanceId,
-        [SemanticResourceAttributes.SERVICE_VERSION]: serviceVersion,
-      }),
-    });
-
-    metrics.setGlobalMeterProvider(meterProvider);
+    const meterReaders: metrics.IMetricReader[] = [];
 
     const otlpConfig = this.config.opentelemetry?.metrics.otlp;
     const url =
@@ -153,12 +145,12 @@ export class MonitoringCollectionPlugin implements Plugin<MonitoringCollectionSe
         }
       }
 
-      const otlpLogLevel = otlpConfig.logLevel.toUpperCase() as keyof typeof DiagLogLevel;
-      diag.setLogger(this.otlpLogger, DiagLogLevel[otlpLogLevel]);
+      const otlpLogLevel = otlpConfig.logLevel.toUpperCase() as keyof typeof api.DiagLogLevel;
+      api.diag.setLogger(this.otlpLogger, api.DiagLogLevel[otlpLogLevel]);
 
       this.logger.debug(`Registering OpenTelemetry metrics exporter to ${url}`);
-      meterProvider.addMetricReader(
-        new PeriodicExportingMetricReader({
+      meterReaders.push(
+        new metrics.PeriodicExportingMetricReader({
           exporter: new OTLPMetricExporter({ url, metadata }),
           exportIntervalMillis: otlpConfig.exportIntervalMillis,
         })
@@ -169,8 +161,19 @@ export class MonitoringCollectionPlugin implements Plugin<MonitoringCollectionSe
       // Add Prometheus exporter
       this.logger.debug(`Starting prometheus exporter at ${PROMETHEUS_PATH}`);
       this.prometheusExporter = new PrometheusExporter();
-      meterProvider.addMetricReader(this.prometheusExporter);
+      meterReaders.push(this.prometheusExporter);
     }
+
+    const meterProvider = new metrics.MeterProvider({
+      resource: resources.resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: serviceName,
+        [ATTR_SERVICE_INSTANCE_ID]: serviceInstanceId,
+        [ATTR_SERVICE_VERSION]: serviceVersion,
+      }),
+      readers: meterReaders,
+    });
+
+    api.metrics.setGlobalMeterProvider(meterProvider);
   }
 
   start() {}

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/instrumentation/lang_tracer.test.ts

@@ -4,21 +4,18 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import {
-  InMemorySpanExporter,
-  BasicTracerProvider,
-  SimpleSpanProcessor,
-  ReadableSpan,
-} from '@opentelemetry/sdk-trace-base';
-import { context } from '@opentelemetry/api';
+import { tracing } from '@elastic/opentelemetry-node/sdk';
+import { context, trace } from '@opentelemetry/api';
 import { LangTracer } from './lang_tracer';
 import { lastValueFrom, of, throwError } from 'rxjs';
 
 describe('langTracer', () => {
-  const provider = new BasicTracerProvider();
-  const memoryExporter = new InMemorySpanExporter();
-  provider.addSpanProcessor(new SimpleSpanProcessor(memoryExporter));
-  provider.register();
+  const memoryExporter = new tracing.InMemorySpanExporter();
+  const provider = new tracing.BasicTracerProvider({
+    spanProcessors: [new tracing.SimpleSpanProcessor(memoryExporter)],
+  });
+
+  trace.setGlobalTracerProvider(provider);
 
   beforeEach(() => {
     memoryExporter.reset();
@@ -36,7 +33,7 @@ describe('langTracer', () => {
       await lastValueFrom(tracer.startActiveSpan('my_span', spanCallback));
 
       const { span } = spanCallback.mock.calls[0][0] as {
-        span: ReadableSpan;
+        span: tracing.ReadableSpan;
       };
 
       expect(span.name).toEqual('my_span');
@@ -69,7 +66,7 @@ describe('langTracer', () => {
       await lastValueFrom(tracer.startActiveSpan('my_span', spanCallback)).catch(errorHandler);
 
       const { span } = spanCallback.mock.calls[0][0] as {
-        span: ReadableSpan;
+        span: tracing.ReadableSpan;
       };
 
       expect(span.status).toEqual({
@@ -95,7 +92,7 @@ describe('langTracer', () => {
       const mappedSpans = memoryExporter.getFinishedSpans().map((span) => ({
         name: span.name,
         id: span.spanContext().spanId,
-        parentId: span.parentSpanId,
+        parentId: span.parentSpanContext?.spanId,
       }));
 
       const parentSpan = mappedSpans.find((span) => span.name === 'parent');

x-pack/solutions/search/plugins/enterprise_search/server/lib/fetch_all.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { isNotNullish } from '@opentelemetry/sdk-metrics-base/build/src/utils';
-
 import { QueryDslQueryContainer, SearchHit } from '@elastic/elasticsearch/lib/api/types';
 import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 
+import { isNotNullish } from '../../common/utils/is_not_nullish';
+
 // TODO add safety to prevent an OOM error if the query results are too enough
 
 export const fetchAll = async <T>(

x-pack/test/api_integration/apis/monitoring_collection/prometheus.ts

@@ -19,7 +19,7 @@ export default function ({ getService }: FtrProviderContext) {
       const response = await supertest.get('/api/monitoring_collection/v1/prometheus').expect(200);
 
       expect(response.text.replace(/\s+/g, ' ')).to.match(
-        /^# HELP request_count_total Counts total number of requests # TYPE request_count_total counter request_count_total [0-9]/
+        /^# HELP target_info Target metadata # TYPE target_info gauge target_info{service_name=".+",service_instance_id=".+",service_version=".+"} 1 # HELP request_count_total Counts total number of requests # TYPE request_count_total counter request_count_total [0-9]/
       );
     });
   });