[OTel Tracing] HTTP instrumentation

.buildkite/scripts/steps/security/third_party_packages.txt

@@ -29,7 +29,6 @@ yaml-language-server
 @atlaskit/pragmatic-drag-and-drop
 @atlaskit/pragmatic-drag-and-drop-hitbox
 @opentelemetry/exporter-metrics-otlp-proto
-@opentelemetry/instrumentation-hapi
 simple-statistics
 monaco-promql
 oxlint

package.json

@@ -1272,7 +1272,6 @@
     "@opentelemetry/exporter-trace-otlp-http": "0.211.0",
     "@opentelemetry/exporter-trace-otlp-proto": "0.211.0",
     "@opentelemetry/instrumentation": "0.211.0",
-    "@opentelemetry/instrumentation-hapi": "0.57.0",
     "@opentelemetry/instrumentation-http": "0.211.0",
     "@opentelemetry/instrumentation-undici": "0.21.0",
     "@opentelemetry/otlp-exporter-base": "0.211.0",

renovate.json

@@ -2456,7 +2456,6 @@
         "@opentelemetry/exporter-trace-otlp-http",
         "@opentelemetry/exporter-trace-otlp-proto",
         "@opentelemetry/instrumentation",
-        "@opentelemetry/instrumentation-hapi",
         "@opentelemetry/instrumentation-http",
         "@opentelemetry/instrumentation-undici",
         "@opentelemetry/otlp-exporter-base",

src/core/packages/http/router-server-internal/moon.yml

@@ -31,6 +31,7 @@ dependsOn:
   - '@kbn/config-mocks'
   - '@kbn/config'
   - '@kbn/core-security-server'
+  - '@kbn/tracing-utils'
 tags:
   - shared-server
   - package

src/core/packages/http/router-server-internal/src/router.ts

@@ -30,7 +30,7 @@ import type {
 } from '@kbn/core-http-server';
 import type { RouteSecurityGetter } from '@kbn/core-http-server';
 import type { Env } from '@kbn/config';
-import { context, defaultTextMapGetter, propagation } from '@opentelemetry/api';
+import { withActiveSpan } from '@kbn/tracing-utils';
 import { CoreVersionedRouter } from './versioned_router';
 import type { CoreKibanaRequest } from './request';
 import { getProtocolFromRequest } from './request';
@@ -186,17 +186,8 @@ export class Router<Context extends RequestHandlerContextBase = RequestHandlerCo
     this.routes.push({
       ...route,
       handler: async (request, responseToolkit) => {
-        /**
-         * Read incoming traceparent headers and create a new context with the traceparent set.
-         * This allows OpenTelemetry spans created in the next context to re-use the traceparent
-         * headers (and thus belonging to the same trace). It does not interfere with Elastic APM,
-         * and is temporary until we fully migrate to [OpenTelemetry
-         * tracing](https://github.com/elastic/kibana/issues/220914).
-         */
-        const ctx = propagation.extract(context.active(), request.headers, defaultTextMapGetter);
-        return context.with(
-          ctx,
-          async () => await this.handle({ request, responseToolkit, handler: route.handler })
+        return withActiveSpan('route handler', () =>
+          this.handle({ request, responseToolkit, handler: route.handler })
         );
       },
     });

src/core/packages/http/router-server-internal/tsconfig.json

@@ -22,7 +22,8 @@
     "@kbn/logging-mocks",
     "@kbn/config-mocks",
     "@kbn/config",
-    "@kbn/core-security-server"
+    "@kbn/core-security-server",
+    "@kbn/tracing-utils"
   ],
   "exclude": [
     "target/**/*",

src/core/packages/http/server-internal/src/http_server.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { context, trace, type Span as OTelSpan } from '@opentelemetry/api';
 import type { Request, Server } from '@hapi/hapi';
 import HapiStaticFiles from '@hapi/inert';
 import url from 'url';
@@ -75,15 +76,15 @@ function startEluMeasurement<T>(
   path: string,
   log: Logger,
   eluMonitorOptions: IHttpEluMonitorConfig | undefined
-): () => void {
+): (httpSpan?: OTelSpan) => void {
   if (!eluMonitorOptions?.enabled) {
     return identity;
   }
 
   const startUtilization = performance.eventLoopUtilization();
   const start = performance.now();
 
-  return function stopEluMeasurement() {
+  return function stopEluMeasurement(httpSpan?: OTelSpan) {
     const { active, utilization } = performance.eventLoopUtilization(startUtilization);
 
     apm.currentTransaction?.addLabels(
@@ -93,6 +94,10 @@ function startEluMeasurement<T>(
       },
       false
     );
+    httpSpan?.setAttributes({
+      'nodejs.eventloop.utilization': utilization,
+      'nodejs.eventloop.active': active,
+    });
 
     const duration = performance.now() - start;
 
@@ -546,17 +551,22 @@ export class HttpServer {
     userActivity?: InternalUserActivityServiceSetup
   ) {
     this.server!.ext('onPreResponse', (request, responseToolkit) => {
-      const stop = (request.app as KibanaRequestState).measureElu;
+      const app = request.app as KibanaRequestState;
+      app.httpSpan?.updateName(`${request.route.method.toUpperCase()} ${request.route.path}`);
+
+      const stop = app.measureElu;
 
       if (!stop) {
         return responseToolkit.continue;
       }
 
       if (isBoom(request.response)) {
-        stop();
+        stop(app.httpSpan);
+        app.otelSubSpan?.end();
       } else {
         request.response.events.once('finish', () => {
-          stop();
+          stop(app.httpSpan);
+          app.otelSubSpan?.end();
         });
       }
 
@@ -584,7 +594,19 @@ export class HttpServer {
 
       if (executionContext && parentContext) {
         executionContext.set(parentContext);
-        apm.addLabels(executionContext.getAsLabels());
+        const labels = executionContext.getAsLabels();
+        apm.addLabels(labels);
+        const { name, id, page } = labels;
+        trace.getActiveSpan()?.setAttributes(
+          omitBy(
+            {
+              'kibana.execution_context.name': name,
+              'kibana.execution_context.id': id,
+              'kibana.execution_context.page': page,
+            },
+            isNil
+          ) as Record<string, string>
+        );
       }
 
       executionContext?.setRequestId(requestId);
@@ -598,6 +620,8 @@ export class HttpServer {
       // The current implementation of the APM agent ends a request transaction before "response" log is emitted.
       app.traceId = apm.currentTraceIds['trace.id'];
       app.span = apm.startSpan('pre-route handler middlewares');
+      app.httpSpan = trace.getActiveSpan();
+      app.otelSubSpan = this.createSubspan('pre-route handler middlewares');
 
       return responseToolkit.continue;
     });
@@ -620,8 +644,13 @@ export class HttpServer {
     });
 
     this.server!.ext('onPreHandler', (request, responseToolkit) => {
-      (request.app as KibanaRequestState).span?.end();
-      (request.app as KibanaRequestState).span = null;
+      const app = request.app as KibanaRequestState;
+      app.span?.end();
+      app.otelSubSpan?.end();
+      // Clear the sub-spans for the handler because it is created when actually calling the handler
+      // in src/core/packages/http/router-server-internal/src/router.ts
+      app.span = null;
+      app.otelSubSpan = undefined;
 
       const user = this.authState.get<AuthenticatedUser>(request).state ?? null;
       const { redactedSessionId } = request.app as KibanaRequestState;
@@ -654,9 +683,26 @@ export class HttpServer {
       return responseToolkit.continue;
     });
 
+    this.server!.ext('onPostHandler', (request, responseToolkit) => {
+      const app = request.app as KibanaRequestState;
+      app.otelSubSpan?.end();
+      app.otelSubSpan = this.createSubspan('post-route handler middlewares');
+
+      return responseToolkit.continue;
+    });
+
     this.instrumentMetrics();
   }
 
+  private createSubspan(name: string): OTelSpan {
+    const span = trace.getTracer('kibana.http').startSpan(name);
+    context.with(context.active(), () => {
+      // Hold the active context until the otelSubSpan ends
+      context.bind(context.active(), span);
+    });
+    return span;
+  }
+
   private instrumentMetrics() {
     const meter = metrics.getMeter('kibana.http.server');
 

src/core/packages/http/server/src/router/request.ts

@@ -8,6 +8,7 @@
  */
 
 import type { URL } from 'url';
+import type { Span as OTelSpan } from '@opentelemetry/api';
 import type { RequestApplicationState, RouteOptionsApp } from '@hapi/hapi';
 import type { Observable } from 'rxjs';
 import type { Span } from 'elastic-apm-node';
@@ -42,9 +43,14 @@ export interface KibanaRequestState extends RequestApplicationState {
   requestUuid: string;
   rewrittenUrl?: URL;
   traceId?: string;
+  /** The top HTTP Otel Span for this request. */
+  httpSpan?: OTelSpan;
+  /** The OTel sub-span: used to group the pre-route handlers, the route handler, and the post-route handlers. */
+  otelSubSpan?: OTelSpan;
+  /** The Elastic APM span to group the pre-route handlers, the route handler, and the post-route handlers. */
   span?: Span | null;
   authzResult?: Record<string, boolean>;
-  measureElu?: () => void;
+  measureElu?: (httpSpan?: OTelSpan) => void;
   startTime: number;
   redactedSessionId?: string;
 }

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/evals_tracing/stateful/classic.stateful.config.ts

@@ -122,7 +122,6 @@ export const servers: ScoutServerConfig = {
     ...defaultConfig.kbnTestServer,
     env: {
       ...defaultConfig.kbnTestServer.env,
-      ...(shouldEnableTracing ? { KBN_OTEL_AUTO_INSTRUMENTATIONS: 'true' } : {}),
     },
     serverArgs: [
       ...defaultConfig.kbnTestServer.serverArgs,

src/platform/packages/shared/kbn-telemetry/src/init_autoinstrumentations.ts

@@ -10,39 +10,36 @@
 import { registerInstrumentations } from '@opentelemetry/instrumentation';
 import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { HapiInstrumentation } from '@opentelemetry/instrumentation-hapi';
+import type { OutgoingHttpHeaders } from 'http';
 
 export function maybeInitAutoInstrumentations() {
-  /**
-   * Auto-instrumentation is intentionally opt-in.
-   *
-   * It can increase trace volume significantly, and Kibana generally relies on explicit
-   * instrumentation for tracing. For evals, enabling this provides request-scoped context
-   * propagation (so W3C baggage like `kibana.evals.run_id` can be extracted and attached).
-   */
-  if (process.env.KBN_OTEL_AUTO_INSTRUMENTATIONS === 'true') {
-    // Register OpenTelemetry auto-instrumentations once per process.
-    // NOTE: these instrumentations must not be enabled alongside Elastic APM.
-    const INSTRUMENTATIONS_REGISTERED = Symbol.for('kbn.tracing.instrumentations_registered');
-    if (!(globalThis as any)[INSTRUMENTATIONS_REGISTERED]) {
-      (globalThis as any)[INSTRUMENTATIONS_REGISTERED] = true;
-      registerInstrumentations({
-        instrumentations: [
-          // Kibana runs on Hapi. This instrumentation gives us higher-level request spans
-          // and ensures context propagation for request-scoped correlation (like eval run ids).
-          new HapiInstrumentation(),
-          // Create incoming HTTP server spans and extract trace context + baggage.
-          new HttpInstrumentation({
-            // Only create outgoing spans when there is an active parent span.
-            // This keeps noise down and ensures spans remain connected to request traces.
-            requireParentforOutgoingSpans: true,
-          }),
-          // undici is used by Elasticsearch client; require a parent so we don't create a new trace per request.
-          new UndiciInstrumentation({
-            requireParentforSpans: true,
-          }),
-        ],
-      });
-    }
+  // Register OpenTelemetry auto-instrumentations once per process.
+  // NOTE: these instrumentations must not be enabled alongside Elastic APM.
+  const INSTRUMENTATIONS_REGISTERED = Symbol.for('kbn.tracing.instrumentations_registered');
+  if (!(globalThis as any)[INSTRUMENTATIONS_REGISTERED]) {
+    (globalThis as any)[INSTRUMENTATIONS_REGISTERED] = true;
+    registerInstrumentations({
+      instrumentations: [
+        // Create incoming HTTP server spans and extract trace context + baggage.
+        new HttpInstrumentation({
+          // Only create outgoing spans when there is an active parent span.
+          // This keeps noise down and ensures spans remain connected to request traces.
+          requireParentforOutgoingSpans: true,
+          // Discard spans for Elasticsearch requests because they are already instrumented by the library.
+          ignoreOutgoingRequestHook: (request) => {
+            const headers = (request.headers || {}) as OutgoingHttpHeaders;
+            // Default headers for the ES client: https://github.com/elastic/kibana/blob/8c43c76a8b76eedc622307c544466f9024c4251c/src/core/packages/elasticsearch/client-server-internal/src/headers.ts#L64-L69
+            const isElasticsearch =
+              headers['x-elastic-product-origin'] === 'kibana' &&
+              headers['user-agent']?.startsWith('Kibana/') === true;
+            return isElasticsearch;
+          },
+        }),
+        // require a parent so we don't create a new trace per request.
+        new UndiciInstrumentation({
+          requireParentforSpans: true,
+        }),
+      ],
+    });
   }
 }

src/platform/packages/shared/kbn-tracing/src/init_tracing.ts

@@ -91,12 +91,6 @@ export function initTracing({
 
   trace.setGlobalTracerProvider(nodeTracerProvider);
 
-  propagation.setGlobalPropagator(
-    new core.CompositePropagator({
-      propagators: [new core.W3CTraceContextPropagator(), new core.W3CBaggagePropagator()],
-    })
-  );
-
   const shutdown = async () => {
     await Promise.all(allSpanProcessors.map((processor) => processor.shutdown()));
   };

yarn.lock

@@ -10948,7 +10948,7 @@
     "@opentelemetry/instrumentation" "^0.57.1"
     "@opentelemetry/semantic-conventions" "^1.27.0"
 
-"@opentelemetry/instrumentation-hapi@0.57.0", "@opentelemetry/instrumentation-hapi@^0.57.0":
+"@opentelemetry/instrumentation-hapi@^0.57.0":
   version "0.57.0"
   resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.57.0.tgz#27b3a44a51444af3100a321f2e40623e89e5bb75"
   integrity sha512-Os4THbvls8cTQTVA8ApLfZZztuuqGEeqog0XUnyRW7QVF0d/vOVBEcBCk1pazPFmllXGEdNbbat8e2fYIWdFbw==