Migrate server-side apm.addLabels to OTel dual-write helpers

src/core/packages/feature-flags/server-internal/moon.yml

@@ -17,6 +17,7 @@ project:
   owner: '@elastic/kibana-core'
   sourceRoot: src/core/packages/feature-flags/server-internal
 dependsOn:
+  - '@kbn/apm-utils'
   - '@kbn/core-base-server-internal'
   - '@kbn/core-feature-flags-server'
   - '@kbn/logging'

src/core/packages/feature-flags/server-internal/src/feature_flags_service.test.ts

@@ -167,19 +167,19 @@ describe('FeatureFlagsService Server', () => {
     test('get boolean flag', async () => {
       const value = false;
       await expect(startContract.getBooleanValue('my-flag', value)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
     });
 
     test('get string flag', async () => {
       const value = 'my-default';
       await expect(startContract.getStringValue('my-flag', value)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
     });
 
     test('get number flag', async () => {
       const value = 42;
       await expect(startContract.getNumberValue('my-flag', value)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
     });
 
     test('observe a boolean flag', async () => {
@@ -189,7 +189,7 @@ describe('FeatureFlagsService Server', () => {
       flag$.subscribe((v) => observedValues.push(v));
       // Initial emission
       await expect(firstValueFrom(flag$)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
       expect(observedValues).toHaveLength(1);
 
       // Does not reevaluate and emit if the other flags are changed
@@ -215,7 +215,7 @@ describe('FeatureFlagsService Server', () => {
       flag$.subscribe((v) => observedValues.push(v));
       // Initial emission
       await expect(firstValueFrom(flag$)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
       expect(observedValues).toHaveLength(1);
 
       // Does not reevaluate and emit if the other flags are changed
@@ -241,7 +241,7 @@ describe('FeatureFlagsService Server', () => {
       flag$.subscribe((v) => observedValues.push(v));
       // Initial emission
       await expect(firstValueFrom(flag$)).resolves.toEqual(value);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-flag': value }, undefined);
       expect(observedValues).toHaveLength(1);
 
       // Does not reevaluate and emit if the other flags are changed
@@ -265,7 +265,7 @@ describe('FeatureFlagsService Server', () => {
       await expect(startContract.getBooleanValue('my-overridden-flag', false)).resolves.toEqual(
         true
       );
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-overridden-flag': true });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-overridden-flag': true }, undefined);
       expect(getBooleanValueSpy).not.toHaveBeenCalled();
 
       // Only to prove the spy works
@@ -280,7 +280,7 @@ describe('FeatureFlagsService Server', () => {
       flag$.subscribe((v) => observedValues.push(v));
       // Initial emission
       await expect(firstValueFrom(flag$)).resolves.toEqual(true);
-      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-overridden-flag': true });
+      expect(apmSpy).toHaveBeenCalledWith({ 'flag_my-overridden-flag': true }, undefined);
       expect(observedValues).toHaveLength(1);
 
       // Does not reevaluate and emit if the other flags are changed

src/core/packages/feature-flags/server-internal/src/feature_flags_service.ts

@@ -15,7 +15,7 @@ import type {
   MultiContextEvaluationContext,
 } from '@kbn/core-feature-flags-server';
 import type { Logger } from '@kbn/logging';
-import apm from 'elastic-apm-node';
+import { addSpanLabels } from '@kbn/apm-utils';
 import { getFlattenedObject } from '@kbn/std';
 import {
   type Client,
@@ -215,7 +215,7 @@ export class FeatureFlagsService {
         ? (override as T)
         : // We have to bind the evaluation or the client will lose its internal context
           await evaluationFn.bind(this.featureFlagsClient)(flagName, fallbackValue);
-    apm.addLabels({ [`flag_${flagName.replaceAll('.', '_')}`]: value });
+    addSpanLabels({ [`flag_${flagName.replaceAll('.', '_')}`]: value });
     // TODO: increment usage counter
     return value;
   }

src/core/packages/feature-flags/server-internal/tsconfig.json

@@ -14,6 +14,7 @@
     "target/**/*"
   ],
   "kbn_references": [
+    "@kbn/apm-utils",
     "@kbn/core-base-server-internal",
     "@kbn/core-feature-flags-server",
     "@kbn/logging",

src/core/packages/http/server-internal/moon.yml

@@ -47,6 +47,7 @@ dependsOn:
   - '@kbn/core-user-activity-server-mocks'
   - '@kbn/core-security-common'
   - '@kbn/spaces-utils'
+  - '@kbn/apm-utils'
 tags:
   - shared-server
   - package

src/core/packages/http/server-internal/src/http_server.ts

@@ -17,6 +17,7 @@ import type { Duration } from 'moment';
 import type { Observable, Subscription } from 'rxjs';
 import { firstValueFrom, pairwise, take } from 'rxjs';
 import apm from 'elastic-apm-node';
+import { addSpanLabels, addTransactionLabels } from '@kbn/apm-utils';
 import Brok from 'brok';
 import type { Logger, LoggerFactory } from '@kbn/logging';
 import type { AuthenticatedUser } from '@kbn/core-security-common';
@@ -76,28 +77,30 @@ function startEluMeasurement<T>(
   path: string,
   log: Logger,
   eluMonitorOptions: IHttpEluMonitorConfig | undefined
-): (httpSpan?: OTelSpan) => void {
+): () => void {
   if (!eluMonitorOptions?.enabled) {
     return identity;
   }
 
   const startUtilization = performance.eventLoopUtilization();
   const start = performance.now();
 
-  return function stopEluMeasurement(httpSpan?: OTelSpan) {
+  return function stopEluMeasurement() {
     const { active, utilization } = performance.eventLoopUtilization(startUtilization);
 
-    apm.currentTransaction?.addLabels(
+    addTransactionLabels(
       {
         event_loop_utilization: utilization,
         event_loop_active: active,
       },
-      false
+      {
+        isString: false,
+        otelAttributes: {
+          'nodejs.eventloop.utilization': utilization,
+          'nodejs.eventloop.active': active,
+        },
+      }
     );
-    httpSpan?.setAttributes({
-      'nodejs.eventloop.utilization': utilization,
-      'nodejs.eventloop.active': active,
-    });
 
     const duration = performance.now() - start;
 
@@ -601,11 +604,11 @@ export class HttpServer {
       }
 
       if (isBoom(request.response)) {
-        stop(app.httpSpan);
+        stop();
         app.otelSubSpan?.end();
       } else {
         request.response.events.once('finish', () => {
-          stop(app.httpSpan);
+          stop();
           app.otelSubSpan?.end();
         });
       }
@@ -635,18 +638,17 @@ export class HttpServer {
       if (executionContext && parentContext) {
         executionContext.set(parentContext);
         const labels = executionContext.getAsLabels();
-        apm.addLabels(labels);
         const { name, id, page } = labels;
-        trace.getActiveSpan()?.setAttributes(
-          omitBy(
+        addSpanLabels(labels, {
+          otelAttributes: omitBy(
             {
               'kibana.execution_context.name': name,
               'kibana.execution_context.id': id,
               'kibana.execution_context.page': page,
             },
             isNil
-          ) as Record<string, string>
-        );
+          ) as Record<string, string>,
+        });
       }
 
       executionContext?.setRequestId(requestId);

src/core/packages/http/server-internal/tsconfig.json

@@ -41,6 +41,7 @@
     "@kbn/core-user-activity-server-mocks",
     "@kbn/core-security-common",
     "@kbn/spaces-utils",
+    "@kbn/apm-utils",
   ],
   "exclude": [
     "target/**/*",

src/platform/packages/shared/kbn-apm-utils/index.ts

@@ -8,3 +8,4 @@
  */
 export { withSpan, type SpanOptions, parseSpanOptions } from './src/with_span';
 export { instrumentAsyncMethods } from './src/instrument_async_methods';
+export { addSpanLabels, addTransactionLabels, type Labels } from './src/add_labels';