[Detection Engine] FTR test to replicate undefined elasticsearch state

.buildkite/scripts/common/setup_job_env.sh

@@ -125,11 +125,11 @@ EOF
   SONAR_LOGIN=$(vault_get sonarqube token)
   export SONAR_LOGIN
 
-  ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url)
-  export ELASTIC_APM_SERVER_URL
+  # ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url)
+  # export ELASTIC_APM_SERVER_URL
 
-  ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key)
-  export ELASTIC_APM_API_KEY
+  # ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key)
+  # export ELASTIC_APM_API_KEY
 }
 
 # Set up GenAI keys

scripts/functional_test_runner.js

@@ -8,4 +8,5 @@
  */
 
 require('../src/setup_node_env');
+require('../src/cli/apm')('functional-test-runner', process.argv);
 require('@kbn/test').runFtrCli();

scripts/functional_tests.js

@@ -8,4 +8,5 @@
  */
 
 require('../src/setup_node_env');
+require('../src/cli/apm')('functional-tests', process.argv);
 require('@kbn/test').runTestsCli();

scripts/functional_tests_server.js

@@ -6,6 +6,6 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
 require('../src/setup_node_env');
+require('../src/cli/apm')('functional-test-server', process.argv);
 require('@kbn/test').startServersCli();

src/platform/packages/private/kbn-apm-config-loader/src/init_apm.ts

@@ -9,6 +9,7 @@
 
 import { loadConfiguration } from './config_loader';
 import { piiFilter } from './filters/pii_filter';
+import { patchMocha } from './patch_mocha';
 
 export const initApm = (
   argv: string[],
@@ -17,7 +18,16 @@ export const initApm = (
   serviceName: string
 ) => {
   const apmConfigLoader = loadConfiguration(argv, rootDir, isDistributable);
-  const apmConfig = apmConfigLoader.getConfig(serviceName);
+
+  process.env.ELASTIC_APM_ACTIVE = 'true';
+  process.env.ELASTIC_APM_CONTEXT_PROPAGATION_ONLY = '';
+  process.env.ELASTIC_APM_TRANSACTION_SAMPLE_RATE = '1.0';
+
+  const baseOptions = apmConfigLoader.getConfig(serviceName);
+
+  const apmConfig = {
+    ...baseOptions,
+  };
 
   const shouldRedactUsers = apmConfigLoader.isUsersRedactionEnabled();
 
@@ -30,5 +40,7 @@ export const initApm = (
     apm.addFilter(piiFilter);
   }
 
+  patchMocha(apm);
+
   apm.start(apmConfig);
 };

src/platform/packages/private/kbn-apm-config-loader/src/patch_mocha.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Agent, Span, Transaction } from 'elastic-apm-node';
+import { REPO_ROOT } from '@kbn/repo-info';
+
+interface Runnable {
+  parent?: Suite;
+  title: string;
+}
+
+interface Suite extends Runnable {
+  root: boolean;
+  file?: string;
+  fullTitle: () => string;
+}
+
+export function patchMocha(agent: Agent) {
+  agent.addPatch('mocha', (Mocha: any) => {
+    const Runner = Mocha.Runner;
+
+    const {
+      EVENT_SUITE_BEGIN,
+      EVENT_HOOK_BEGIN,
+      EVENT_HOOK_END,
+      EVENT_TEST_BEGIN,
+      EVENT_TEST_END,
+      EVENT_SUITE_END,
+      EVENT_TEST_FAIL,
+    } = Mocha.Runner.constants;
+
+    const originalRunnerRun = Runner.prototype.run;
+
+    Runner.prototype.run = function (fn: Function) {
+      const runner = this;
+
+      const fileTransactions = new Map<string, Transaction>(); // file -> transaction
+      const suiteSpanMap = new WeakMap<Suite, Span>(); // suite -> span
+      const spanMap = new WeakMap<Runnable, Span>(); // hook/test -> span/
+      const fileSuiteCount = new Map<string, number>(); // file -> count of active suites
+
+      function getFileName(runnable: Suite | Runnable) {
+        const file = 'file' in runnable ? runnable.file : runnable.parent?.file;
+
+        if (!file) {
+          return 'unknown file';
+        }
+        return file.replace(REPO_ROOT, '').substring(1);
+      }
+
+      runner
+        .on(EVENT_SUITE_BEGIN, function onSuiteBegin(suite: Suite) {
+          if (suite.root) return;
+
+          const file = getFileName(suite);
+
+          // Get or create file transaction
+          let fileTx = fileTransactions.get(file);
+          if (!fileTx) {
+            fileTx = agent.startTransaction(file, 'test.file', {
+              childOf: agent.currentTraceparent ?? undefined,
+            });
+            fileTransactions.set(file, fileTx);
+            fileSuiteCount.set(file, 0);
+          }
+
+          // Increment suite count for this file
+          fileSuiteCount.set(file, (fileSuiteCount.get(file) ?? 0) + 1);
+
+          // Create suite span within file transaction
+          const suiteSpan = fileTx.startSpan(suite.fullTitle(), 'test.suite');
+
+          if (suiteSpan) {
+            suiteSpanMap.set(suite, suiteSpan);
+          }
+        })
+        .on(EVENT_HOOK_BEGIN, function onHookBegin(hook: Runnable) {
+          if (hook.parent?.root || !hook.parent) return;
+          const suiteSpan = suiteSpanMap.get(hook.parent);
+          if (!suiteSpan) return;
+          const span = agent.startSpan(hook.title, 'suite.hook', {
+            childOf: suiteSpan.traceparent,
+          });
+          if (span) {
+            spanMap.set(hook, span);
+          }
+        })
+        .on(EVENT_HOOK_END, function onHookEnd(hook: Runnable) {
+          spanMap.get(hook)?.end();
+        })
+        .on(EVENT_TEST_BEGIN, function onTestBegin(test: Runnable) {
+          const suiteSpan = test.parent ? suiteSpanMap.get(test.parent) : undefined;
+          if (!suiteSpan) return;
+          const span = agent.startSpan(test.title, 'test', { childOf: suiteSpan.traceparent });
+          if (span) {
+            spanMap.set(test, span);
+          }
+        })
+        .on(EVENT_TEST_END, function onTestEnd(test: Runnable) {
+          spanMap.get(test)?.end();
+        })
+        .on(EVENT_SUITE_END, function onSuiteEnd(suite: Suite) {
+          if (suite.root) return;
+
+          const suiteSpan = suiteSpanMap.get(suite);
+          suiteSpan?.end();
+
+          const file = getFileName(suite);
+
+          // Decrement suite count and end file transaction if this was the last suite
+          const currentCount = (fileSuiteCount.get(file) ?? 0) - 1;
+          fileSuiteCount.set(file, currentCount);
+
+          if (currentCount === 0) {
+            const fileTx = fileTransactions.get(file);
+            fileTx?.end();
+            fileTransactions.delete(file);
+            fileSuiteCount.delete(file);
+          }
+        })
+        .on(EVENT_TEST_FAIL, function onTestFail(test: Runnable, error: Error) {
+          const span = spanMap.get(test);
+          span?.setOutcome('failure');
+
+          const fileName = getFileName(test);
+          const fileTx = fileTransactions.get(fileName);
+          fileTx?.setOutcome('failure');
+        });
+
+      return originalRunnerRun.call(runner, fn);
+    };
+
+    return Mocha;
+  });
+}

src/platform/packages/private/kbn-apm-config-loader/src/utils/get_config_file_paths.ts

@@ -19,7 +19,10 @@ import { getArgValues } from './read_argv';
  * `-c` and `--config` options from process.argv, and fallbacks to `@kbn/utils`'s `getConfigPath()`
  */
 export const getConfigurationFilePaths = (argv: string[]): string[] => {
-  const rawPaths = getArgValues(argv, ['-c', '--config']);
+  const rawPaths = getArgValues(argv, ['-c', '--config']).filter((path) => {
+    return path.endsWith('.yml');
+  });
+
   if (rawPaths.length) {
     return rawPaths.map((path) => resolve(process.cwd(), path));
   }

src/platform/packages/private/kbn-apm-config-loader/tsconfig.json

@@ -16,6 +16,7 @@
     "@kbn/config-schema",
     "@kbn/std",
     "@kbn/telemetry-config",
+    "@kbn/repo-info",
   ],
   "exclude": [
     "target/**/*",

src/platform/packages/private/kbn-journeys/journey/journey_apm_config.ts

@@ -7,17 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
-const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443';
-const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB';
-
 export const JOURNEY_APM_CONFIG = {
-  serverUrl: APM_SERVER_URL,
-  secretToken: APM_PUBLIC_TOKEN,
-  active: 'true',
-  contextPropagationOnly: 'false',
-  environment: process.env.ELASTIC_APM_ENVIRONMENT || (process.env.CI ? 'ci' : 'development'),
-  transactionSampleRate: '1.0',
   // capture request body for both errors and request transactions
   // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body
   captureBody: 'all',

src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts

@@ -95,12 +95,6 @@ export function makeFtrConfigProvider(
         ],
 
         env: {
-          ELASTIC_APM_ACTIVE: JOURNEY_APM_CONFIG.active,
-          ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: JOURNEY_APM_CONFIG.contextPropagationOnly,
-          ELASTIC_APM_ENVIRONMENT: JOURNEY_APM_CONFIG.environment,
-          ELASTIC_APM_TRANSACTION_SAMPLE_RATE: JOURNEY_APM_CONFIG.transactionSampleRate,
-          ELASTIC_APM_SERVER_URL: JOURNEY_APM_CONFIG.serverUrl,
-          ELASTIC_APM_SECRET_TOKEN: JOURNEY_APM_CONFIG.secretToken,
           ELASTIC_APM_CAPTURE_BODY: JOURNEY_APM_CONFIG.captureBody,
           ELASTIC_APM_CAPTURE_HEADERS: JOURNEY_APM_CONFIG.captureRequestHeaders,
           ELASTIC_APM_LONG_FIELD_MAX_LENGTH: JOURNEY_APM_CONFIG.longFieldMaxLength,

src/platform/packages/private/kbn-journeys/journey/journey_ftr_harness.ts

@@ -8,7 +8,7 @@
  */
 
 import Url from 'url';
-import { inspect, format } from 'util';
+import { inspect } from 'util';
 import { setTimeout as setTimer } from 'timers/promises';
 import * as Rx from 'rxjs';
 import apmNode from 'elastic-apm-node';
@@ -117,35 +117,7 @@ export class JourneyFtrHarness {
     // Update labels before start for consistency b/w APM services
     await this.updateTelemetryAndAPMLabels(journeyLabels);
 
-    this.apm = apmNode.start({
-      serviceName: 'functional test runner',
-      environment: process.env.CI ? 'ci' : 'development',
-      active: kbnTestServerEnv.ELASTIC_APM_ACTIVE !== 'false',
-      serverUrl: kbnTestServerEnv.ELASTIC_APM_SERVER_URL,
-      secretToken: kbnTestServerEnv.ELASTIC_APM_SECRET_TOKEN,
-      globalLabels: kbnTestServerEnv.ELASTIC_APM_GLOBAL_LABELS,
-      transactionSampleRate: kbnTestServerEnv.ELASTIC_APM_TRANSACTION_SAMPLE_RATE,
-      logger: {
-        warn: (...args: any[]) => {
-          this.log.warning('APM WARN', ...args);
-        },
-        info: (...args: any[]) => {
-          this.log.info('APM INFO', ...args);
-        },
-        fatal: (...args: any[]) => {
-          this.log.error(format('APM FATAL', ...args));
-        },
-        error: (...args: any[]) => {
-          this.log.error(format('APM ERROR', ...args));
-        },
-        debug: (...args: any[]) => {
-          this.log.debug('APM DEBUG', ...args);
-        },
-        trace: (...args: any[]) => {
-          this.log.verbose('APM TRACE', ...args);
-        },
-      },
-    });
+    this.apm = apmNode;
 
     if (this.currentTransaction) {
       throw new Error(`Transaction exist, end prev transaction ${this.currentTransaction?.name}`);

src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts

@@ -14,6 +14,7 @@ import type { Client } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/tooling-log';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { KbnClient } from '@kbn/test';
+import { withSpan } from '@kbn/apm-utils';
 import type { LoadActionPerfOptions } from './lib';
 import {
   saveAction,
@@ -97,16 +98,18 @@ export class EsArchiver {
       performance?: LoadActionPerfOptions;
     } = {}
   ) {
-    return await loadAction({
-      inputDir: this.findArchive(path),
-      skipExisting: !!skipExisting,
-      useCreate: !!useCreate,
-      docsOnly,
-      client: this.client,
-      log: this.log,
-      kbnClient: this.kbnClient,
-      performance,
-    });
+    return await withSpan('es_archiver load', () =>
+      loadAction({
+        inputDir: this.findArchive(path),
+        skipExisting: !!skipExisting,
+        useCreate: !!useCreate,
+        docsOnly,
+        client: this.client,
+        log: this.log,
+        kbnClient: this.kbnClient,
+        performance,
+      })
+    );
   }
 
   /**
@@ -115,12 +118,14 @@ export class EsArchiver {
    * @param {String} path - relative path to the archive to unload, resolved relative to this.baseDir which defaults to REPO_ROOT
    */
   async unload(path: string) {
-    return await unloadAction({
-      inputDir: this.findArchive(path),
-      client: this.client,
-      log: this.log,
-      kbnClient: this.kbnClient,
-    });
+    return await withSpan('es_archiver unload', () =>
+      unloadAction({
+        inputDir: this.findArchive(path),
+        client: this.client,
+        log: this.log,
+        kbnClient: this.kbnClient,
+      })
+    );
   }
 
   /**
@@ -164,10 +169,12 @@ export class EsArchiver {
    * Cleanup saved object indices, preserving the space:default saved object.
    */
   async emptyKibanaIndex() {
-    return await emptyKibanaIndexAction({
-      client: this.client,
-      log: this.log,
-    });
+    return await withSpan('es_archiver empty_kibana_index', () =>
+      emptyKibanaIndexAction({
+        client: this.client,
+        log: this.log,
+      })
+    );
   }
 
   /**