diff --git a/.buildkite/scripts/common/setup_job_env.sh b/.buildkite/scripts/common/setup_job_env.sh index 058258b50cf69..45b43a499058c 100644 --- a/.buildkite/scripts/common/setup_job_env.sh +++ b/.buildkite/scripts/common/setup_job_env.sh @@ -125,11 +125,11 @@ EOF SONAR_LOGIN=$(vault_get sonarqube token) export SONAR_LOGIN - ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url) - export ELASTIC_APM_SERVER_URL + # ELASTIC_APM_SERVER_URL=$(vault_get project-kibana-ci-apm apm_server_url) + # export ELASTIC_APM_SERVER_URL - ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key) - export ELASTIC_APM_API_KEY + # ELASTIC_APM_API_KEY=$(vault_get project-kibana-ci-apm apm_server_api_key) + # export ELASTIC_APM_API_KEY } # Set up GenAI keys diff --git a/scripts/functional_test_runner.js b/scripts/functional_test_runner.js index b663db4861f9d..ea141e98832b0 100644 --- a/scripts/functional_test_runner.js +++ b/scripts/functional_test_runner.js @@ -8,4 +8,5 @@ */ require('../src/setup_node_env'); +require('../src/cli/apm')('functional-test-runner', process.argv); require('@kbn/test').runFtrCli(); diff --git a/scripts/functional_tests.js b/scripts/functional_tests.js index a73197f3df84c..8207d697ba7ef 100644 --- a/scripts/functional_tests.js +++ b/scripts/functional_tests.js @@ -8,4 +8,5 @@ */ require('../src/setup_node_env'); +require('../src/cli/apm')('functional-tests', process.argv); require('@kbn/test').runTestsCli(); diff --git a/scripts/functional_tests_server.js b/scripts/functional_tests_server.js index f7f8c8a5fc006..4046b40acaacc 100644 --- a/scripts/functional_tests_server.js +++ b/scripts/functional_tests_server.js @@ -6,6 +6,6 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ - require('../src/setup_node_env'); +require('../src/cli/apm')('functional-test-server', process.argv); require('@kbn/test').startServersCli(); diff --git a/src/platform/packages/private/kbn-apm-config-loader/src/init_apm.ts b/src/platform/packages/private/kbn-apm-config-loader/src/init_apm.ts index 86589d06f900c..710a0a770aced 100644 --- a/src/platform/packages/private/kbn-apm-config-loader/src/init_apm.ts +++ b/src/platform/packages/private/kbn-apm-config-loader/src/init_apm.ts @@ -9,6 +9,7 @@ import { loadConfiguration } from './config_loader'; import { piiFilter } from './filters/pii_filter'; +import { patchMocha } from './patch_mocha'; export const initApm = ( argv: string[], @@ -17,7 +18,16 @@ export const initApm = ( serviceName: string ) => { const apmConfigLoader = loadConfiguration(argv, rootDir, isDistributable); - const apmConfig = apmConfigLoader.getConfig(serviceName); + + process.env.ELASTIC_APM_ACTIVE = 'true'; + process.env.ELASTIC_APM_CONTEXT_PROPAGATION_ONLY = ''; + process.env.ELASTIC_APM_TRANSACTION_SAMPLE_RATE = '1.0'; + + const baseOptions = apmConfigLoader.getConfig(serviceName); + + const apmConfig = { + ...baseOptions, + }; const shouldRedactUsers = apmConfigLoader.isUsersRedactionEnabled(); @@ -30,5 +40,7 @@ export const initApm = ( apm.addFilter(piiFilter); } + patchMocha(apm); + apm.start(apmConfig); }; diff --git a/src/platform/packages/private/kbn-apm-config-loader/src/patch_mocha.ts b/src/platform/packages/private/kbn-apm-config-loader/src/patch_mocha.ts new file mode 100644 index 0000000000000..db6ce231c594f --- /dev/null +++ b/src/platform/packages/private/kbn-apm-config-loader/src/patch_mocha.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Agent, Span, Transaction } from 'elastic-apm-node'; +import { REPO_ROOT } from '@kbn/repo-info'; + +interface Runnable { + parent?: Suite; + title: string; +} + +interface Suite extends Runnable { + root: boolean; + file?: string; + fullTitle: () => string; +} + +export function patchMocha(agent: Agent) { + agent.addPatch('mocha', (Mocha: any) => { + const Runner = Mocha.Runner; + + const { + EVENT_SUITE_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_TEST_END, + EVENT_SUITE_END, + EVENT_TEST_FAIL, + } = Mocha.Runner.constants; + + const originalRunnerRun = Runner.prototype.run; + + Runner.prototype.run = function (fn: Function) { + const runner = this; + + const fileTransactions = new Map(); // file -> transaction + const suiteSpanMap = new WeakMap(); // suite -> span + const spanMap = new WeakMap(); // hook/test -> span/ + const fileSuiteCount = new Map(); // file -> count of active suites + + function getFileName(runnable: Suite | Runnable) { + const file = 'file' in runnable ? runnable.file : runnable.parent?.file; + + if (!file) { + return 'unknown file'; + } + return file.replace(REPO_ROOT, '').substring(1); + } + + runner + .on(EVENT_SUITE_BEGIN, function onSuiteBegin(suite: Suite) { + if (suite.root) return; + + const file = getFileName(suite); + + // Get or create file transaction + let fileTx = fileTransactions.get(file); + if (!fileTx) { + fileTx = agent.startTransaction(file, 'test.file', { + childOf: agent.currentTraceparent ?? undefined, + }); + fileTransactions.set(file, fileTx); + fileSuiteCount.set(file, 0); + } + + // Increment suite count for this file + fileSuiteCount.set(file, (fileSuiteCount.get(file) ?? 0) + 1); + + // Create suite span within file transaction + const suiteSpan = fileTx.startSpan(suite.fullTitle(), 'test.suite'); + + if (suiteSpan) { + suiteSpanMap.set(suite, suiteSpan); + } + }) + .on(EVENT_HOOK_BEGIN, function onHookBegin(hook: Runnable) { + if (hook.parent?.root || !hook.parent) return; + const suiteSpan = suiteSpanMap.get(hook.parent); + if (!suiteSpan) return; + const span = agent.startSpan(hook.title, 'suite.hook', { + childOf: suiteSpan.traceparent, + }); + if (span) { + spanMap.set(hook, span); + } + }) + .on(EVENT_HOOK_END, function onHookEnd(hook: Runnable) { + spanMap.get(hook)?.end(); + }) + .on(EVENT_TEST_BEGIN, function onTestBegin(test: Runnable) { + const suiteSpan = test.parent ? suiteSpanMap.get(test.parent) : undefined; + if (!suiteSpan) return; + const span = agent.startSpan(test.title, 'test', { childOf: suiteSpan.traceparent }); + if (span) { + spanMap.set(test, span); + } + }) + .on(EVENT_TEST_END, function onTestEnd(test: Runnable) { + spanMap.get(test)?.end(); + }) + .on(EVENT_SUITE_END, function onSuiteEnd(suite: Suite) { + if (suite.root) return; + + const suiteSpan = suiteSpanMap.get(suite); + suiteSpan?.end(); + + const file = getFileName(suite); + + // Decrement suite count and end file transaction if this was the last suite + const currentCount = (fileSuiteCount.get(file) ?? 0) - 1; + fileSuiteCount.set(file, currentCount); + + if (currentCount === 0) { + const fileTx = fileTransactions.get(file); + fileTx?.end(); + fileTransactions.delete(file); + fileSuiteCount.delete(file); + } + }) + .on(EVENT_TEST_FAIL, function onTestFail(test: Runnable, error: Error) { + const span = spanMap.get(test); + span?.setOutcome('failure'); + + const fileName = getFileName(test); + const fileTx = fileTransactions.get(fileName); + fileTx?.setOutcome('failure'); + }); + + return originalRunnerRun.call(runner, fn); + }; + + return Mocha; + }); +} diff --git a/src/platform/packages/private/kbn-apm-config-loader/src/utils/get_config_file_paths.ts b/src/platform/packages/private/kbn-apm-config-loader/src/utils/get_config_file_paths.ts index 4ff84c208d846..3ab405998207c 100644 --- a/src/platform/packages/private/kbn-apm-config-loader/src/utils/get_config_file_paths.ts +++ b/src/platform/packages/private/kbn-apm-config-loader/src/utils/get_config_file_paths.ts @@ -19,7 +19,10 @@ import { getArgValues } from './read_argv'; * `-c` and `--config` options from process.argv, and fallbacks to `@kbn/utils`'s `getConfigPath()` */ export const getConfigurationFilePaths = (argv: string[]): string[] => { - const rawPaths = getArgValues(argv, ['-c', '--config']); + const rawPaths = getArgValues(argv, ['-c', '--config']).filter((path) => { + return path.endsWith('.yml'); + }); + if (rawPaths.length) { return rawPaths.map((path) => resolve(process.cwd(), path)); } diff --git a/src/platform/packages/private/kbn-apm-config-loader/tsconfig.json b/src/platform/packages/private/kbn-apm-config-loader/tsconfig.json index 08bc1d4c98f2e..670334830ae7f 100644 --- a/src/platform/packages/private/kbn-apm-config-loader/tsconfig.json +++ b/src/platform/packages/private/kbn-apm-config-loader/tsconfig.json @@ -16,6 +16,7 @@ "@kbn/config-schema", "@kbn/std", "@kbn/telemetry-config", + "@kbn/repo-info", ], "exclude": [ "target/**/*", diff --git a/src/platform/packages/private/kbn-journeys/journey/journey_apm_config.ts b/src/platform/packages/private/kbn-journeys/journey/journey_apm_config.ts index 5e970f381e0b1..42a2fbe886b5d 100644 --- a/src/platform/packages/private/kbn-journeys/journey/journey_apm_config.ts +++ b/src/platform/packages/private/kbn-journeys/journey/journey_apm_config.ts @@ -7,17 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could -const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443'; -const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB'; - export const JOURNEY_APM_CONFIG = { - serverUrl: APM_SERVER_URL, - secretToken: APM_PUBLIC_TOKEN, - active: 'true', - contextPropagationOnly: 'false', - environment: process.env.ELASTIC_APM_ENVIRONMENT || (process.env.CI ? 'ci' : 'development'), - transactionSampleRate: '1.0', // capture request body for both errors and request transactions // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body captureBody: 'all', diff --git a/src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts b/src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts index c6042daea5acd..4fe61c197e2fa 100644 --- a/src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts +++ b/src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts @@ -95,12 +95,6 @@ export function makeFtrConfigProvider( ], env: { - ELASTIC_APM_ACTIVE: JOURNEY_APM_CONFIG.active, - ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: JOURNEY_APM_CONFIG.contextPropagationOnly, - ELASTIC_APM_ENVIRONMENT: JOURNEY_APM_CONFIG.environment, - ELASTIC_APM_TRANSACTION_SAMPLE_RATE: JOURNEY_APM_CONFIG.transactionSampleRate, - ELASTIC_APM_SERVER_URL: JOURNEY_APM_CONFIG.serverUrl, - ELASTIC_APM_SECRET_TOKEN: JOURNEY_APM_CONFIG.secretToken, ELASTIC_APM_CAPTURE_BODY: JOURNEY_APM_CONFIG.captureBody, ELASTIC_APM_CAPTURE_HEADERS: JOURNEY_APM_CONFIG.captureRequestHeaders, ELASTIC_APM_LONG_FIELD_MAX_LENGTH: JOURNEY_APM_CONFIG.longFieldMaxLength, diff --git a/src/platform/packages/private/kbn-journeys/journey/journey_ftr_harness.ts b/src/platform/packages/private/kbn-journeys/journey/journey_ftr_harness.ts index 42a3d78029c9b..537b441cb1d9c 100644 --- a/src/platform/packages/private/kbn-journeys/journey/journey_ftr_harness.ts +++ b/src/platform/packages/private/kbn-journeys/journey/journey_ftr_harness.ts @@ -8,7 +8,7 @@ */ import Url from 'url'; -import { inspect, format } from 'util'; +import { inspect } from 'util'; import { setTimeout as setTimer } from 'timers/promises'; import * as Rx from 'rxjs'; import apmNode from 'elastic-apm-node'; @@ -117,35 +117,7 @@ export class JourneyFtrHarness { // Update labels before start for consistency b/w APM services await this.updateTelemetryAndAPMLabels(journeyLabels); - this.apm = apmNode.start({ - serviceName: 'functional test runner', - environment: process.env.CI ? 'ci' : 'development', - active: kbnTestServerEnv.ELASTIC_APM_ACTIVE !== 'false', - serverUrl: kbnTestServerEnv.ELASTIC_APM_SERVER_URL, - secretToken: kbnTestServerEnv.ELASTIC_APM_SECRET_TOKEN, - globalLabels: kbnTestServerEnv.ELASTIC_APM_GLOBAL_LABELS, - transactionSampleRate: kbnTestServerEnv.ELASTIC_APM_TRANSACTION_SAMPLE_RATE, - logger: { - warn: (...args: any[]) => { - this.log.warning('APM WARN', ...args); - }, - info: (...args: any[]) => { - this.log.info('APM INFO', ...args); - }, - fatal: (...args: any[]) => { - this.log.error(format('APM FATAL', ...args)); - }, - error: (...args: any[]) => { - this.log.error(format('APM ERROR', ...args)); - }, - debug: (...args: any[]) => { - this.log.debug('APM DEBUG', ...args); - }, - trace: (...args: any[]) => { - this.log.verbose('APM TRACE', ...args); - }, - }, - }); + this.apm = apmNode; if (this.currentTransaction) { throw new Error(`Transaction exist, end prev transaction ${this.currentTransaction?.name}`); diff --git a/src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts b/src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts index f95022ed90e4f..7beae704695cd 100644 --- a/src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts +++ b/src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts @@ -14,6 +14,7 @@ import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { REPO_ROOT } from '@kbn/repo-info'; import { KbnClient } from '@kbn/test'; +import { withSpan } from '@kbn/apm-utils'; import type { LoadActionPerfOptions } from './lib'; import { saveAction, @@ -97,16 +98,18 @@ export class EsArchiver { performance?: LoadActionPerfOptions; } = {} ) { - return await loadAction({ - inputDir: this.findArchive(path), - skipExisting: !!skipExisting, - useCreate: !!useCreate, - docsOnly, - client: this.client, - log: this.log, - kbnClient: this.kbnClient, - performance, - }); + return await withSpan('es_archiver load', () => + loadAction({ + inputDir: this.findArchive(path), + skipExisting: !!skipExisting, + useCreate: !!useCreate, + docsOnly, + client: this.client, + log: this.log, + kbnClient: this.kbnClient, + performance, + }) + ); } /** @@ -115,12 +118,14 @@ export class EsArchiver { * @param {String} path - relative path to the archive to unload, resolved relative to this.baseDir which defaults to REPO_ROOT */ async unload(path: string) { - return await unloadAction({ - inputDir: this.findArchive(path), - client: this.client, - log: this.log, - kbnClient: this.kbnClient, - }); + return await withSpan('es_archiver unload', () => + unloadAction({ + inputDir: this.findArchive(path), + client: this.client, + log: this.log, + kbnClient: this.kbnClient, + }) + ); } /** @@ -164,10 +169,12 @@ export class EsArchiver { * Cleanup saved object indices, preserving the space:default saved object. */ async emptyKibanaIndex() { - return await emptyKibanaIndexAction({ - client: this.client, - log: this.log, - }); + return await withSpan('es_archiver empty_kibana_index', () => + emptyKibanaIndexAction({ + client: this.client, + log: this.log, + }) + ); } /** diff --git a/src/platform/packages/shared/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts b/src/platform/packages/shared/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts index b1d2849851262..c46bcd41d4bd9 100644 --- a/src/platform/packages/shared/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts +++ b/src/platform/packages/shared/kbn-es-archiver/src/lib/docs/index_doc_records_stream.ts @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import agent from 'elastic-apm-node'; import type { Client } from '@elastic/elasticsearch'; import AggregateError from 'aggregate-error'; import { Writable } from 'stream'; @@ -31,7 +32,7 @@ export function createIndexDocRecordsStream( const ops = new WeakMap(); const errors: string[] = []; - await client.helpers.bulk( + const bulkStats = await client.helpers.bulk( { retries: 5, concurrency: performance?.concurrency || DEFAULT_PERFORMANCE_OPTIONS.concurrency, @@ -61,6 +62,8 @@ export function createIndexDocRecordsStream( } ); + agent.currentSpan?.setLabel('es_archiver_bulk_stats', JSON.stringify(bulkStats)); + if (errors.length) { throw new AggregateError(errors); } diff --git a/src/platform/packages/shared/kbn-es-archiver/tsconfig.json b/src/platform/packages/shared/kbn-es-archiver/tsconfig.json index 12112f3dce111..b936559cce9a3 100644 --- a/src/platform/packages/shared/kbn-es-archiver/tsconfig.json +++ b/src/platform/packages/shared/kbn-es-archiver/tsconfig.json @@ -21,6 +21,7 @@ "@kbn/dev-cli-errors", "@kbn/repo-info", "@kbn/jest-serializers", + "@kbn/apm-utils", ], "exclude": [ "target/**/*", diff --git a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts index 7eb7dd3328e67..617ebcfb95b74 100644 --- a/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts +++ b/src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/run_tests.ts @@ -8,6 +8,7 @@ */ import * as Rx from 'rxjs'; +import agent from 'elastic-apm-node'; import { Lifecycle } from '../lifecycle'; import { Mocha } from '../../fake_mocha_types'; @@ -26,6 +27,10 @@ export async function runTests(lifecycle: Lifecycle, mocha: Mocha, abortSignal?: runComplete = true; }); + lifecycle.cleanup.add(async () => { + await agent.flush().catch((error) => {}); + }); + Rx.race( lifecycle.cleanup.before$, abortSignal ? Rx.fromEvent(abortSignal, 'abort').pipe(Rx.take(1)) : Rx.NEVER diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/es_archiver.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/es_archiver.ts new file mode 100644 index 0000000000000..c359d8d8002e3 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/es_archiver.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const logger = getService('log'); + + describe('@ess @serverless @serverlessQA ES_ARCHIVER TESTS', () => { + describe('reloading archives before each test', () => { + beforeEach(async () => { + await esArchiver.load( + 'x-pack/solutions/security/test/fixtures/es_archives/security_solution/timestamp_override_3' + ); + }); + + afterEach(async () => { + await esArchiver.unload( + 'x-pack/solutions/security/test/fixtures/es_archives/security_solution/timestamp_override_3' + ); + }); + + describe('assertions on inserted documents', () => { + Array(100) + .fill(0) + .forEach((_, index) => { + it(`finds one document from the timestamp_override_3 archive (run ${index})`, async () => { + const searchResponse = await es.search({ + index: 'myfakeindex-3', + }); + + // eslint-disable-next-line no-console + console.log('searchResponse', JSON.stringify(searchResponse, null, 2)); + + expect(searchResponse.hits.hits).toEqual([ + expect.objectContaining({ + _index: 'myfakeindex-3', + _source: { + message: 'hello world 3', + '@timestamp': '2020-12-16T15:16:18.570Z', + }, + }), + ]); + }); + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/index.ts index 5dd370394831f..a92ceb1dc4f9d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/basic_license_essentials_tier/index.ts @@ -9,11 +9,12 @@ import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Rule execution logic API - Basic License/Essentials Tier', function () { - loadTestFile(require.resolve('./ecs_field_duplication')); - loadTestFile(require.resolve('./keyword_family')); - loadTestFile(require.resolve('./ignore_fields')); - loadTestFile(require.resolve('./runtime')); - loadTestFile(require.resolve('./non_ecs_fields')); - loadTestFile(require.resolve('./timestamps')); + loadTestFile(require.resolve('./es_archiver')); + // loadTestFile(require.resolve('./ecs_field_duplication')); + // loadTestFile(require.resolve('./keyword_family')); + // loadTestFile(require.resolve('./ignore_fields')); + // loadTestFile(require.resolve('./runtime')); + // loadTestFile(require.resolve('./non_ecs_fields')); + // loadTestFile(require.resolve('./timestamps')); }); }