From 6a72a1e897064defc2c77b87f6fdbfc144fdbf79 Mon Sep 17 00:00:00 2001 From: lizozom Date: Tue, 29 Nov 2022 19:54:53 +0200 Subject: [PATCH 1/5] Start ES once --- src/dev/performance/run_performance_cli.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dev/performance/run_performance_cli.ts b/src/dev/performance/run_performance_cli.ts index fea2b27cc2616..ae7df8105ac08 100644 --- a/src/dev/performance/run_performance_cli.ts +++ b/src/dev/performance/run_performance_cli.ts @@ -84,19 +84,20 @@ run( log.info(`Found ${journeys.length} journeys to run`); const failedJourneys = []; + await startEs(); for (const journey of journeys) { try { - await startEs(); await runWarmup(journey); await runTest(journey); - await procRunner.stop('es'); } catch (e) { log.error(e); failedJourneys.push(journey); } } + await procRunner.stop('es'); + if (failedJourneys.length > 0) { throw new Error(`${failedJourneys.length} journeys failed: ${failedJourneys.join(',')}`); } From f37b43470dafba7d638914073416ceca2ee94aaa Mon Sep 17 00:00:00 2001 From: lizozom Date: Wed, 30 Nov 2022 19:19:00 +0200 Subject: [PATCH 2/5] load data only on warmup empty the kibana index on test teardown --- .../journey/journey_ftr_harness.ts | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/kbn-journeys/journey/journey_ftr_harness.ts b/packages/kbn-journeys/journey/journey_ftr_harness.ts index 7b9d8cc0266fc..e210c45c4ab6d 100644 --- a/packages/kbn-journeys/journey/journey_ftr_harness.ts +++ b/packages/kbn-journeys/journey/journey_ftr_harness.ts @@ -117,15 +117,18 @@ export class JourneyFtrHarness { } private async onSetup() { - await Promise.all([ - this.setupBrowserAndPage(), - asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { - await this.esArchiver.load(esArchive); - }), - asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => { - await this.kibanaServer.importExport.load(kbnArchive); - }), - ]); + await this.setupBrowserAndPage(); + + if (process.env.TEST_PERFORMANCE_PHASE === 'WARMUP') { + await Promise.all([ + asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { + await this.esArchiver.load(esArchive); + }), + asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => { + await this.kibanaServer.importExport.load(kbnArchive); + }), + ]); + } // It is important that we start the APM transaction after we open the browser and all the test data is loaded // so that the scalability data extractor can focus on just the APM data produced by Kibana running under test. @@ -189,14 +192,16 @@ export class JourneyFtrHarness { // unloading the test data so that the scalability data extractor can focus on just the APM data produced // by Kibana running under test. await this.teardownApm(); - await Promise.all([ - asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { + + if (process.env.TEST_PERFORMANCE_PHASE === 'TEST') { + // unload data + await asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { await this.esArchiver.unload(esArchive); - }), - asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => { - await this.kibanaServer.importExport.unload(kbnArchive); - }), - ]); + }); + // empty the kibana index + await this.esArchiver.emptyKibanaIndex(); + // TODO: delete the kibana index \ clean mappings? + } } private async onStepSuccess(step: AnyStep) { From 1281e93c876f51e623deb4b2e4ccc36cdbd5e27b Mon Sep 17 00:00:00 2001 From: lizozom Date: Wed, 30 Nov 2022 11:26:55 +0200 Subject: [PATCH 3/5] disable scalability --- .buildkite/pipelines/performance/daily.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipelines/performance/daily.yml b/.buildkite/pipelines/performance/daily.yml index ea7e406ba63d8..a37dedfd98235 100644 --- a/.buildkite/pipelines/performance/daily.yml +++ b/.buildkite/pipelines/performance/daily.yml @@ -27,11 +27,11 @@ steps: - exit_status: '*' limit: 1 - - label: '🚢 Performance Tests dataset extraction for scalability benchmarking' - command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh - agents: - queue: n2-2 - depends_on: tests + # - label: '🚢 Performance Tests dataset extraction for scalability benchmarking' + # command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh + # agents: + # queue: n2-2 + # depends_on: tests - label: '📈 Report performance metrics to ci-stats' command: .buildkite/scripts/steps/functional/report_performance_metrics.sh From 127a1ef7138c1eb8ec1fb3d1d2c8c0024971fefc Mon Sep 17 00:00:00 2001 From: lizozom Date: Wed, 30 Nov 2022 20:57:45 +0200 Subject: [PATCH 4/5] fix --- packages/kbn-journeys/journey/journey_ftr_harness.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-journeys/journey/journey_ftr_harness.ts b/packages/kbn-journeys/journey/journey_ftr_harness.ts index e210c45c4ab6d..9d0c2ce2f8c33 100644 --- a/packages/kbn-journeys/journey/journey_ftr_harness.ts +++ b/packages/kbn-journeys/journey/journey_ftr_harness.ts @@ -119,7 +119,7 @@ export class JourneyFtrHarness { private async onSetup() { await this.setupBrowserAndPage(); - if (process.env.TEST_PERFORMANCE_PHASE === 'WARMUP') { + if (process.env.TEST_PERFORMANCE_PHASE === 'WARMUP' || process.env.TEST_PERFORMANCE_PHASE === undefined) { await Promise.all([ asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { await this.esArchiver.load(esArchive); @@ -193,7 +193,7 @@ export class JourneyFtrHarness { // by Kibana running under test. await this.teardownApm(); - if (process.env.TEST_PERFORMANCE_PHASE === 'TEST') { + if (process.env.TEST_PERFORMANCE_PHASE === 'TEST' || process.env.TEST_PERFORMANCE_PHASE === undefined) { // unload data await asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { await this.esArchiver.unload(esArchive); From 56c1becd32821839b521e7e0c9b5c5b3d9f48154 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:03:50 +0000 Subject: [PATCH 5/5] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- packages/kbn-journeys/journey/journey_ftr_harness.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/kbn-journeys/journey/journey_ftr_harness.ts b/packages/kbn-journeys/journey/journey_ftr_harness.ts index 9d0c2ce2f8c33..6bf9fea87fcfa 100644 --- a/packages/kbn-journeys/journey/journey_ftr_harness.ts +++ b/packages/kbn-journeys/journey/journey_ftr_harness.ts @@ -119,7 +119,10 @@ export class JourneyFtrHarness { private async onSetup() { await this.setupBrowserAndPage(); - if (process.env.TEST_PERFORMANCE_PHASE === 'WARMUP' || process.env.TEST_PERFORMANCE_PHASE === undefined) { + if ( + process.env.TEST_PERFORMANCE_PHASE === 'WARMUP' || + process.env.TEST_PERFORMANCE_PHASE === undefined + ) { await Promise.all([ asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { await this.esArchiver.load(esArchive); @@ -193,7 +196,10 @@ export class JourneyFtrHarness { // by Kibana running under test. await this.teardownApm(); - if (process.env.TEST_PERFORMANCE_PHASE === 'TEST' || process.env.TEST_PERFORMANCE_PHASE === undefined) { + if ( + process.env.TEST_PERFORMANCE_PHASE === 'TEST' || + process.env.TEST_PERFORMANCE_PHASE === undefined + ) { // unload data await asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { await this.esArchiver.unload(esArchive);