From 2765399daeacf704328e47772390058beed9df3d Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Fri, 9 Aug 2024 12:25:07 -0700 Subject: [PATCH] Changes for adding benchmark compare feature (#4931) Signed-off-by: Rishabh Singh --- .../opensearch/benchmark-compare.jenkinsfile | 102 ++++++++++++++++ .../benchmark-pull-request.jenkinsfile | 53 ++++++--- scripts/benchmark/benchmark-pull-request.sh | 2 +- tests/jenkins/TestBenchmarkPullRequest.groovy | 10 +- tests/jenkins/TestCompareBenchmarks.groovy | 111 ++++++++++++++++++ .../benchmark-compare.jenkinsfile.txt | 41 +++++++ ...hmark-pull-request-generic.jenkinsfile.txt | 102 ++++++++++++++-- ...enchmark-pull-request-user.jenkinsfile.txt | 20 ++-- .../compare-benchmarks.jenkinsfile.txt | 41 +++++++ 9 files changed, 445 insertions(+), 37 deletions(-) create mode 100644 jenkins/opensearch/benchmark-compare.jenkinsfile create mode 100644 tests/jenkins/TestCompareBenchmarks.groovy create mode 100644 tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-compare.jenkinsfile.txt create mode 100644 tests/jenkins/jenkinsjob-regression-files/opensearch/compare-benchmarks.jenkinsfile.txt diff --git a/jenkins/opensearch/benchmark-compare.jenkinsfile b/jenkins/opensearch/benchmark-compare.jenkinsfile new file mode 100644 index 0000000000..ba74658792 --- /dev/null +++ b/jenkins/opensearch/benchmark-compare.jenkinsfile @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +lib = library(identifier: 'jenkins@6.8.2', retriever: modernSCM([ + + $class: 'GitSCMSource', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', +])) + +pipeline { + agent { label 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' } + options { + timeout(time: 1, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) + throttleJobProperty( + categories: [], + limitOneJobWithMatchingParams: false, + maxConcurrentPerNode: 0, + maxConcurrentTotal: 20, + paramsToUseForLimit: '', + throttleEnabled: true, + throttleOption: 'project', + ) + } + environment { + JOB_NAME = 'benchmark-compare' + } + parameters { + string( + name: 'BASELINE_TEST_EXECUTION_ID', + description: 'opensearch-benchmark test-execution-id for baseline run, sample format: `61a34b4e-4ae4-40b7-9d58-6570b9166f6a`', + trim: true + ) + string( + name: 'CONTENDER_TEST_EXECUTION_ID', + description: 'opensearch-benchmark test-execution-id of candidate for comparison, mainly from pull requests.', + trim: true + ) + string( + name: 'PULL_REQUEST_NUMBER', + trim: true + ) + string( + name: 'REPOSITORY', + defaultValue: 'opensearch-project/OpenSearch' + ) + } + stages { + stage('validate-and-set-parameters') { + steps { + script { + if (BASELINE_TEST_EXECUTION_ID == '' || CONTENDER_TEST_EXECUTION_ID == '') { + currentBuild.result = 'ABORTED' + error('Benchmark test failed to start. Please provide BASELINE_TEST_EXECUTION_ID and CONTENDER_TEST_EXECUTION_ID to run the job.') + } + } + } + } + stage('benchmark-pull-request') { + steps { + script { + runBenchmarkTestScript( + command: 'compare', + baseline: "${BASELINE_TEST_EXECUTION_ID}", + contender: "${CONTENDER_TEST_EXECUTION_ID}", + suffix: "${BUILD_NUMBER}" + ) + } + } + post { + success { + sh """ + echo '
Benchmark Baseline Comparison Results' > temp_result_${BUILD_NUMBER}.md + echo '' >> temp_result_${BUILD_NUMBER}.md + echo '#### Benchmark Results for Job: ${BUILD_URL}' >> temp_result_${BUILD_NUMBER}.md + cat final_result_${BUILD_NUMBER}.md >> temp_result_${BUILD_NUMBER}.md + echo '
' >> temp_result_${BUILD_NUMBER}.md + mv temp_result_${BUILD_NUMBER}.md final_result_${BUILD_NUMBER}.md + """ + script { + if ("${PULL_REQUEST_NUMBER}" != '' && "${REPOSITORY}" != '') { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def pull_request = Integer.parseInt("${PULL_REQUEST_NUMBER}") + sh ("gh pr comment ${pull_request} --repo ${REPOSITORY} --body-file final_result_${BUILD_NUMBER}.md") + } + } + } + postCleanup() + } + failure { + postCleanup() + } + } + } + } +} diff --git a/jenkins/opensearch/benchmark-pull-request.jenkinsfile b/jenkins/opensearch/benchmark-pull-request.jenkinsfile index 0872e61b02..1c9a6e1348 100644 --- a/jenkins/opensearch/benchmark-pull-request.jenkinsfile +++ b/jenkins/opensearch/benchmark-pull-request.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@6.8.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.8.2', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', @@ -158,6 +158,7 @@ pipeline { genericVariables: [ [key: 'pull_request_number', value: '$.pull_request_number'], [key: 'repository', value: '$.repository'], + [key: 'baseline_cluster_config', value: '$.baseline_cluster_config'], [key: 'DISTRIBUTION_URL', value: '$.DISTRIBUTION_URL'], [key: 'DISTRIBUTION_VERSION', value: '$.DISTRIBUTION_VERSION'], [key: 'TEST_WORKLOAD', value: '$.TEST_WORKLOAD'], @@ -190,14 +191,14 @@ pipeline { ) parameterizedCron ''' - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true @@ -278,9 +279,29 @@ pipeline { """ script { if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { - withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { - def pull_request = Integer.parseInt("${pull_request_number}") - sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md") + if ("${pull_request_number}" != '' && "${repository}" != '') { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def pull_request = Integer.parseInt("${pull_request_number}") + sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md") + } + if ("${baseline_cluster_config}" != '') { + Map benchmarkTestIds = getCompareBenchmarkIds( + baselineClusterConfig: "${baseline_cluster_config}", + distributionVersion: "${DISTRIBUTION_VERSION}-SNAPSHOT", + workload: "${TEST_WORKLOAD}", + pullRequestNumber: "${pull_request_number}" + ) + // Trigger benchmark-compare job + build job: 'benchmark-compare', + parameters: [ + string(name:'BASELINE_TEST_EXECUTION_ID', value: "${benchmarkTestIds.baseline}"), + string(name:'BASELINE_TEST_EXECUTION_ID', value: "${benchmarkTestIds.contender}"), + string(name:'PULL_REQUEST_NUMBER', value: "${pull_request_number}"), + string(name:'REPOSITORY', value: "${repository}") + ], + wait: false, + propagate: false + } } } } @@ -290,11 +311,14 @@ pipeline { failure { script { if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { - withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { - def pull_request = Integer.parseInt("${pull_request_number}") - sh ("gh pr comment ${pull_request} --repo ${repository} --body \"The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.\"") + if ("${pull_request_number}" != '' && "${repository}" != '') { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { + def pull_request = Integer.parseInt("${pull_request_number}") + sh ("gh pr comment ${pull_request} --repo ${repository} --body \"The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.\"") + } } } + postCleanup() } } @@ -325,7 +349,6 @@ pipeline { } } postCleanup() - } } } diff --git a/scripts/benchmark/benchmark-pull-request.sh b/scripts/benchmark/benchmark-pull-request.sh index af864bb1dc..d4aef04777 100755 --- a/scripts/benchmark/benchmark-pull-request.sh +++ b/scripts/benchmark/benchmark-pull-request.sh @@ -14,7 +14,7 @@ JENKINS_URL="https://build.ci.opensearch.org" TRIGGER_TOKEN=$1 -PAYLOAD_JSON="{\"pull_request_number\": \"$PR_NUMBER\", \"repository\": \"$REPOSITORY\", \"DISTRIBUTION_URL\": \"$DISTRIBUTION_URL\", \"DISTRIBUTION_VERSION\": \"$OPENSEARCH_VERSION\", \"SECURITY_ENABLED\": \"$SECURITY_ENABLED\", \"SINGLE_NODE_CLUSTER\": \"$SINGLE_NODE_CLUSTER\", \"MIN_DISTRIBUTION\": \"$MIN_DISTRIBUTION\", \"TEST_WORKLOAD\": \"$TEST_WORKLOAD\", \"MANAGER_NODE_COUNT\": \"$MANAGER_NODE_COUNT\", \"DATA_NODE_COUNT\": \"$DATA_NODE_COUNT\", \"DATA_INSTANCE_TYPE\": \"$DATA_INSTANCE_TYPE\", \"DATA_NODE_STORAGE\": \"$DATA_NODE_STORAGE\", \"JVM_SYS_PROPS\": \"$JVM_SYS_PROPS\", \"ADDITIONAL_CONFIG\": \"$ADDITIONAL_CONFIG\", \"USER_TAGS\": \"$USER_TAGS\", \"WORKLOAD_PARAMS\": $WORKLOAD_PARAMS, \"TEST_PROCEDURE\": \"$TEST_PROCEDURE\", \"EXCLUDE_TASKS\": \"$EXCLUDE_TASKS\", \"INCLUDE_TASKS\": \"$INCLUDE_TASKS\", \"CAPTURE_NODE_STAT\": \"$CAPTURE_NODE_STAT\"}" +PAYLOAD_JSON="{\"pull_request_number\": \"$PR_NUMBER\", \"repository\": \"$REPOSITORY\", \"baseline_cluster_config\": \"$BASELINE_CLUSTER_CONFIG\", \"DISTRIBUTION_URL\": \"$DISTRIBUTION_URL\", \"DISTRIBUTION_VERSION\": \"$OPENSEARCH_VERSION\", \"SECURITY_ENABLED\": \"$SECURITY_ENABLED\", \"SINGLE_NODE_CLUSTER\": \"$SINGLE_NODE_CLUSTER\", \"MIN_DISTRIBUTION\": \"$MIN_DISTRIBUTION\", \"TEST_WORKLOAD\": \"$TEST_WORKLOAD\", \"MANAGER_NODE_COUNT\": \"$MANAGER_NODE_COUNT\", \"DATA_NODE_COUNT\": \"$DATA_NODE_COUNT\", \"DATA_INSTANCE_TYPE\": \"$DATA_INSTANCE_TYPE\", \"DATA_NODE_STORAGE\": \"$DATA_NODE_STORAGE\", \"JVM_SYS_PROPS\": \"$JVM_SYS_PROPS\", \"ADDITIONAL_CONFIG\": \"$ADDITIONAL_CONFIG\", \"USER_TAGS\": \"$USER_TAGS\", \"WORKLOAD_PARAMS\": $WORKLOAD_PARAMS, \"TEST_PROCEDURE\": \"$TEST_PROCEDURE\", \"EXCLUDE_TASKS\": \"$EXCLUDE_TASKS\", \"INCLUDE_TASKS\": \"$INCLUDE_TASKS\", \"CAPTURE_NODE_STAT\": \"$CAPTURE_NODE_STAT\"}" echo "Trigger Jenkins workflows" JENKINS_REQ=`curl -s -XPOST \ -H "Authorization: Bearer $TRIGGER_TOKEN" \ diff --git a/tests/jenkins/TestBenchmarkPullRequest.groovy b/tests/jenkins/TestBenchmarkPullRequest.groovy index 91b71c05c6..a866c1e6bd 100644 --- a/tests/jenkins/TestBenchmarkPullRequest.groovy +++ b/tests/jenkins/TestBenchmarkPullRequest.groovy @@ -28,7 +28,7 @@ class TestBenchmarkPullRequest extends BuildPipelineTest { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('6.8.0') + .defaultVersion('6.8.2') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -104,15 +104,22 @@ class TestBenchmarkPullRequest extends BuildPipelineTest { binding.setVariable('TELEMETRY_PARAMS', '') binding.setVariable('pull_request_number', '1234') binding.setVariable('pull_request', 1234) + binding.setVariable('baseline_cluster_config', 'test-cluster-config') binding.setVariable('repository','opensearch-project/OpenSearch') helper.registerAllowedMethod("GenericTrigger", [Map], { println 'GenericTrigger called with params: ' + it }) + helper.registerAllowedMethod("sh", [Map.class], { map -> + return '{"hits":{"total":{"value":1},"hits":[{"_source":{"test-execution-id":"test-id"}}]}}' + }) } @Test public void testBenchmarkPullRequestGenericCause_verifyPipeline() { binding.getVariable('currentBuild').rawBuild = [:] binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" } + helper.registerAllowedMethod('getCompareBenchmarkIds', [Map.class], { params -> + return [baseline: "mockBaseline", contender: "mockContender"] + }) super.testPipeline("jenkins/opensearch/benchmark-pull-request.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile") @@ -147,6 +154,7 @@ class TestBenchmarkPullRequest extends BuildPipelineTest { } assertThat(testGhCliCommand.size(), equalTo(1)) assertThat(testGhCliCommand, hasItem('gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md')) + assertCallStack().contains(" benchmark-pull-request.getCompareBenchmarkIds({baselineClusterConfig=test-cluster-config, distributionVersion=3.0.0-SNAPSHOT, workload=nyc-taxis, pullRequestNumber=1234})") } @Test diff --git a/tests/jenkins/TestCompareBenchmarks.groovy b/tests/jenkins/TestCompareBenchmarks.groovy new file mode 100644 index 0000000000..d19bcd873a --- /dev/null +++ b/tests/jenkins/TestCompareBenchmarks.groovy @@ -0,0 +1,111 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource + +class TestCompareBenchmarks extends BuildPipelineTest { + @Override + @Before + void setUp() { + super.setUp() + + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('6.8.2') + .allowOverride(true) + .implicit(true) + .targetPath('vars') + .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) + .build() + ) + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + helper.registerAllowedMethod("downloadBuildManifest", [Map], { + c -> lib.jenkins.BuildManifest.new(readYaml(file: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml')) + }) + helper.registerAllowedMethod("throttleJobProperty", [Map]) + + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('env', ['BUILD_NUMBER': '307']) + binding.setVariable('BUILD_URL', 'test://artifact.url') + binding.setVariable('BUILD_NUMBER', '307') + binding.setVariable('JOB_NAME', 'compare-benchmarks') + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('PULL_REQUEST_NUMBER', '1234') + binding.setVariable('pull_request', 1234) + binding.setVariable('REPOSITORY','opensearch-project/OpenSearch') + binding.setVariable('BASELINE_TEST_EXECUTION_ID', 'baseline-id') + binding.setVariable('CONTENDER_TEST_EXECUTION_ID', 'contender-id') + } + + @Test + public void testCompareBenchmark_verifyPipeline() { + + super.testPipeline("jenkins/opensearch/benchmark-compare.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-compare.jenkinsfile") + } + + @Test + void testCompareBenchmark_verifyScriptExecutions() { + runScript("jenkins/opensearch/benchmark-compare.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems("set +x && ./test.sh benchmark-test compare baseline-id contender-id --benchmark-config /tmp/workspace/benchmark.ini --suffix 307")) + + def testGhCliCommand = getCommandExecutions('sh', 'gh').findAll { + shCommand -> shCommand.contains('gh') + } + assertThat(testGhCliCommand.size(), equalTo(1)) + assertThat(testGhCliCommand, hasItem('gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md')) + assertCallStack().contains("benchmark-compare.runBenchmarkTestScript({command=compare, baseline=baseline-id, contender=contender-id, suffix=307})") + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } + +} diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-compare.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-compare.jenkinsfile.txt new file mode 100644 index 0000000000..7ddc8975f1 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-compare.jenkinsfile.txt @@ -0,0 +1,41 @@ + benchmark-compare.run() + benchmark-compare.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-compare.library({identifier=jenkins@6.8.2, retriever=null}) + benchmark-compare.pipeline(groovy.lang.Closure) + benchmark-compare.timeout({time=1, unit=HOURS}) + benchmark-compare.logRotator({daysToKeepStr=30}) + benchmark-compare.buildDiscarder(null) + benchmark-compare.throttleJobProperty({categories=[], limitOneJobWithMatchingParams=false, maxConcurrentPerNode=0, maxConcurrentTotal=20, paramsToUseForLimit=, throttleEnabled=true, throttleOption=project}) + benchmark-compare.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) + benchmark-compare.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-compare.script(groovy.lang.Closure) + benchmark-compare.stage(benchmark-pull-request, groovy.lang.Closure) + benchmark-compare.script(groovy.lang.Closure) + benchmark-compare.runBenchmarkTestScript({command=compare, baseline=baseline-id, contender=contender-id, suffix=307}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.8.2, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare baseline-id contender-id --benchmark-config /tmp/workspace/benchmark.ini --suffix 307) + benchmark-compare.sh( + echo '
Benchmark Baseline Comparison Results' > temp_result_307.md + echo '' >> temp_result_307.md + echo '#### Benchmark Results for Job: test://artifact.url' >> temp_result_307.md + cat final_result_307.md >> temp_result_307.md + echo '
' >> temp_result_307.md + mv temp_result_307.md final_result_307.md + ) + benchmark-compare.script(groovy.lang.Closure) + benchmark-compare.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + benchmark-compare.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + benchmark-compare.sh(gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md) + benchmark-compare.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt index cdf3f311ba..45c6d78108 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-generic.jenkinsfile.txt @@ -1,22 +1,22 @@ benchmark-pull-request.run() benchmark-pull-request.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-pull-request.library({identifier=jenkins@6.8.0, retriever=null}) + benchmark-pull-request.library({identifier=jenkins@6.8.2, retriever=null}) benchmark-pull-request.pipeline(groovy.lang.Closure) benchmark-pull-request.timeout({time=24, unit=HOURS}) benchmark-pull-request.logRotator({daysToKeepStr=30}) benchmark-pull-request.buildDiscarder(null) benchmark-pull-request.throttleJobProperty({categories=[], limitOneJobWithMatchingParams=false, maxConcurrentPerNode=0, maxConcurrentTotal=20, paramsToUseForLimit=, throttleEnabled=true, throttleOption=project}) benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) - benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) + benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=baseline_cluster_config, value=$.baseline_cluster_config}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) benchmark-pull-request.parameterizedCron( - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true @@ -38,7 +38,7 @@ benchmark-pull-request.echo(security-enabled: false) benchmark-pull-request.runBenchmarkTestScript({command=execute-test, distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=true, minDistribution=true, use50PercentHeap=true, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, userTag=run-type:test,security-enabled:false, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@6.8.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.8.2, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) @@ -64,5 +64,87 @@ benchmark-pull-request.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) benchmark-pull-request.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) benchmark-pull-request.sh(gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md) + benchmark-pull-request.getCompareBenchmarkIds({baselineClusterConfig=test-cluster-config, distributionVersion=3.0.0-SNAPSHOT, workload=nyc-taxis, pullRequestNumber=1234}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-nlb-endpoint, variable=DATASTORE_ENDPOINT}) + getCompareBenchmarkIds.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD, DATASTORE_ENDPOINT], groovy.lang.Closure) + getCompareBenchmarkIds.sh({script= + curl -X POST "https://DATASTORE_ENDPOINT/benchmark-results-*/_search" -ku DATASTORE_USER:DATASTORE_PASSWORD -H 'Content-Type: application/json' -d '{ + "size": 1, + "query": { + "bool": { + "must": [ + { + "term": { + "user-tags.cluster-config": "test-cluster-config" + } + }, + { + "term": { + "workload": "nyc-taxis" + } + }, + { + "term": { + "distribution-version": "3.0.0-SNAPSHOT" + } + }, + { + "range": { + "test-execution-timestamp": { + "gte": "now-5d/d", + "lte": "now/d" + } + } + } + ] + } + }, + "sort": [ + { + "test-execution-timestamp": { + "order": "desc" + } + } + ], + "_source": ["test-execution-id"] + }' + , returnStdout=true}) + getCompareBenchmarkIds.echo(Latest test-execution-id: test-id) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-nlb-endpoint, variable=DATASTORE_ENDPOINT}) + getCompareBenchmarkIds.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD, DATASTORE_ENDPOINT], groovy.lang.Closure) + getCompareBenchmarkIds.sh({script= + curl -X POST "https://DATASTORE_ENDPOINT/benchmark-results-*/_search" -ku DATASTORE_USER:DATASTORE_PASSWORD -H 'Content-Type: application/json' -d '{ + "size": 1, + "query": { + "bool": { + "must": [ + { + "term": { + "user-tags.pull_request_number": "1234" + } + } + ] + } + }, + "sort": [ + { + "test-execution-timestamp": { + "order": "desc" + } + } + ], + "_source": ["test-execution-id"] + }' + , returnStdout=true}) + getCompareBenchmarkIds.echo(Latest test-execution-id: test-id) + benchmark-pull-request.string({name=BASELINE_TEST_EXECUTION_ID, value=test-id}) + benchmark-pull-request.string({name=BASELINE_TEST_EXECUTION_ID, value=test-id}) + benchmark-pull-request.string({name=PULL_REQUEST_NUMBER, value=1234}) + benchmark-pull-request.string({name=REPOSITORY, value=opensearch-project/OpenSearch}) + benchmark-pull-request.build({job=benchmark-compare, parameters=[null, null, null, null], wait=false, propagate=false}) benchmark-pull-request.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt index a4ee5de678..3d556c971c 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-pull-request-user.jenkinsfile.txt @@ -1,22 +1,22 @@ benchmark-pull-request.run() benchmark-pull-request.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-pull-request.library({identifier=jenkins@6.8.0, retriever=null}) + benchmark-pull-request.library({identifier=jenkins@6.8.2, retriever=null}) benchmark-pull-request.pipeline(groovy.lang.Closure) benchmark-pull-request.timeout({time=24, unit=HOURS}) benchmark-pull-request.logRotator({daysToKeepStr=30}) benchmark-pull-request.buildDiscarder(null) benchmark-pull-request.throttleJobProperty({categories=[], limitOneJobWithMatchingParams=false, maxConcurrentPerNode=0, maxConcurrentTotal=20, paramsToUseForLimit=, throttleEnabled=true, throttleOption=project}) benchmark-pull-request.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) - benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) + benchmark-pull-request.GenericTrigger({genericVariables=[{key=pull_request_number, value=$.pull_request_number}, {key=repository, value=$.repository}, {key=baseline_cluster_config, value=$.baseline_cluster_config}, {key=DISTRIBUTION_URL, value=$.DISTRIBUTION_URL}, {key=DISTRIBUTION_VERSION, value=$.DISTRIBUTION_VERSION}, {key=TEST_WORKLOAD, value=$.TEST_WORKLOAD}, {key=SECURITY_ENABLED, value=$.SECURITY_ENABLED}, {key=SINGLE_NODE_CLUSTER, value=$.SINGLE_NODE_CLUSTER}, {key=MIN_DISTRIBUTION, value=$.MIN_DISTRIBUTION}, {key=MANAGER_NODE_COUNT, value=$.MANAGER_NODE_COUNT}, {key=DATA_NODE_COUNT, value=$.DATA_NODE_COUNT}, {key=CLIENT_NODE_COUNT, value=$.CLIENT_NODE_COUNT}, {key=INGEST_NODE_COUNT, value=$.INGEST_NODE_COUNT}, {key=ML_NODE_COUNT, value=$.ML_NODE_COUNT}, {key=DATA_INSTANCE_TYPE, value=$.DATA_INSTANCE_TYPE}, {key=DATA_NODE_STORAGE, value=$.DATA_NODE_STORAGE}, {key=ML_NODE_STORAGE, value=$.ML_NODE_STORAGE}, {key=JVM_SYS_PROPS, value=$.JVM_SYS_PROPS}, {key=ADDITIONAL_CONFIG, value=$.ADDITIONAL_CONFIG}, {key=USE_50_PERCENT_HEAP, value=$.USE_50_PERCENT_HEAP}, {key=USER_TAGS, value=$.USER_TAGS}, {key=WORKLOAD_PARAMS, value=$.WORKLOAD_PARAMS}, {key=TEST_PROCEDURE, value=$.TEST_PROCEDURE}, {key=EXCLUDE_TASKS, value=$.EXCLUDE_TASKS}, {key=INCLUDE_TASKS, value=$.INCLUDE_TASKS}, {key=CAPTURE_NODE_STAT, value=$.CAPTURE_NODE_STAT}, {key=TELEMETRY_PARAMS, value=$.TELEMETRY_PARAMS}], tokenCredentialId=jenkins-pr-benchmark-generic-webhook-token, causeString=Triggered by comment on PR on OpenSearch core repository, printContributedVariables=true, printPostContent=true}) benchmark-pull-request.parameterizedCron( - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true + H 4 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true - H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.16.0-SNAPSHOT/opensearch-min-2.16.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.16.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};EXCLUDE_TASKS=type:search;MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true + H 5 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.17.0-SNAPSHOT/opensearch-min-2.17.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=2.17.0;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;USER_TAGS=cluster-config:x64-r5.xlarge-single-node-1-shard-0-replica-baseline;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true H 6 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-linux-x64-latest.tar.gz;DISTRIBUTION_VERSION=3.0.0;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,cluster-config:x64-r5.xlarge-1-shard-0-replica-snapshot-baseline;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-300","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-300","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true;MIN_DISTRIBUTION=true @@ -38,7 +38,7 @@ benchmark-pull-request.echo(security-enabled: false) benchmark-pull-request.runBenchmarkTestScript({command=execute-test, distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=true, minDistribution=true, use50PercentHeap=true, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, userTag=run-type:test,security-enabled:false, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@6.8.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.8.2, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/compare-benchmarks.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/compare-benchmarks.jenkinsfile.txt new file mode 100644 index 0000000000..7cef878c02 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/compare-benchmarks.jenkinsfile.txt @@ -0,0 +1,41 @@ + compare-benchmarks.run() + compare-benchmarks.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + compare-benchmarks.library({identifier=jenkins@6.8.2, retriever=null}) + compare-benchmarks.pipeline(groovy.lang.Closure) + compare-benchmarks.timeout({time=1, unit=HOURS}) + compare-benchmarks.logRotator({daysToKeepStr=30}) + compare-benchmarks.buildDiscarder(null) + compare-benchmarks.throttleJobProperty({categories=[], limitOneJobWithMatchingParams=false, maxConcurrentPerNode=0, maxConcurrentTotal=20, paramsToUseForLimit=, throttleEnabled=true, throttleOption=project}) + compare-benchmarks.echo(Executing on agent [label:Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test]) + compare-benchmarks.stage(validate-and-set-parameters, groovy.lang.Closure) + compare-benchmarks.script(groovy.lang.Closure) + compare-benchmarks.stage(benchmark-pull-request, groovy.lang.Closure) + compare-benchmarks.script(groovy.lang.Closure) + compare-benchmarks.runBenchmarkTestScript({command=compare, baseline=baseline-id, contender=contender-id, suffix=307}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.8.2, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare baseline-id contender-id --benchmark-config /tmp/workspace/benchmark.ini --suffix 307) + compare-benchmarks.sh( + echo '
Benchmark Baseline Comparison Results' > temp_result_307.md + echo '' >> temp_result_307.md + echo '#### Benchmark Results for Job: test://artifact.url' >> temp_result_307.md + cat final_result_307.md >> temp_result_307.md + echo '
' >> temp_result_307.md + mv temp_result_307.md final_result_307.md + ) + compare-benchmarks.script(groovy.lang.Closure) + compare-benchmarks.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + compare-benchmarks.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + compare-benchmarks.sh(gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md) + compare-benchmarks.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})