-
Notifications
You must be signed in to change notification settings - Fork 274
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Changes for adding benchmark compare feature
Signed-off-by: Rishabh Singh <[email protected]>
- Loading branch information
1 parent
b6040a3
commit 43e9f7a
Showing
8 changed files
with
383 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,7 +7,7 @@ | |
| * compatible open source license. | ||
| */ | ||
|
|
||
| lib = library(identifier: '[email protected].0', retriever: modernSCM([ | ||
| lib = library(identifier: '[email protected].1', retriever: modernSCM([ | ||
|
|
||
| $class: 'GitSCMSource', | ||
| remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', | ||
|
|
@@ -158,6 +158,7 @@ pipeline { | |
| genericVariables: [ | ||
| [key: 'pull_request_number', value: '$.pull_request_number'], | ||
| [key: 'repository', value: '$.repository'], | ||
| [key: 'baseline_cluster_config', value: '$.baseline_cluster_config'], | ||
| [key: 'DISTRIBUTION_URL', value: '$.DISTRIBUTION_URL'], | ||
| [key: 'DISTRIBUTION_VERSION', value: '$.DISTRIBUTION_VERSION'], | ||
| [key: 'TEST_WORKLOAD', value: '$.TEST_WORKLOAD'], | ||
|
|
@@ -278,9 +279,29 @@ pipeline { | |
| """ | ||
| script { | ||
| if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { | ||
| withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { | ||
| def pull_request = Integer.parseInt("${pull_request_number}") | ||
| sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md") | ||
| if ("${pull_request_number}" != '' && "${repository}" != '') { | ||
| withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { | ||
| def pull_request = Integer.parseInt("${pull_request_number}") | ||
| sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md") | ||
| } | ||
| if ("${baseline_cluster_config}" != '') { | ||
| Map <String, String> benchmarkTestIds = getCompareBenchmarkIds( | ||
| baselineClusterConfig: "${baseline_cluster_config}", | ||
| distributionVersion: "${DISTRIBUTION_VERSION}-SNAPSHOT", | ||
| workload: "${TEST_WORKLOAD}", | ||
| pullRequestNumber: "${pull_request_number}" | ||
| ) | ||
| // Trigger compare-benchmarks job | ||
| build job: 'compare-benchmarks', | ||
| parameters: [ | ||
| string(name:'BASELINE_TEST_EXECUTION_ID', value: "${benchmarkTestIds.baseline}"), | ||
| string(name:'BASELINE_TEST_EXECUTION_ID', value: "${benchmarkTestIds.contender}"), | ||
| string(name:'PULL_REQUEST_NUMBER', value: "${pull_request_number}"), | ||
| string(name:'REPOSITORY', value: "${repository}") | ||
| ], | ||
| wait: false, | ||
| propagate: false | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -290,11 +311,14 @@ pipeline { | |
| failure { | ||
| script { | ||
| if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) { | ||
| withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { | ||
| def pull_request = Integer.parseInt("${pull_request_number}") | ||
| sh ("gh pr comment ${pull_request} --repo ${repository} --body \"The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.\"") | ||
| if ("${pull_request_number}" != '' && "${repository}" != '') { | ||
| withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { | ||
| def pull_request = Integer.parseInt("${pull_request_number}") | ||
| sh ("gh pr comment ${pull_request} --repo ${repository} --body \"The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.\"") | ||
| } | ||
| } | ||
| } | ||
| postCleanup() | ||
| } | ||
|
|
||
| } | ||
|
|
@@ -325,7 +349,6 @@ pipeline { | |
| } | ||
| } | ||
| postCleanup() | ||
|
|
||
| } | ||
| } | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| lib = library(identifier: '[email protected]', retriever: modernSCM([ | ||
|
|
||
| $class: 'GitSCMSource', | ||
| remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', | ||
| ])) | ||
|
|
||
| pipeline { | ||
| agent { label 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' } | ||
| options { | ||
| timeout(time: 24, unit: 'HOURS') | ||
| buildDiscarder(logRotator(daysToKeepStr: '30')) | ||
| throttleJobProperty( | ||
| categories: [], | ||
| limitOneJobWithMatchingParams: false, | ||
| maxConcurrentPerNode: 0, | ||
| maxConcurrentTotal: 20, | ||
| paramsToUseForLimit: '', | ||
| throttleEnabled: true, | ||
| throttleOption: 'project', | ||
| ) | ||
| } | ||
| environment { | ||
| JOB_NAME = 'compare-benchmarks' | ||
| } | ||
| parameters { | ||
| string( | ||
| name: 'BASELINE_TEST_EXECUTION_ID', | ||
| description: 'opensearch-benchmark test-execution-id for baseline run, sample format: `61a34b4e-4ae4-40b7-9d58-6570b9166f6a`', | ||
| trim: true | ||
| ) | ||
| string( | ||
| name: 'CONTENDER_TEST_EXECUTION_ID', | ||
| description: 'opensearch-benchmark test-execution-id of candidate for comparison, mainly from pull requests.', | ||
| trim: true | ||
| ) | ||
| string( | ||
| name: 'PULL_REQUEST_NUMBER', | ||
| trim: true | ||
| ) | ||
| string( | ||
| name: 'REPOSITORY', | ||
| defaultValue: 'opensearch-project/OpenSearch' | ||
| ) | ||
| } | ||
| stages { | ||
| stage('validate-and-set-parameters') { | ||
| steps { | ||
| script { | ||
| if (BASELINE_TEST_EXECUTION_ID == '' || CONTENDER_TEST_EXECUTION_ID == '') { | ||
| currentBuild.result = 'ABORTED' | ||
| error('Benchmark test failed to start. Please provide BASELINE_TEST_EXECUTION_ID and CONTENDER_TEST_EXECUTION_ID to run the job.') | ||
| } | ||
| } | ||
| } | ||
| } | ||
| stage('benchmark-pull-request') { | ||
| steps { | ||
| script { | ||
| runBenchmarkTestScript( | ||
| command: 'compare', | ||
| baseline: "${BASELINE_TEST_EXECUTION_ID}", | ||
| contender: "${CONTENDER_TEST_EXECUTION_ID}", | ||
| suffix: "${BUILD_NUMBER}" | ||
| ) | ||
| } | ||
| } | ||
| post { | ||
| success { | ||
| sh """ | ||
| echo '<details><summary>Benchmark Baseline Comparison Results</summary>' > temp_result_${BUILD_NUMBER}.md | ||
| echo '' >> temp_result_${BUILD_NUMBER}.md | ||
| echo '#### Benchmark Results for Job: ${BUILD_URL}' >> temp_result_${BUILD_NUMBER}.md | ||
| cat final_result_${BUILD_NUMBER}.md >> temp_result_${BUILD_NUMBER}.md | ||
| echo '</details>' >> temp_result_${BUILD_NUMBER}.md | ||
| mv temp_result_${BUILD_NUMBER}.md final_result_${BUILD_NUMBER}.md | ||
| """ | ||
| script { | ||
| if ("${PULL_REQUEST_NUMBER}" != '' && "${REPOSITORY}" != '') { | ||
| withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { | ||
| def pull_request = Integer.parseInt("${PULL_REQUEST_NUMBER}") | ||
| sh ("gh pr comment ${pull_request} --repo ${REPOSITORY} --body-file final_result_${BUILD_NUMBER}.md") | ||
| } | ||
| } | ||
| } | ||
| postCleanup() | ||
| } | ||
| failure { | ||
| postCleanup() | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| import jenkins.tests.BuildPipelineTest | ||
| import org.junit.Before | ||
| import org.junit.Test | ||
|
|
||
| import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString | ||
| import static org.hamcrest.CoreMatchers.equalTo | ||
| import static org.hamcrest.CoreMatchers.hasItem | ||
| import static org.hamcrest.CoreMatchers.hasItems | ||
| import static org.hamcrest.MatcherAssert.assertThat | ||
|
|
||
| import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library | ||
| import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource | ||
|
|
||
| class TestCompareBenchmarks extends BuildPipelineTest { | ||
| @Override | ||
| @Before | ||
| void setUp() { | ||
| super.setUp() | ||
|
|
||
| helper.registerSharedLibrary( | ||
| library().name('jenkins') | ||
| .defaultVersion('6.8.1') | ||
| .allowOverride(true) | ||
| .implicit(true) | ||
| .targetPath('vars') | ||
| .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) | ||
| .build() | ||
| ) | ||
| helper.registerAllowedMethod("s3Download", [Map]) | ||
| helper.registerAllowedMethod("uploadTestResults", [Map]) | ||
| helper.registerAllowedMethod("withAWS", [Map, Closure], { | ||
| args, | ||
| closure -> | ||
| closure.delegate = delegate | ||
| return helper.callClosure(closure) | ||
| }) | ||
| helper.registerAllowedMethod('findFiles', [Map.class], null) | ||
| helper.registerAllowedMethod("withCredentials", [Map]) | ||
| helper.registerAllowedMethod("downloadBuildManifest", [Map], { | ||
| c -> lib.jenkins.BuildManifest.new(readYaml(file: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml')) | ||
| }) | ||
| helper.registerAllowedMethod("throttleJobProperty", [Map]) | ||
|
|
||
| binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test') | ||
| binding.setVariable('ARCHITECTURE', 'x64') | ||
| binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') | ||
| binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') | ||
| binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') | ||
| binding.setVariable('env', ['BUILD_NUMBER': '307']) | ||
| binding.setVariable('BUILD_URL', 'test://artifact.url') | ||
| binding.setVariable('BUILD_NUMBER', '307') | ||
| binding.setVariable('JOB_NAME', 'compare-benchmarks') | ||
| binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') | ||
| binding.setVariable('PULL_REQUEST_NUMBER', '1234') | ||
| binding.setVariable('pull_request', 1234) | ||
| binding.setVariable('REPOSITORY','opensearch-project/OpenSearch') | ||
| binding.setVariable('BASELINE_TEST_EXECUTION_ID', 'baseline-id') | ||
| binding.setVariable('CONTENDER_TEST_EXECUTION_ID', 'contender-id') | ||
| } | ||
|
|
||
| @Test | ||
| public void testCompareBenchmark_verifyPipeline() { | ||
|
|
||
| super.testPipeline("jenkins/opensearch/compare-benchmarks.jenkinsfile", | ||
| "tests/jenkins/jenkinsjob-regression-files/opensearch/compare-benchmarks.jenkinsfile") | ||
| } | ||
|
|
||
| @Test | ||
| void testCompareBenchmark_verifyScriptExecutions() { | ||
| runScript("jenkins/opensearch/compare-benchmarks.jenkinsfile") | ||
|
|
||
| def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { | ||
| shCommand -> shCommand.contains('./test.sh') | ||
| } | ||
| assertThat(testScriptCommands.size(), equalTo(1)) | ||
| assertThat(testScriptCommands, hasItems("set +x && ./test.sh benchmark-test compare baseline-id contender-id --benchmark-config /tmp/workspace/benchmark.ini --suffix 307")) | ||
|
|
||
| def testGhCliCommand = getCommandExecutions('sh', 'gh').findAll { | ||
| shCommand -> shCommand.contains('gh') | ||
| } | ||
| assertThat(testGhCliCommand.size(), equalTo(1)) | ||
| assertThat(testGhCliCommand, hasItem('gh pr comment 1234 --repo opensearch-project/OpenSearch --body-file final_result_307.md')) | ||
| assertCallStack().contains("compare-benchmarks.runBenchmarkTestScript({command=compare, baseline=baseline-id, contender=contender-id, suffix=307})") | ||
| } | ||
|
|
||
| def getCommandExecutions(methodName, command) { | ||
| def shCommands = helper.callStack.findAll { | ||
| call -> | ||
| call.methodName == methodName | ||
| }. | ||
| collect { | ||
| call -> | ||
| callArgsToString(call) | ||
| }.findAll { | ||
| shCommand -> | ||
| shCommand.contains(command) | ||
| } | ||
|
|
||
| return shCommands | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.