From 8a7acc9c6f6739f47bae483615ee47257bce0f36 Mon Sep 17 00:00:00 2001 From: Will Broadbelt Date: Wed, 23 Feb 2022 17:44:02 +0000 Subject: [PATCH] SDKQE-2411: Dotnet combination test pipeline matrix improvements Motivations: The combination pipeline does not run all the server versions when they are run, only the first version will run. Also the combination tests are always skipped and will not run unless replayed. Modifications: - Introduce a matrix for the combination tests so that all the versions will be run in parallel - Fix logic of when the combination tests will be skipped/run - Made combination test script more resilient to errors during setup that may prevent tests from running --- dotnet/sdk/build-pipeline.groovy | 238 ++++++++++++++++--------------- 1 file changed, 123 insertions(+), 115 deletions(-) diff --git a/dotnet/sdk/build-pipeline.groovy b/dotnet/sdk/build-pipeline.groovy index f6e868c..90cf5cf 100644 --- a/dotnet/sdk/build-pipeline.groovy +++ b/dotnet/sdk/build-pipeline.groovy @@ -3,21 +3,13 @@ // to the sdkbuilds-jenkinsfile repository. def DOTNET_SDK_VERSIONS = ["2.1.816", "3.1.410", "5.0.404", "6.0.101"] def DOTNET_SDK_VERSION = "6.0.101" -def CB_SERVER_VERSIONS = [ - "7.0-stable", - "6.6-stable", - "6.5.2", - "6.0.5", - "5.5.6" -] def SUFFIX = "r${BUILD_NUMBER}" def BRANCH = "" // use Replay and change this line to force Combination tests to run, even on a gerrit-trigger build. // useful for testing test-only changes. def FORCE_COMBINATION_TEST_RUN = false -def _SKIP_COMBINATION_TESTS = env.SKIP_COMBINATION_TESTS != null && SKIP_COMBINATION_TESTS -def _RUN_COMBINATION_TESTS = FORCE_COMBINATION_TEST_RUN || (IS_GERRIT_TRIGGER.toBoolean() != true && !_SKIP_COMBINATION_TESTS) +def _RUN_COMBINATION_TESTS = FORCE_COMBINATION_TEST_RUN || (!IS_GERRIT_TRIGGER.toBoolean() && !SKIP_COMBINATION_TESTS.toBoolean()) pipeline { agent none @@ -122,14 +114,29 @@ pipeline { } } } - stage("combination-test") { - agent { label "sdkqe-centos7" } - when { + stage("Test Matrix"){ + when { // see top of file for criteria expression { _RUN_COMBINATION_TESTS } } - steps { - doCombinationTests(CB_SERVER_VERSIONS, DOTNET_SDK_VERSION, BRANCH, DOTNET_SDK_VERSIONS) + matrix { + axes { + axis { + name 'ServerVersion' + values "7.1.0-2335", "7.0.3", "6.6.5", "6.0.5", "5.5.6" + } + } + stages { + stage("combination-test") { + agent { label "sdkqe-centos7" } + environment { + CB_SERVER_VERSION = "${ServerVersion}" + } + steps { + doCombinationTests(ServerVersion, DOTNET_SDK_VERSION, BRANCH, DOTNET_SDK_VERSIONS) + } + } + } } } stage("package") { @@ -193,117 +200,118 @@ def packProject(PROJ_FILE, nugetSignKey, version, DOTNET_SDK_VERSION) { batWithEcho("${depsDir}\\dotnet-core-sdk-all\\dotnet pack ${PROJ_FILE} -c Release /p:SignAssembly=true /p:AssemblyOriginatorKeyFile=%NUGET_SIGN_KEY% ${versionParam} /p:IncludeSymbols=true /p:IncludeSource=true /p:SourceLinkCreate=true") } -def doCombinationTests(CB_SERVER_VERSIONS, DOTNET_SDK_VERSION, BRANCH, DOTNET_SDK_VERSIONS) { +def doCombinationTests(CB_SERVER_VERSION, DOTNET_SDK_VERSION, BRANCH, DOTNET_SDK_VERSIONS) { installSdksForPlatform("ubuntu20", DOTNET_SDK_VERSIONS) sh("cbdyncluster ps -a") - for (j in CB_SERVER_VERSIONS) { - cleanWs(patterns: [[pattern: 'deps/**', type: 'EXCLUDE']]) - unstash "couchbase-net-client-ubuntu16" - def cluster_version = j - def caps = "kv,index,n1ql,fts" - if (cluster_version>="5.5"){ - caps=caps+",cbas" - } + cleanWs(patterns: [[pattern: 'deps/**', type: 'EXCLUDE']]) + unstash "couchbase-net-client-ubuntu16" + def cluster_version = CB_SERVER_VERSION + def caps = "kv,index,n1ql,fts" + if (cluster_version>="5.5"){ + caps=caps+",cbas" + } - stage("combination-test ${cluster_version}") { - environment { - CB_SERVER_VERSION = cluster_version + def clusterId = null + shWithEcho("echo ${CB_SERVER_VERSION}") + + try { + // Allocate the cluster + clusterId = sh(script: "cbdyncluster allocate --num-nodes=3 --server-version=" + cluster_version, returnStdout: true) + echo "Got cluster ID $clusterId" + + // Find the cluster IP + def ips = sh(script: "cbdyncluster ips $clusterId", returnStdout: true).trim() + echo "Got raw cluster IPs " + ips + ips = ips.tokenize(',') + def ip = ips[0] + echo "Got cluster IP http://" + ip + ":8091" + + shWithEcho("cbdyncluster --node ${caps} --node kv --node kv --bucket default setup ${clusterId}") + + // Create the cluster + shWithEcho("curl -v -X POST -u Administrator:password -d 'memoryQuota=2048' http://${ip}:8091/pools/default" ) + shWithEcho("cbdyncluster --name=beer-sample add-sample-bucket $clusterId") + shWithEcho("cbdyncluster --name=travel-sample add-sample-bucket $clusterId") + + // setup buckets, users, storage mode + // TODO: one command that does all the default bucket setup (flush, replica, etc...) + shWithEcho("curl -vv -X POST -u Administrator:password -d'storageMode=plasma' http://${ip}:8091/settings/indexes") + shWithEcho("curl -v -X POST -u Administrator:password -d flushEnabled=1 http://${ip}:8091/pools/default/buckets/default") + shWithEcho("curl -v -X POST -u Administrator:password -d replicaNumber=2 http://${ip}:8091/pools/default/buckets/default") + // give time for bucket to populate, before rebalancing + sleep(30) + shWithEcho("curl http://Administrator:password@${ip}:8091/pools/default/buckets/beer-sample") + shWithEcho("curl http://Administrator:password@${ip}:8091/pools/default/buckets/travel-sample") + shWithEcho("curl -vv -X POST -u Administrator:password -d 'knownNodes=ns_1%40${ip}%2Cns_1%40${ips[1]}%2Cns_1%40${ips[2]}' http://${ip}:8091/controller/rebalance") + waitUntilRebalanceComplete("${ip}") + shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `default` USING GSI'") + shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `beer-sample` USING GSI'") + shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `travel-sample` USING GSI'") + + // sleep a bit so the index should probably be ready. + sleep(30) + + + if (BRANCH == "master") { + def configFile = "couchbase-net-client/tests/Couchbase.IntegrationTests/config.json" + def projFile = "couchbase-net-client/tests/Couchbase.IntegrationTests/Couchbase.IntegrationTests.csproj" + def configFileManagement = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/config.json" + def testResults = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/TestResults/TestResults.xml" + def projFileManagement = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/Couchbase.IntegrationTests.Management.csproj" + + // replace hostname in config.json + shWithEcho("sed -i -e 's/localhost/${ip}/' ${configFile}") + shWithEcho("cat ${configFile}") + shWithEcho("sed -i -e 's/localhost/${ip}/' ${configFileManagement}") + shWithEcho("cat ${configFile}") + + // run management tests to set up environment + try{ + shWithEcho("deps/dotnet-core-sdk-all/dotnet test -f net5.0 --filter DisplayName~VerifyEnvironment ${projFileManagement}") + } catch (Exception e) { + echo "Got exception on VerifyEnvironment - continuing anyway, may add some test failures" + } + finally { + sleep(30); } - def clusterId = null + + // run integration tests try { - // Allocate the cluster - clusterId = sh(script: "cbdyncluster allocate --num-nodes=3 --server-version=" + cluster_version, returnStdout: true) - echo "Got cluster ID $clusterId" - - // Find the cluster IP - def ips = sh(script: "cbdyncluster ips $clusterId", returnStdout: true).trim() - echo "Got raw cluster IPs " + ips - ips = ips.tokenize(',') - def ip = ips[0] - echo "Got cluster IP http://" + ip + ":8091" - - shWithEcho("cbdyncluster --node ${caps} --node kv --node kv --bucket default setup ${clusterId}") - - // Create the cluster - shWithEcho("curl -v -X POST -u Administrator:password -d 'memoryQuota=2048' http://${ip}:8091/pools/default" ) - shWithEcho("cbdyncluster --name=beer-sample add-sample-bucket $clusterId") - shWithEcho("cbdyncluster --name=travel-sample add-sample-bucket $clusterId") - - // setup buckets, users, storage mode - // TODO: one command that does all the default bucket setup (flush, replica, etc...) - shWithEcho("curl -vv -X POST -u Administrator:password -d'storageMode=plasma' http://${ip}:8091/settings/indexes") - shWithEcho("curl -v -X POST -u Administrator:password -d flushEnabled=1 http://${ip}:8091/pools/default/buckets/default") - shWithEcho("curl -v -X POST -u Administrator:password -d replicaNumber=2 http://${ip}:8091/pools/default/buckets/default") - // give time for bucket to populate, before rebalancing - sleep(30) - shWithEcho("curl http://Administrator:password@${ip}:8091/pools/default/buckets/beer-sample") - shWithEcho("curl http://Administrator:password@${ip}:8091/pools/default/buckets/travel-sample") - shWithEcho("curl -vv -X POST -u Administrator:password -d 'knownNodes=ns_1%40${ip}%2Cns_1%40${ips[1]}%2Cns_1%40${ips[2]}' http://${ip}:8091/controller/rebalance") - waitUntilRebalanceComplete("${ip}") - shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `default` USING GSI'") - shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `beer-sample` USING GSI'") - shWithEcho("curl -vv -XPOST http://Administrator:password@${ip}:8093/query/service -d 'statement=CREATE PRIMARY INDEX ON `travel-sample` USING GSI'") - - // sleep a bit so the index should probably be ready. - sleep(30) - - - if (BRANCH == "master") { - def configFile = "couchbase-net-client/tests/Couchbase.IntegrationTests/config.json" - def projFile = "couchbase-net-client/tests/Couchbase.IntegrationTests/Couchbase.IntegrationTests.csproj" - def configFileManagement = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/config.json" - def testResults = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/TestResults/TestResults.xml" - def projFileManagement = "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/Couchbase.IntegrationTests.Management.csproj" - - // replace hostname in config.json - shWithEcho("sed -i -e 's/localhost/${ip}/' ${configFile}") - shWithEcho("cat ${configFile}") - shWithEcho("sed -i -e 's/localhost/${ip}/' ${configFileManagement}") - shWithEcho("cat ${configFile}") - - // run management tests to set up environment - shWithEcho("deps/dotnet-core-sdk-all/dotnet test -f net5.0 --filter DisplayName~VerifyEnvironment ${projFileManagement}") - sleep(30); - - // run integration tests - try { - shWithEcho("deps/dotnet-core-sdk-all/dotnet test -f net5.0 --test-adapter-path:. --logger:junit ${projFile}") - } - finally { - junit allowEmptyResults: true, testResults: "couchbase-net-client/tests/Couchbase.IntegrationTests/TestResults/TestResults.xml" - } + shWithEcho("deps/dotnet-core-sdk-all/dotnet test -f net5.0 --test-adapter-path:. --logger:junit ${projFile}") + } + finally { + junit allowEmptyResults: true, testResults: "couchbase-net-client/tests/Couchbase.IntegrationTests/TestResults/TestResults.xml" + } - // run integration tests - try { - shWithEcho("deps/dotnet-core-sdk-all/dotnet test --test-adapter-path:. --logger:junit -f net5.0 ${projFileManagement}") - } - finally { - junit allowEmptyResults: true, testResults: "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/TestResults.xml" - } - } - else if (BRANCH == "release27") { - // This does not actually work as it doesn't appear the integration tests - // in the 2.7 releases actually supports our CI environment currently. - - // replace hostname in config.json - //shWithEcho("sed -i -e 's/localhost/${ip}/' couchbase-net-client/Src/Couchbase.IntegrationTests/config.json") - //shWithEcho("cat couchbase-net-client/Src/Couchbase.IntegrationTests/config.json") - - // run integration tests - //shWithEcho("deps/dotnet-core-sdk-all/dotnet test couchbase-net-client/Src/Couchbase.IntegrationTests/Couchbase.IntegrationTests.csproj") - } - else { - echo "Unknown gerrit branch ${BRANCH}" - } + // run integration tests + try { + shWithEcho("deps/dotnet-core-sdk-all/dotnet test --test-adapter-path:. --logger:junit -f net5.0 ${projFileManagement}") } finally { - if (clusterId != null) { - // Easy to run out of resources during iterating, so cleanup even - // though cluster will be auto-removed after a time - shWithEcho("cbdyncluster rm $clusterId") - } + junit allowEmptyResults: true, testResults: "couchbase-net-client/tests/Couchbase.IntegrationTests.Management/TestResults.xml" } } + else if (BRANCH == "release27") { + // This does not actually work as it doesn't appear the integration tests + // in the 2.7 releases actually supports our CI environment currently. + + // replace hostname in config.json + //shWithEcho("sed -i -e 's/localhost/${ip}/' couchbase-net-client/Src/Couchbase.IntegrationTests/config.json") + //shWithEcho("cat couchbase-net-client/Src/Couchbase.IntegrationTests/config.json") + + // run integration tests + //shWithEcho("deps/dotnet-core-sdk-all/dotnet test couchbase-net-client/Src/Couchbase.IntegrationTests/Couchbase.IntegrationTests.csproj") + } + else { + echo "Unknown gerrit branch ${BRANCH}" + } + } + finally { + if (clusterId != null) { + // Easy to run out of resources during iterating, so cleanup even + // though cluster will be auto-removed after a time + shWithEcho("cbdyncluster rm $clusterId") + } } }