diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index f93408d6fd3a..4357bb0d0cc6 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -38,6 +38,7 @@ pipeline { OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3' OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3' OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3' + OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS = 'output-jdk17-hadoop3-backwards' PROJECT = 'hbase' PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' @@ -57,6 +58,9 @@ pipeline { ASF_NIGHTLIES = 'https://nightlies.apache.org' ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}" ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}" + // These are dependent on the branch + HADOOP3_VERSIONS = "3.3.5,3.3.6,3.4.0" + HADOOP3_DEFAULT_VERSION = "3.3.5" } parameters { booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release. @@ -153,35 +157,47 @@ pipeline { } } stage ('hadoop 3 cache') { - environment { - HADOOP3_VERSION="3.3.5" - } steps { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir('downloads-hadoop-3') { - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" + script { + hadoop3_versions = env.HADOOP3_VERSIONS.split(","); + env.HADOOP3_VERSIONS_REGEX = "[" + hadoop3_versions.join("|") + "]"; + for (hadoop3_version in hadoop3_versions) { + env.HADOOP3_VERSION = hadoop3_version; + echo "env.HADOOP3_VERSION" + env.hadoop3_version; + stage ('Hadoop 3 cache inner stage') { + // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( + dir("downloads-hadoop-${HADOOP3_VERSION}") { + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" ''' - } - sh '''#!/usr/bin/env bash - set -e - echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-hadoop-3" \ - --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ - "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" - for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do - echo "Delete stale hadoop 3 cache ${stale}" - rm -rf $stale - done - ''' - stash name: 'hadoop-3', includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz" - } - } - } - } + } //dir + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-hadoop-${HADOOP3_VERSION}" \ + --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ + --verify-tar-gz \ + "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ + "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" + for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do + echo "Delete stale hadoop 3 cache ${stale}" + rm -rf $stale + done + ''' + stash name: "hadoop-${HADOOP3_VERSION}", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz" + script { + if (env.HADOOP3_VERSIONS == env.HADOOP3_DEFAULT_VERSION) { + stash(name: "hadoop-3", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz") + } //if + } //script + } //stage ('Hadoop 3 cache inner stage') + } //for + } //script + } //steps + } //stage ('hadoop 3 cache') { + } //parallel + } //stage ('thirdparty installs') stage ('init health results') { steps { // stash with given name for all tests we might run, so that we can unstash all of them even if @@ -191,6 +207,12 @@ pipeline { stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match" stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match" stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match" + script { + for (hadoop3_version in hadoop3_versions) { + // confusing environment vs Groovy variables + stash(name: "jdk17-hadoop3-backwards-result-${hadoop3_version}", allowEmpty: true, includes: "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${hadoop3_version}/doesn't-match") + } + } stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match" } } @@ -748,6 +770,130 @@ pipeline { } } } + // If/when we transition to transient runners, we could run every Hadoop check as a matrix job + stage ('yetus jdk17 hadoop3 backwards compatibility checks') { + agent { + node { + label 'hbase' + } + } + environment { + BASEDIR = "${env.WORKSPACE}/component" + TESTS = "${env.DEEP_CHECKS}" + SET_JAVA_HOME = "/usr/lib/jvm/java-17" + // Activates hadoop 3.0 profile in maven runs. + HADOOP_PROFILE = '3.0' + // HADOOP_THREE_VERSION is set in script for loop + TEST_PROFILE = 'runDevTests' + SKIP_ERRORPRONE = true + } + steps { + script { + for (hadoop3_version in hadoop3_versions) { + //HADOOP_THREE_VERSION is the environment variable name expected by the nightly shell script + env.HADOOP_THREE_VERSION = hadoop3_version; + env.OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${env.HADOOP_THREE_VERSION}" + env.OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${env.HADOOP_THREE_VERSION}" + try { + stage ('yetus jdk17 hadoop3 backwards compatibility checks inner stage') { + // Must do prior to anything else, since if one of them timesout we'll stash the commentfile + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}" + rm -f "${OUTPUT_DIR}/commentfile" + ''' + unstash 'yetus' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then + echo '(/) {color:green}+1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}' > "${OUTPUT_DIR}/commentfile" + else + echo '(x) {color:red}-1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}' > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see jdk17 report|${BUILD_URL}JDK17_20Nightly_20Build_20Report_20_28Hadoop3_29/]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of + // test output. See HBASE-26339 for more details. + currentBuild.result = 'UNSTABLE' + } + } //script + } //stage ('yetus jdk17 hadoop3 backwards compatibility checks inner stage') { + } //try + finally { + stash name: "jdk17-hadoop3-backwards-result-${HADOOP_THREE_VERSION}", includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true + // zip surefire reports. + sh '''#!/bin/bash -e + if [ ! -f "${OUTPUT_DIR}/commentfile" ]; then + echo "(x) {color:red}-1 jdk17 hadoop ${HADOOP_THREE_VERSION} backward compatibility checks{color}" >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + fi + if [ -d "${OUTPUT_DIR}/archiver" ]; then + count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi + ''' + sshPublisher(publishers: [ + sshPublisherDesc(configName: 'Nightlies', + transfers: [ + sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}", + sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip" + ) + ] + ) + ]) + // remove the big test logs zip file, store the nightlies url in test_logs.html + sh '''#!/bin/bash -e + if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then + echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" + rm -rf "${OUTPUT_DIR}/test_logs.zip" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + else + echo "No test_logs.zip, skipping" + fi + ''' + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*" + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE. + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'console-report.html', + reportName : "JDK17 Nightly Build Report (Hadoop ${HADOOP_THREE_VERSION} backwards compatibility)" + ] + } //finally + } // for + } //script + } //steps + } //stage ('yetus jdk17 hadoop3 backwards compatibility checks') // This is meant to mimic what a release manager will do to create RCs. // See http://hbase.apache.org/book.html#maven.release @@ -782,8 +928,7 @@ pipeline { # remove old hadoop tarballs in workspace rm -rf hadoop-2*.tar.gz rm -rf hadoop-3*.tar.gz - echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile - echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile + rm -f "output-integration/commentfile" ''' sh '''#!/usr/bin/env bash set -e @@ -855,70 +1000,90 @@ pipeline { echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile exit 2 fi + echo "(/) {color:green}+1 client integration test for HBase 2 {color}" >output-integration/commentfile else echo "Skipping to run against Hadoop 2 for branch ${BRANCH}" fi ''' - unstash 'hadoop-3' - sh '''#!/bin/bash -e - echo "Attempting to use run an instance on top of Hadoop 3." - artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) - tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" - # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 - ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3 - hbase_install_dir="hbase-install" - hbase_client_dir="hbase-client" - if [ -d "hbase-hadoop3-install" ]; then - echo "run hadoop3 client integration test against hbase hadoop3 binaries" - hbase_install_dir="hbase-hadoop3-install" - hbase_client_dir="hbase-hadoop3-client" - fi - docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ - -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ - --workdir=/hbase hbase-integration-test \ - component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ - --single-process \ - --working-dir output-integration/hadoop-3 \ - --hbase-client-install ${hbase_client_dir} \ - ${hbase_install_dir} \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-3.log 2>&1 - if [ $? -ne 0 ]; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile - exit 2 - fi - echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program." - docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ - -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ - -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ - --workdir=/hbase hbase-integration-test \ - component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ - --single-process \ - --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ - --working-dir output-integration/hadoop-3-shaded \ - --hbase-client-install ${hbase_client_dir} \ - ${hbase_install_dir} \ - hadoop-3/bin/hadoop \ - hadoop-3/share/hadoop/yarn/timelineservice \ - hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ - hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ - hadoop-3/bin/mapred \ - >output-integration/hadoop-3-shaded.log 2>&1 - if [ $? -ne 0 ]; then - echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile - exit 2 - fi - echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile - ''' - } + script { + for (hadoop3_version in hadoop3_versions) { + env.HADOOP3_VERSION = hadoop3_version; + echo "env.HADOOP3_VERSION" + env.hadoop3_version; + stage ("packaging and integration Hadoop 3 inner stage ") { + unstash "hadoop-" + env.HADOOP3_VERSION + sh '''#!/bin/bash -e + echo "Attempting to use run an instance on top of Hadoop ${HADOOP3_VERSION}." + # Clean up any previous tested Hadoop3 files before unpacking the current one + rm -rf hadoop-3/* + # Create working dir + rm -rf "output-integration/hadoop-${HADOOP3_VERSION}" && mkdir "output-integration/hadoop-${HADOOP3_VERSION}" + rm -rf "output-integration/hadoop-${HADOOP3_VERSION}-shaded" && mkdir "output-integration/hadoop-${HADOOP3_VERSION}-shaded" + artifact=$(ls -1 "${WORKSPACE}"/hadoop-${HADOOP3_VERSION}-bin.tar.gz | head -n 1) + tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" + # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 + ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3 + hbase_install_dir="hbase-install" + hbase_client_dir="hbase-client" + if [ -d "hbase-hadoop3-install" ]; then + echo "run hadoop3 client integration test against hbase hadoop3 binaries" + hbase_install_dir="hbase-hadoop3-install" + hbase_client_dir="hbase-hadoop3-client" + fi + docker build -t hbase-integration-test -f "${BASEDIR}/dev-support/docker/Dockerfile" . + docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ + -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ + --workdir=/hbase hbase-integration-test \ + component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ + --single-process \ + --working-dir output-integration/hadoop-${HADOOP3_VERSION} \ + --hbase-client-install ${hbase_client_dir} \ + ${hbase_install_dir} \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/timelineservice \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-3/bin/mapred \ + >output-integration/hadoop-${HADOOP3_VERSION}.log 2>&1 + if [ $? -ne 0 ]; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop ${HADOOP3_VERSION}. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-${HADOOP3_VERSION}.log]. (note that this means we didn't check the Hadoop ${HADOOP3_VERSION} shaded client)" >> output-integration/commentfile + exit 2 + fi + echo "Attempting to use run an instance on top of Hadoop ${HADOOP3_VERSION}, relying on the Hadoop client artifacts for the example client program." + docker run --rm -v "${WORKSPACE}":/hbase -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -u `id -u`:`id -g` -e JAVA_HOME="/usr/lib/jvm/java-17" \ + -e HADOOP_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED" \ + --workdir=/hbase hbase-integration-test \ + component/dev-support/hbase_nightly_pseudo-distributed-test.sh \ + --single-process \ + --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \ + --working-dir output-integration/hadoop-${HADOOP3_VERSION}-shaded \ + --hbase-client-install ${hbase_client_dir} \ + ${hbase_install_dir} \ + hadoop-3/bin/hadoop \ + hadoop-3/share/hadoop/yarn/timelineservice \ + hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \ + hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \ + hadoop-3/bin/mapred \ + >output-integration/hadoop-${HADOOP3_VERSION}-shaded.log 2>&1 + if [ $? -ne 0 ]; then + echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop ${HADOOP3_VERSION} using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-${HADOOP3_VERSION}-shaded.log]." >> output-integration/commentfile + exit 2 + fi + echo "(/) {color:green}+1 client integration test for ${HADOOP3_VERSION} {color}" >> output-integration/commentfile + ''' + } //stage ("packaging and integration Hadoop 3 inner stage ") + } //for + } // script + } //steps post { always { + sh '''#!/bin/bash -e + if [ ! -f "output-integration/commentfile" ]; then + echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile + echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile + fi + ''' stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile" sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', @@ -944,12 +1109,12 @@ pipeline { archiveArtifacts artifacts: 'output-srctarball/**/*' archiveArtifacts artifacts: 'output-integration/*' archiveArtifacts artifacts: 'output-integration/**/*' - } - } - } - } - } - } + } //always + } //post + } //stage packaging + } // parallel + } //stage:_health checks + } //stages post { always { script { @@ -963,6 +1128,7 @@ pipeline { rm -rf ${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3} rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3} + rm -rf ${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-* rm -rf output-srctarball rm -rf output-integration ''' @@ -977,9 +1143,13 @@ pipeline { "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile", "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile", - "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile", - 'output-srctarball/commentfile', - 'output-integration/commentfile'] + "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/commentfile"] + for (hadoop3_version in hadoop3_versions) { + unstash("jdk17-hadoop3-backwards-result-${hadoop3_version}") + results.add("${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3_BACKWARDS}-${hadoop3_version}/commentfile") + } + results.add('output-srctarball/commentfile') + results.add('output-integration/commentfile') echo env.BRANCH_NAME echo env.BUILD_URL echo currentBuild.result diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index a6a3d95e937a..46f08276c651 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -111,6 +111,14 @@ function personality_parse_args delete_parameter "${i}" HADOOP_PROFILE=${i#*=} ;; + --hadoop-three-version=*) + delete_parameter "${i}" + HADOOP_THREE_VERSION=${i#*=} + ;; + --test-profile=*) + delete_parameter "${i}" + TEST_PROFILE=${i#*=} + ;; --skip-errorprone) delete_parameter "${i}" SKIP_ERRORPRONE=true @@ -182,6 +190,11 @@ function personality_modules extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}" fi + # If we have HADOOP_THREE_VERSION specified it pass along in the hadoop.version system property. + if [[ -n "${HADOOP_THREE_VERSION}" ]] ; then + extra="${extra} -Dhadoop-three.version=${HADOOP_THREE_VERSION}" + fi + # BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on # full source code instead. In this case, do full compiles, tests, etc instead of per # module. @@ -233,8 +246,15 @@ function personality_modules # tests respectively. if [[ ${testtype} == unit ]]; then local tests_arg="" + + if [ -n "${TEST_PROFILE}" ]; then + extra="${extra} -P${TEST_PROFILE}" + else + extra="${extra} -PrunAllTests" + fi + get_include_exclude_tests_arg tests_arg - extra="${extra} -PrunAllTests ${tests_arg}" + extra="${extra} ${tests_arg}" # Inject the jenkins build-id for our surefire invocations # Used by zombie detection stuff, even though we're not including that yet. @@ -589,6 +609,7 @@ function hadoopcheck_rebuild fi if [[ "${PATCH_BRANCH}" = *"branch-2.5"* ]]; then + # TODO remove this on non 2.5 branches ? yetus_info "Setting Hadoop 3 versions to test based on branch-2.5 rules" if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then hbase_hadoop3_versions="3.2.4 3.3.6 3.4.0" @@ -597,6 +618,7 @@ function hadoopcheck_rebuild fi else yetus_info "Setting Hadoop 3 versions to test based on branch-2.6+/master/feature branch rules" + # Isn't runnung these tests with the default Hadoop version redundant ? if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then hbase_hadoop3_versions="3.3.6 3.4.0" else diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh index 9292222cf52b..bbe5a70c664e 100755 --- a/dev-support/hbase_nightly_pseudo-distributed-test.sh +++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh @@ -281,7 +281,10 @@ trap cleanup EXIT SIGQUIT echo "Starting up Hadoop" -if [ "${hadoop_version%.*.*}" -gt 2 ]; then +if [ "${hadoop_version%.*.*.*}" = "3.4.0" ]; then + # writeDetails is broken in 3.4.0. See MAPREDUCE-7492 + "${mapred_exec}" minicluster -format -writeConfig "${working_dir}/hbase-conf/core-site.xml" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" & +elif [ "${hadoop_version%.*.*}" -gt 2 ]; then "${mapred_exec}" minicluster -format -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" & else HADOOP_CLASSPATH="${timeline_service_dir}/*:${timeline_service_dir}/lib/*:${yarn_server_tests_test_jar}" "${hadoop_exec}" jar "${mapred_jobclient_test_jar}" minicluster -format -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" & diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index 4cee1efde560..0f16827bfddf 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -74,7 +74,7 @@ if [[ -n "${INCLUDE_TESTS_URL}" ]]; then YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}") fi -# For testing with specific hadoop version. Activates corresponding profile in maven runs. +# For testing with specific hadoop major version. Activates corresponding profile in maven runs. if [[ -n "${HADOOP_PROFILE}" ]]; then # Master has only Hadoop3 support. We don't need to activate any profile. # The Jenkinsfile should not attempt to run any Hadoop2 tests. @@ -83,6 +83,16 @@ if [[ -n "${HADOOP_PROFILE}" ]]; then fi fi +# For testing with specific hadoop version. Activates corresponding profile in maven runs. +if [[ -n "${HADOOP_THREE_VERSION}" ]]; then + YETUS_ARGS=("--hadoop-three-version=${HADOOP_THREE_VERSION}" "${YETUS_ARGS[@]}") +fi + +if [[ -n "${TEST_PROFILE}" ]]; then + # i.e. runAllTests / runDevTests + YETUS_ARGS=("--test-profile=${TEST_PROFILE}" "${YETUS_ARGS[@]}") +fi + if [[ "${SKIP_ERRORPRONE}" = "true" ]]; then YETUS_ARGS=("--skip-errorprone" "${YETUS_ARGS[@]}") fi