diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile index c43976f61dd1..10cd1a8cb78a 100644 --- a/dev-support/create-release/hbase-rm/Dockerfile +++ b/dev-support/create-release/hbase-rm/Dockerfile @@ -15,11 +15,11 @@ # limitations under the License. # -# Image for building HBase releases. Based on Ubuntu 18.04. +# Image for building HBase releases. Based on Ubuntu 22.04. # # Includes: # * Java 8 -FROM ubuntu:18.04 +FROM ubuntu:22.04 # Install extra needed repos and refresh. @@ -28,40 +28,46 @@ FROM ubuntu:18.04 # the most current package versions (instead of potentially using old versions cached by docker). RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \ - curl='7.58.0-*' \ - git='1:2.17.1-*' \ - gnupg='2.2.4-*' \ - libcurl4-openssl-dev='7.58.0-*' \ - libxml2-dev='2.9.4+dfsg1-*' \ - lsof='4.89+dfsg-*' \ + curl='7.81.0-*' \ + git='1:2.34.1-*' \ + gnupg='2.2.27-*' \ + libcurl4-openssl-dev='7.81.0-*' \ + libxml2-dev='2.9.13+dfsg-*' \ + libxml2-utils='2.9.13+dfsg-*' \ + lsof='4.93.2+dfsg-*' \ openjdk-8-jdk='8u*' \ - python-pip='9.0.1-*' \ - subversion='1.9.7-*' \ - wget='1.19.4-*' \ + python3-pip='22.0.2+dfsg-*' \ + subversion='1.14.1-*' \ + wget='1.21.2-*' \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ - && pip install \ - python-dateutil==2.8.1 + && pip3 install \ + python-dateutil==2.8.2 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Install mvn 3.6.3. -ARG MAVEN_VERSION=3.6.3 -ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0 -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries +# Install maven +ENV MAVEN_VERSION=3.8.6 +ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" +ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26" RUN mkdir -p /opt/maven \ - && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && echo "${SHA} /tmp/apache-maven.tar.gz" | sha512sum -c - \ + && curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \ + && echo "${MAVEN_SHA512} /tmp/apache-maven.tar.gz" | sha512sum -c - \ && tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \ && rm -f /tmp/apache-maven.tar.gz \ && ln -s /opt/maven/bin/mvn /usr/bin/mvn # Install Apache Yetus -ENV YETUS_VERSION 0.12.0 -RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \ - tar xvz -C /opt -ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION} +ENV YETUS_VERSION 0.14.1 +ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" +ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109" +RUN mkdir -p /opt/yetus \ + && curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \ + && echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \ + && tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \ + && rm -f /tmp/apache-yetus.tar.gz +ENV YETUS_HOME /opt/yetus ARG UID ARG RM_USER diff --git a/dev-support/create-release/prepend_changes.py b/dev-support/create-release/prepend_changes.py index bba51594809f..7e9dfbb4e51f 100755 --- a/dev-support/create-release/prepend_changes.py +++ b/dev-support/create-release/prepend_changes.py @@ -24,7 +24,7 @@ print("usage: %s " % sys.argv[0]) sys.exit(1) -pattern = re.compile(r'^## Release .+ - Unreleased .+$') +pattern = re.compile(r'^## Release .+$') with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w: line = prev_r.readline() while line: diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index d471aa931e72..fa9844a2df08 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -149,7 +149,7 @@ if [[ "$1" == "tag" ]]; then # check their position when generating release data. We can not put them under the source code # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when # generating src release tarball, we will reset the git repo - if [[ $(is_tracked "CHANGES.md") == 0 ]]; then + if is_tracked "CHANGES.md"; then git add RELEASENOTES.md CHANGES.md git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" cp CHANGES.md ../ diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index fdb078810cef..64b481ac3c29 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -60,8 +60,11 @@ function read_config { } function parse_version { - grep -e '.*' | \ - head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1 + xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" - +} + +function parse_revision { + xmllint --xpath "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='revision']/text()" - } function banner { @@ -135,6 +138,8 @@ function get_api_diff_version { # Get all branches that begin with 'branch-', the hbase convention for # release branches, sort them and then pop off the most recent. function get_release_info { + init_xmllint + PROJECT="$(read_config "PROJECT" "$PROJECT")" export PROJECT @@ -164,6 +169,14 @@ function get_release_info { local version version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | parse_version)" + # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html + # If we use ${revision} as placeholder, we need to parse the revision property to + # get maven version + # shellcheck disable=SC2016 + if [[ "${version}" == '${revision}' ]]; then + version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | + parse_revision)" + fi log "Current branch VERSION is $version." NEXT_VERSION="$version" @@ -344,6 +357,17 @@ function init_locale { export LANG="$locale_value" } +# Check whether xmllint is available +function init_xmllint { + if ! [ -x "$(command -v xmllint)" ]; then + log "Error: xmllint is not available, we need to use it for parsing pom.xml." >&2 + log "Ubuntu: apt install libxml2-utils" >&2 + log "CentOS: yum install xmlstarlet" >&2 + log "Mac OS: brew install xmlstarlet" >&2 + exit 1 + fi +} + # Initializes JAVA_VERSION to the version of the JVM in use. function init_java { if [ -z "$JAVA_HOME" ]; then @@ -355,10 +379,10 @@ function init_java { } function init_python { - if ! [ -x "$(command -v python2)" ]; then - error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' + if ! [ -x "$(command -v python3)" ]; then + error 'python3 needed by yetus and api report. Install or add link?' fi - log "python version: $(python2 --version)" + log "python3 version: $(python3 --version)" } # Set MVN @@ -440,8 +464,8 @@ function git_clone_overwrite { log "Clone will be of the gitbox repo for ${PROJECT}." if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then # Ugly! - encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')") - encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')") + encoded_username=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_USERNAME''', ''))") + encoded_password=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_PASSWORD''', ''))") GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}" else GIT_REPO="https://${asf_repo}" @@ -533,7 +557,7 @@ function get_jira_name { # Update the CHANGES.md # DOES NOT DO COMMITS! Caller should do that. # requires yetus to have a defined home already. -# yetus requires python2 to be on the path. +# yetus requires python3 to be on the path. function update_releasenotes { local project_dir="$1" local jira_fix_version="$2" @@ -686,28 +710,13 @@ function kick_gpg_agent { function maven_set_version { #input: local this_version="$1" local use_revision='false' - local -a version_splits=() - IFS='.' read -ar version_splits <<< "$(maven_get_version)" - - # Do the right thing based on project and release line. - if [ "${PROJECT}" = 'hbase' ] ; then - if [ "${version_splits[0]}" -le 1 ] ; then - use_revision='false' - elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -le 4 ] ; then - use_revision='false' - elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -ge 5 ] ; then - use_revision='true' - elif [ "${version_splits[0]}" -ge 3 ] ; then - use_revision='true' - fi - elif [ "${PROJECT}" = 'hbase-thirdparty' ] ; then - use_revision='false' - elif [ "${PROJECT}" = 'hbase-connectors' ] ; then + local maven_version + maven_version="$(parse_version < pom.xml)" + # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html + # If we use ${revision} as placeholder, the way to bump maven version will be different + # shellcheck disable=SC2016 + if [[ "${maven_version}" == '${revision}' ]]; then use_revision='true' - elif [ "${PROJECT}" = 'hbase-filesystem' ] ; then - use_revision='false' - elif [ "${PROJECT}" = 'hbase-operator-tools' ] ; then - use_revision='false' fi if [ "${use_revision}" = 'false' ] ; then