diff --git a/.gitignore b/.gitignore index 0ae87d9d0854..5b33192f8c93 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ linklint/ .checkstyle **/.checkstyle .java-version +*.log +**/*.log diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt index 0d62671eb3ca..62703a176bdc 100644 --- a/dev-support/create-release/README.txt +++ b/dev-support/create-release/README.txt @@ -5,9 +5,11 @@ For usage, pass '-h': $ ./do-release-docker.sh -h -To run a build w/o invoking docker (not recommeneded!), use -_do_release.sh_. It does not take parameters. It will ask -you what commands to run with taking defaults from environment. +To run a build w/o invoking docker (not recommended!), use _do_release.sh_. + +Both scripts will query interactively for needed parameters and passphrases. +For explanation of the parameters, execute: + $ release-build.sh --help Before starting the RC build, run a reconciliation of what is in JIRA with what is in the commit log. Make sure they align and that @@ -19,7 +21,6 @@ Running a build on GCE is easy enough. Here are some notes if of use. Create an instance. 4CPU/15G/10G disk seems to work well enough. Once up, run the below to make your machine fit for RC building: - # Presuming debian-compatible OS $ sudo apt-get install -y git openjdk-8-jdk maven gnupg gnupg-agent # Install docker @@ -37,13 +38,14 @@ $ sudo add-apt-repository -y \ $ sudo apt-get update $ sudo apt-get install -y docker-ce docker-ce-cli containerd.io $ sudo usermod -a -G docker $USERID -# LOGOUT and then LOGIN again so $USERID shows as part of docker groupl +# LOGOUT and then LOGIN again so $USERID shows as part of docker group # Copy up private key for $USERID export from laptop and import on gce. $ gpg --import stack.duboce.net.asc $ export GPG_TTY=$(tty) # https://github.com/keybase/keybase-issues/issues/2798 $ eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400) -$ git clone https://github.com/apache/hbase.git -$ cd hbase +$ export PROJECT="${PROJECT:-hbase}" +$ git clone https://github.com/apache/${PROJECT}.git +$ cd "${PROJECT}" $ mkdir ~/build $ ./dev-resources/create-release/do-release-docker.sh -d ~/build # etc. diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index 3012d3cb2151..645e8f236d65 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -47,15 +47,16 @@ # set -e -# Set this building other hbase repos: e.g. PROJECT=hbase-operator-tools +# Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools export PROJECT="${PROJECT:-hbase}" -SELF=$(cd $(dirname "$0") && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" function usage { local NAME - NAME="$(basename "$0")" + NAME="$(basename "${BASH_SOURCE[0]}")" cat < 0 )); then + error "Arguments can only be provided with option flags, invalid args: $*" +fi if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then error "Work directory (-d) must be defined and exist. Run with -h for help." @@ -145,7 +152,6 @@ NEXT_VERSION=$NEXT_VERSION RELEASE_VERSION=$RELEASE_VERSION RELEASE_TAG=$RELEASE_TAG GIT_REF=$GIT_REF -PACKAGE_VERSION=$PACKAGE_VERSION ASF_USERNAME=$ASF_USERNAME GIT_NAME=$GIT_NAME GIT_EMAIL=$GIT_EMAIL @@ -153,19 +159,20 @@ GPG_KEY=$GPG_KEY ASF_PASSWORD=$ASF_PASSWORD GPG_PASSPHRASE=$GPG_PASSPHRASE RELEASE_STEP=$RELEASE_STEP -RELEASE_STEP=$RELEASE_STEP API_DIFF_TAG=$API_DIFF_TAG EOF -JAVA_VOL= +JAVA_VOL=() if [ -n "$JAVA" ]; then echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE" - JAVA_VOL="--volume $JAVA:/opt/hbase-java" + JAVA_VOL=(--volume "$JAVA:/opt/hbase-java") fi echo "Building $RELEASE_TAG; output will be at $WORKDIR/output" -docker run -ti \ +cmd=(docker run -ti \ --env-file "$ENVFILE" \ --volume "$WORKDIR:/opt/hbase-rm" \ - $JAVA_VOL \ - "hbase-rm:$IMGTAG" + "${JAVA_VOL[@]}" \ + "hbase-rm:$IMGTAG") +echo "${cmd[*]}" +"${cmd[@]}" diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh index 1c10dfc1dc0a..313d277f6aec 100755 --- a/dev-support/create-release/do-release.sh +++ b/dev-support/create-release/do-release.sh @@ -23,24 +23,32 @@ # and passwords to use building. export PROJECT="${PROJECT:-hbase}" -SELF=$(cd $(dirname $0) && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" -while getopts "bn" opt; do +while getopts "b:fs:" opt; do case $opt in - b) GIT_BRANCH=$OPTARG ;; - n) DRY_RUN=1 ;; + b) export GIT_BRANCH=$OPTARG ;; + f) export DRY_RUN=0 ;; # "force", ie actually publish this release (otherwise defaults to dry run) + s) RELEASE_STEP="$OPTARG" ;; ?) error "Invalid option: $OPTARG" ;; esac done +shift $((OPTIND-1)) +if (( $# > 0 )); then + error "Arguments can only be provided with option flags, invalid args: $*" +fi # If running in docker, import and then cache keys. if [ "$RUNNING_IN_DOCKER" = "1" ]; then # Run gpg agent. - eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400) - echo "GPG Version: `gpg --version`" - # Inside docker, need to import the GPG key stored in the current directory. - echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" + eval "$(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase \ + --default-cache-ttl=86400 --max-cache-ttl=86400)" + echo "GPG Version: $(gpg --version)" + # Inside docker, need to import the GPG keyfile stored in the current directory. + # (On workstation, assume GPG has access to keychain/cache with key_id already imported.) + echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key" # We may need to adjust the path since JAVA_HOME may be overridden by the driver script. if [ -n "$JAVA_HOME" ]; then @@ -53,7 +61,15 @@ else # Outside docker, need to ask for information about the release. get_release_info fi -export GPG_TTY=$(tty) +GPG_TTY="$(tty)" +export GPG_TTY + +if [[ -z "$RELEASE_STEP" ]]; then + # If doing all stages, leave out 'publish-snapshot' + RELEASE_STEP="tag_publish-dist_publish-release" + # and use shared maven local repo for efficiency + export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" +fi function should_build { local WHAT=$1 @@ -66,26 +82,30 @@ function should_build { fi } -if should_build "tag" && [ $SKIP_TAG = 0 ]; then +if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \ - "$SELF/release-tag.sh" - echo "It may take some time for the tag to be synchronized to github." - echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available." - read + "$SELF/release-build.sh" tag + if is_dry_run; then + export TAG_SAME_DRY_RUN="true"; + fi else echo "Skipping tag creation for $RELEASE_TAG." fi -if should_build "build"; then - run_silent "Building ${PROJECT}..." "build.log" \ - "$SELF/release-build.sh" build +if should_build "publish-dist"; then + run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \ + "$SELF/release-build.sh" publish-dist else - echo "Skipping build step." + echo "Skipping publish-dist step." fi -if should_build "publish"; then - run_silent "Publishing release" "publish.log" \ +if should_build "publish-snapshot"; then + run_silent "Publishing snapshot" "publish-snapshot.log" \ + "$SELF/release-build.sh" publish-snapshot + +elif should_build "publish-release"; then + run_silent "Publishing release" "publish-release.log" \ "$SELF/release-build.sh" publish-release else - echo "Skipping publish step." + echo "Skipping publish-release step." fi diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 8d344104c09d..ba6f3e817ae6 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -20,34 +20,58 @@ trap cleanup EXIT # Source in utils. -SELF=$(cd $(dirname $0) && pwd) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=SCRIPTDIR/release-util.sh . "$SELF/release-util.sh" # Print usage and exit. function exit_with_usage { - cat << EOF -Usage: release-build.sh -Creates build deliverables from a tag/commit. -Arguments: - build Create binary packages and commit to dist.apache.org/repos/dist/dev/hbase/ - publish-snapshot Publish snapshot release to Apache snapshots - publish-release Publish a release to Apache release repo - -All other inputs are environment variables: - GIT_REF - Release tag or commit to build from - PACKAGE_VERSION - Release identifier in top level package directory (e.g. 2.1.2RC1) - VERSION - (optional) Version of project being built (e.g. 2.1.2) - ASF_USERNAME - Username of ASF committer account - ASF_PASSWORD - Password of ASF committer account - GPG_KEY - GPG key used to sign release artifacts - GPG_PASSPHRASE - Passphrase for GPG key - PROJECT - The project to build. No default. - -Set REPO environment to full path to repo to use -to avoid re-downloading dependencies on each run. + cat <<'EOF' +Usage: release-build.sh +Creates release deliverables from a tag or commit. +Argument: one of 'tag', 'publish-dist', 'publish-snapshot', or 'publish-release' + tag Prepares for release on specified git branch: Set release version, + update CHANGES and RELEASENOTES, create release tag, + increment version for ongoing dev, and publish to Apache git repo. + publish-dist Build and publish distribution packages (tarballs) to Apache dist repo + publish-snapshot Build and publish maven artifacts snapshot release to Apache snapshots repo + publish-release Build and publish maven artifacts release to Apache release repo, and + construct vote email from template + +All other inputs are environment variables. Please use do-release-docker.sh or +do-release.sh to set up the needed environment variables. This script, release-build.sh, +is not intended to be called stand-alone, and such use is untested. The env variables used are: + +Used for 'tag' and 'publish' stages: + PROJECT - The project to build. No default. + RELEASE_VERSION - Version used in pom files for release (e.g. 2.1.2) + Required for 'tag'; defaults for 'publish' to the version in pom at GIT_REF + RELEASE_TAG - Name of release tag (e.g. 2.1.2RC0), also used by + publish-dist as package version name in dist directory path + ASF_USERNAME - Username of ASF committer account + ASF_PASSWORD - Password of ASF committer account + DRY_RUN - 1:true (default), 0:false. If "1", does almost all the work, but doesn't actually + publish anything to upstream source or object repositories. It defaults to "1", so if you want + to actually publish you have to set '-f' (force) flag in do-release.sh or do-release-docker.sh. + +Used only for 'tag': + GIT_NAME - Name to use with git + GIT_EMAIL - E-mail address to use with git + GIT_BRANCH - Git branch on which to make release. Tag is always placed at HEAD of this branch. + NEXT_VERSION - Development version after release (e.g. 2.1.3-SNAPSHOT) + +Used only for 'publish': + GIT_REF - Release tag or commit to build from (defaults to $RELEASE_TAG; only need to + separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time) + If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master. + GPG_KEY - GPG key id (usually email addr) used to sign release artifacts + GPG_PASSPHRASE - Passphrase for GPG key + REPO - Set to full path of a directory to use as maven local repo (dependencies cache) + to avoid re-downloading dependencies for each stage. It is automatically set if you + request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh. For example: - $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh build + $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist EOF exit 1 } @@ -55,79 +79,122 @@ EOF set -e function cleanup { - echo "Cleaning up temp settings file." >&2 - rm "${tmp_settings}" &> /dev/null || true # If REPO was set, then leave things be. Otherwise if we defined a repo clean it out. - if [[ -z "${REPO}" ]] && [[ -n "${tmp_repo}" ]]; then - echo "Cleaning up temp repo in '${tmp_repo}'. set REPO to reuse downloads." >&2 - rm -rf "${tmp_repo}" &> /dev/null || true + if [[ -z "${REPO}" ]] && [[ -n "${MAVEN_LOCAL_REPO}" ]]; then + echo "Cleaning up temp repo in '${MAVEN_LOCAL_REPO}'. Set REPO to reuse downloads." >&2 + rm -f "${MAVEN_SETTINGS_FILE}" &> /dev/null || true + rm -rf "${MAVEN_LOCAL_REPO}" &> /dev/null || true fi } -if [ $# -eq 0 ]; then +if [ $# -ne 1 ]; then exit_with_usage fi -if [[ $@ == *"help"* ]]; then +if [[ "$*" == *"help"* ]]; then exit_with_usage fi -# Read in the ASF password. -if [[ -z "$ASF_PASSWORD" ]]; then - echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' - echo - stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo -fi +init_locale +init_java +init_mvn +init_python +# Print out subset of perl version (used in git hooks and japi-compliance-checker) +perl --version | grep 'This is' -# Read in the GPG passphrase -if [[ -z "$GPG_PASSPHRASE" ]]; then - echo 'The environment variable GPG_PASSPHRASE is not set. Enter the passphrase to' - echo 'unlock the GPG signing key that will be used to sign the release!' - echo - stty -echo && printf "GPG passphrase: " && read GPG_PASSPHRASE && printf '\n' && stty echo - export GPG_PASSPHRASE - export GPG_TTY=$(tty) -fi +rm -rf "${PROJECT}" + +if [[ "$1" == "tag" ]]; then + # for 'tag' stage + set -o pipefail + set -x # detailed logging during action + check_get_passwords ASF_PASSWORD + check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \ + GIT_EMAIL GIT_NAME GIT_BRANCH + ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" + encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")" + encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")" + git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}" + + # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified + # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*' + # projects share the same HBASE jira name. To make this work, by convention, the HBASE jira "Fix Version" + # field values have the sub-project name pre-pended, as in "hbase-operator-tools-1.0.0". + # So, here we prepend the project name to the version, but only for the hbase sub-projects. + jira_fix_version="${RELEASE_VERSION}" + shopt -s nocasematch + if [[ "${PROJECT}" =~ ^hbase- ]]; then + jira_fix_version="${PROJECT}-${RELEASE_VERSION}" + fi + shopt -u nocasematch + update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" + + cd "${PROJECT}" + + git config user.name "$GIT_NAME" + git config user.email "$GIT_EMAIL" -for env in ASF_USERNAME GPG_PASSPHRASE GPG_KEY; do - if [ -z "${!env}" ]; then - echo "ERROR: $env must be set to run this script" - exit_with_usage + # Create release version + maven_set_version "$RELEASE_VERSION" + git add RELEASENOTES.md CHANGES.md + + git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" + echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" + git tag "$RELEASE_TAG" + + # Create next version + maven_set_version "$NEXT_VERSION" + + git commit -a -m "Preparing development version $NEXT_VERSION" + + if ! is_dry_run; then + # Push changes + git push origin "$RELEASE_TAG" + git push origin "HEAD:$GIT_BRANCH" + wait_for_tag "$RELEASE_TAG" + cd .. + rm -rf "${PROJECT}" + else + cd .. + mv "${PROJECT}" "${PROJECT}.tag" + echo "Dry run: Clone with version changes and tag available as ${PROJECT}.tag in the output directory." fi -done + exit 0 +fi -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 +### Below is for 'publish-*' stages ### +check_get_passwords ASF_PASSWORD +if [[ -z "$GPG_PASSPHRASE" ]]; then + check_get_passwords GPG_PASSPHRASE + GPG_TTY="$(tty)" + export GPG_TTY +fi +check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY GPG_PASSPHRASE # Commit ref to checkout when building -GIT_REF=${GIT_REF:-master} -RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" BASE_DIR=$(pwd) +GIT_REF=${GIT_REF:-master} +if [[ "$PROJECT" =~ ^hbase ]]; then + RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/hbase" +else + RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/${PROJECT}" +fi -init_java -init_mvn -init_python -# Print out subset of perl version. -perl --version | grep 'This is' - -rm -rf ${PROJECT} -ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" -git clone "$ASF_REPO" -cd ${PROJECT} -git checkout $GIT_REF -git_hash=`git rev-parse --short HEAD` -echo "Checked out ${PROJECT} git hash $git_hash" - -if [ -z "$VERSION" ]; then - # Run $MVN in a separate command so that 'set -e' does the right thing. - TMP=$(mktemp) - $MVN help:evaluate -Dexpression=project.version > $TMP - VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download) - rm $TMP +# in case of dry run, enable publish steps to chain from tag step +if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then + ln -s "${PROJECT}.tag" "${PROJECT}" +else + ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}" + git clone "$ASF_REPO" "${PROJECT}" fi +cd "${PROJECT}" +git checkout "$GIT_REF" +git_hash="$(git rev-parse --short HEAD)" +echo "Checked out ${PROJECT} at ${GIT_REF} commit $git_hash" -# Profiles for publishing snapshots and release to Maven Central -PUBLISH_PROFILES="-P apache-release,release" +if [ -z "${RELEASE_VERSION}" ]; then + RELEASE_VERSION="$(maven_get_version)" +fi # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins # machines by explicitly calling /usr/sbin/lsof. Please see SPARK-22377 and the discussion @@ -137,120 +204,105 @@ if ! hash $LSOF 2>/dev/null; then LSOF=/usr/sbin/lsof fi -if [ -z "$PACKAGE_VERSION" ]; then - PACKAGE_VERSION="${VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" +package_version_name="$RELEASE_TAG" +if [ -z "$package_version_name" ]; then + package_version_name="${RELEASE_VERSION}-$(date +%Y_%m_%d_%H_%M)-${git_hash}" fi -DEST_DIR_NAME="$PACKAGE_VERSION" - git clean -d -f -x cd .. +set -x # detailed logging during action -tmp_repo="${REPO:-`pwd`/$(mktemp -d hbase-repo-XXXXX)}" -tmp_settings="/${tmp_repo}/tmp-settings.xml" -echo "" > "$tmp_settings" -echo "apache.snapshots.https$ASF_USERNAME" >> "$tmp_settings" -echo "$ASF_PASSWORD" >> "$tmp_settings" -echo "apache.releases.https$ASF_USERNAME" >> "$tmp_settings" -echo "$ASF_PASSWORD" >> "$tmp_settings" -echo "" >> "$tmp_settings" -echo "" >> "$tmp_settings" -export tmp_settings - -if [[ "$1" == "build" ]]; then +if [[ "$1" == "publish-dist" ]]; then # Source and binary tarballs echo "Packaging release source tarballs" - make_src_release "${PROJECT}" "${VERSION}" + make_src_release "${PROJECT}" "${RELEASE_VERSION}" + + echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Building binary dist" + make_binary_release "${PROJECT}" "${RELEASE_VERSION}" + echo "$(date -u +'%Y-%m-%dT%H:%M:%SZ') Done building binary distribution" - # Add timestamps to mvn logs. - MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" + if [[ "$PROJECT" =~ ^hbase- ]]; then + DEST_DIR_NAME="${PROJECT}-${package_version_name}" + else + DEST_DIR_NAME="$package_version_name" + fi + svn_target="svn-${PROJECT}" + svn co --depth=empty "$RELEASE_STAGING_LOCATION" "$svn_target" + rm -rf "${svn_target:?}/${DEST_DIR_NAME}" + mkdir -p "$svn_target/${DEST_DIR_NAME}" + + echo "Copying release tarballs" + cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/" + cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/" + cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/" + shopt -s nocasematch + # Generate api report only if project is hbase for now. + if [ "${PROJECT}" == "hbase" ]; then + # This script usually reports an errcode along w/ the report. + generate_api_report "./${PROJECT}" "${API_DIFF_TAG}" "${GIT_REF}" || true + cp api*.html "$svn_target/${DEST_DIR_NAME}/" + fi + shopt -u nocasematch - echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Building binary dist" - make_binary_release "${PROJECT}" "${VERSION}" - echo "`date -u +'%Y-%m-%dT%H:%M:%SZ'` Done building binary distribution" + svn add "$svn_target/${DEST_DIR_NAME}" if ! is_dry_run; then - svn co --depth=empty $RELEASE_STAGING_LOCATION svn-hbase - rm -rf "svn-hbase/${DEST_DIR_NAME}" - mkdir -p "svn-hbase/${DEST_DIR_NAME}" - - echo "Copying release tarballs" - cp ${PROJECT}-*.tar.* "svn-hbase/${DEST_DIR_NAME}/" - cp ${PROJECT}/CHANGES.md "svn-hbase/${DEST_DIR_NAME}/" - cp ${PROJECT}/RELEASENOTES.md "svn-hbase/${DEST_DIR_NAME}/" - shopt -s nocasematch - # Generate api report only if project is hbase for now. - if [ "${PROJECT}" == "hbase" ]; then - # This script usually reports an errcode along w/ the report. - generate_api_report ./${PROJECT} "${API_DIFF_TAG}" "${PACKAGE_VERSION}" || true - cp api*.html "svn-hbase/${DEST_DIR_NAME}/" - fi - shopt -u nocasematch - - svn add "svn-hbase/${DEST_DIR_NAME}" - - cd svn-hbase - svn ci --username $ASF_USERNAME --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $PACKAGE_VERSION" --no-auth-cache + cd "$svn_target" + svn ci --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache ${PROJECT} $package_version_name" --no-auth-cache cd .. - rm -rf svn-hbase + rm -rf "$svn_target" + else + mv "$svn_target/${DEST_DIR_NAME}" "${svn_target}_${DEST_DIR_NAME}.dist" + echo "Dry run: svn-managed 'dist' directory with release tarballs, CHANGES.md and RELEASENOTES.md available as $(pwd)/${svn_target}_${DEST_DIR_NAME}.dist" + rm -rf "$svn_target" fi exit 0 fi if [[ "$1" == "publish-snapshot" ]]; then + ( cd "${PROJECT}" - # Publish ${PROJECT} to Maven release repo - echo "Deploying ${PROJECT} SNAPSHOT at '$GIT_REF' ($git_hash)" - echo "Publish version is $VERSION" - if [[ ! $VERSION == *"SNAPSHOT"* ]]; then - echo "ERROR: Snapshots must have a version containing SNAPSHOT" - echo "ERROR: You gave version '$VERSION'" - exit 1 + mvn_log="${BASE_DIR}/mvn_deploy_snapshot.log" + echo "Publishing snapshot to nexus" + maven_deploy snapshot "$mvn_log" + if ! is_dry_run; then + echo "Snapshot artifacts successfully published to repo." + rm "$mvn_log" + else + echo "Dry run: Snapshot artifacts successfully built, but not published due to dry run." fi - # Coerce the requested version - $MVN versions:set -DnewVersion=$VERSION - $MVN --settings $tmp_settings -DskipTests "$PUBLISH_PROFILES" deploy - cd .. - exit 0 + ) + exit $? fi if [[ "$1" == "publish-release" ]]; then ( cd "${PROJECT}" - # Publish ${PROJECT} to Maven release repo - echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" - echo "Publish version is $VERSION" - # Coerce the requested version - $MVN versions:set -DnewVersion=$VERSION - declare -a mvn_goals=(clean install) - declare staged_repo_id="dryrun-no-repo" - if ! is_dry_run; then - mvn_goals=("${mvn_goals[@]}" deploy) - fi + mvn_log="${BASE_DIR}/mvn_deploy_release.log" echo "Staging release in nexus" - if ! MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings "$tmp_settings" \ - -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES}" \ - -Dmaven.repo.local="${tmp_repo}" \ - "${mvn_goals[@]}" > "${BASE_DIR}/mvn_deploy.log"; then - echo "Staging build failed, see 'mvn_deploy.log' for details." >&2 - exit 1 - fi + maven_deploy release "$mvn_log" + declare staged_repo_id="dryrun-no-repo" if ! is_dry_run; then - staged_repo_id=$(grep -o "Closing staging repository with ID .*" "${BASE_DIR}/mvn_deploy.log" \ + staged_repo_id=$(grep -o "Closing staging repository with ID .*" "$mvn_log" \ | sed -e 's/Closing staging repository with ID "\([^"]*\)"./\1/') - echo "Artifacts successfully staged to repo ${staged_repo_id}" + echo "Release artifacts successfully published to repo ${staged_repo_id}" + rm "$mvn_log" else - echo "Artifacts successfully built. not staged due to dry run." + echo "Dry run: Release artifacts successfully built, but not published due to dry run." fi # Dump out email to send. Where we find vote.tmpl depends # on where this script is run from - export PROJECT_TEXT=$(echo "${PROJECT}" | sed "s/-/ /g") - eval "echo \"$(< ${SELF}/vote.tmpl)\"" |tee "${BASE_DIR}/vote.txt" + PROJECT_TEXT="${PROJECT//-/ }" #substitute like 's/-/ /g' + export PROJECT_TEXT + eval "echo \"$(< "${SELF}/vote.tmpl")\"" |tee "${BASE_DIR}/vote.txt" ) - exit 0 + exit $? fi +set +x # done with detailed logging cd .. rm -rf "${PROJECT}" -echo "ERROR: expects to be called with 'install', 'publish-release' or 'publish-snapshot'" +echo "ERROR: expects to be called with 'tag', 'publish-dist', 'publish-release', or 'publish-snapshot'" >&2 +exit_with_usage diff --git a/dev-support/create-release/release-tag.sh b/dev-support/create-release/release-tag.sh deleted file mode 100755 index 28ade2c0915f..000000000000 --- a/dev-support/create-release/release-tag.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Tags release. Updates releasenotes and changes. -SELF=$(cd $(dirname $0) && pwd) -. "$SELF/release-util.sh" - -function exit_with_usage { - local NAME=$(basename $0) - cat << EOF -usage: $NAME -Tags an $PROJECT release on a particular branch. - -Inputs are specified with the following environment variables: -ASF_USERNAME - Apache Username -ASF_PASSWORD - Apache Password -GIT_NAME - Name to use with git -GIT_EMAIL - E-mail address to use with git -GIT_BRANCH - Git branch on which to make release -RELEASE_VERSION - Version used in pom files for release -RELEASE_TAG - Name of release tag -NEXT_VERSION - Development version after release -EOF - exit 1 -} - -set -e -set -o pipefail - -if [[ $@ == *"help"* ]]; then - exit_with_usage -fi - -if [[ -z "$ASF_PASSWORD" ]]; then - echo 'The environment variable ASF_PASSWORD is not set. Enter the password.' - echo - stty -echo && printf "ASF password: " && read ASF_PASSWORD && printf '\n' && stty echo -fi - -for env in ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL \ - GIT_NAME GIT_BRANCH GPG_KEY; do - if [ -z "${!env}" ]; then - echo "$env must be set to run this script" - exit 1 - fi -done - -init_java -init_mvn - -rm -rf ${PROJECT} - -ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git" -# Ugly! -encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')") -encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')") -git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b $GIT_BRANCH -# NOTE: Here we are prepending project name on version for fetching -# changes from the HBASE JIRA. It has issues for hbase, hbase-conectors, -# hbase-operator-tools, etc. -shopt -s nocasematch -if [ "${PROJECT}" != "hbase" ]; then - # Needs the '-' on the end. - prefix="${PROJECT}-" -fi -shopt -u nocasematch -update_releasenotes `pwd`/${PROJECT} "${prefix}${RELEASE_VERSION}" - -cd ${PROJECT} - -git config user.name "$GIT_NAME" -git config user.email $GIT_EMAIL - -# Create release version -$MVN versions:set -DnewVersion=$RELEASE_VERSION | grep -v "no value" # silence logs -git add RELEASENOTES.md CHANGES.md - -git commit -a -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" -echo "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" -git tag $RELEASE_TAG - -# Create next version -$MVN versions:set -DnewVersion=$NEXT_VERSION | grep -v "no value" # silence logs - -git commit -a -m "Preparing development version $NEXT_VERSION" - -if ! is_dry_run; then - # Push changes - git push origin $RELEASE_TAG - git push origin HEAD:$GIT_BRANCH - cd .. - rm -rf ${PROJECT} -else - cd .. - mv ${PROJECT} ${PROJECT}.tag - echo "Clone with version changes and tag available as ${PROJECT}.tag in the output directory." -fi - diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 99a8e902d0dc..397a20c77835 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -16,13 +16,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -DRY_RUN=${DRY_RUN:-0} +DRY_RUN=${DRY_RUN:-1} #default to dry run GPG="gpg --pinentry-mode loopback --no-tty --batch" YETUS_VERSION=${YETUS_VERSION:-0.11.1} +# Maven Profiles for publishing snapshots and release to Maven Central and Dist +PUBLISH_PROFILES=("-P" "apache-release,release") + set -e function error { - echo "$*" + echo "Error: $*" >&2 exit 1 } @@ -31,10 +34,10 @@ function read_config { local DEFAULT="$2" local REPLY= - read -p "$PROMPT [$DEFAULT]: " REPLY + read -r -p "$PROMPT [$DEFAULT]: " REPLY local RETVAL="${REPLY:-$DEFAULT}" if [ -z "$RETVAL" ]; then - error "$PROMPT is must be provided." + error "$PROMPT must be provided." fi echo "$RETVAL" } @@ -50,7 +53,7 @@ function run_silent { shift 2 echo "========================" - echo "= $BANNER" + echo "=== $BANNER" echo "Command: $*" echo "Log file: $LOG_FILE" @@ -62,6 +65,7 @@ function run_silent { tail "$LOG_FILE" exit $EC fi + echo "=== SUCCESS" } function fcreate_secure { @@ -75,31 +79,52 @@ function check_for_tag { curl -s --head --fail "$ASF_GITHUB_REPO/releases/tag/$1" > /dev/null } +function wait_for_tag { + # Confirm the tag synchronizes to github. This can take a couple minutes, + # but usually it just takes a few seconds. + local max_propagation_time=300 + local prop_delay=30 + while ! check_for_tag "$1"; do + if (( max_propagation_time <= 0 )); then + echo "ERROR: Taking more than 5 minutes to propagate Release Tag $1 to github mirror." >&2 + echo "Please wait and resume other create-release steps when $1 is available in github." >&2 + exit 1 + fi + echo "Waiting up to $max_propagation_time seconds for tag to propagate to github mirror..." + sleep $prop_delay + max_propagation_time=$((max_propagation_time - prop_delay)) + done +} + # API compare version. function get_api_diff_version { - local version=$1 - local rev=$(echo "$version" | cut -d . -f 3) + local version="$1" + local rev local api_diff_tag - if [ $rev != 0 ]; then - local short_version=$(echo "$version" | cut -d . -f 1-2) + rev=$(echo "$version" | cut -d . -f 3) + if [ "$rev" != 0 ]; then + local short_version + short_version="$(echo "$version" | cut -d . -f 1-2)" api_diff_tag="rel/${short_version}.$((rev - 1))" else - local major=$(echo "$version" | cut -d . -f 1) - local minor=$(echo "$version" | cut -d . -f 2) - if [ $minor != 0 ]; then + local major minor + major="$(echo "$version" | cut -d . -f 1)" + minor="$(echo "$version" | cut -d . -f 2)" + if [ "$minor" != 0 ]; then api_diff_tag="rel/${major}.$((minor - 1)).0" else api_diff_tag="rel/$((major - 1)).0.0" fi fi - api_diff_tag=$(read_config "api_diff_tag", "$api_diff_tag") - echo $api_diff_tag + api_diff_tag="$(read_config "api_diff_tag" "$api_diff_tag")" + echo "$api_diff_tag" } # Get all branches that begin with 'branch-', the hbase convention for # release branches, sort them and then pop off the most recent. function get_release_info { - export PROJECT=$(read_config "PROJECT" "$PROJECT") + PROJECT="$(read_config "PROJECT" "$PROJECT")" + export PROJECT if [[ -z "${ASF_REPO}" ]]; then ASF_REPO="https://gitbox.apache.org/repos/asf/${PROJECT}.git" @@ -112,31 +137,34 @@ function get_release_info { fi if [ -z "$GIT_BRANCH" ]; then # If no branch is specified, find out the latest branch from the repo. - GIT_BRANCH=$(git ls-remote --heads "$ASF_REPO" | + GIT_BRANCH="$(git ls-remote --heads "$ASF_REPO" | grep refs/heads/branch- | awk '{print $2}' | sort -r | head -n 1 | - cut -d/ -f3) + cut -d/ -f3)" fi - export GIT_BRANCH=$(read_config "GIT_BRANCH" "$GIT_BRANCH") + GIT_BRANCH="$(read_config "GIT_BRANCH" "$GIT_BRANCH")" + export GIT_BRANCH # Find the current version for the branch. - local VERSION=$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | - parse_version) - echo "Current branch VERSION is $VERSION." + local version + version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" | + parse_version)" + echo "Current branch VERSION is $version." - NEXT_VERSION="$VERSION" + NEXT_VERSION="$version" RELEASE_VERSION="" - SHORT_VERSION=$(echo "$VERSION" | cut -d . -f 1-2) - if [[ ! $VERSION =~ .*-SNAPSHOT ]]; then - RELEASE_VERSION="$VERSION" + SHORT_VERSION="$(echo "$version" | cut -d . -f 1-2)" + if [[ ! "$version" =~ .*-SNAPSHOT ]]; then + RELEASE_VERSION="$version" else - RELEASE_VERSION="${VERSION/-SNAPSHOT/}" + RELEASE_VERSION="${version/-SNAPSHOT/}" fi - local REV=$(echo "${RELEASE_VERSION}" | cut -d . -f 3) + local REV + REV="$(echo "${RELEASE_VERSION}" | cut -d . -f 3)" # Find out what RC is being prepared. # - If the current version is "x.y.0", then this is RC0 of the "x.y.0" release. @@ -144,7 +172,7 @@ function get_release_info { # - If it has, then we're building RC0 of the current version. # - If it has not, we're building the next RC of the previous version. local RC_COUNT - if [ $REV != 0 ]; then + if [ "$REV" != 0 ]; then local PREV_REL_REV=$((REV - 1)) PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}" if check_for_tag "$PREV_REL_TAG"; then @@ -153,7 +181,7 @@ function get_release_info { NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT" else RELEASE_VERSION="${SHORT_VERSION}.${PREV_REL_REV}" - RC_COUNT=$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l) + RC_COUNT="$(git ls-remote --tags "$ASF_REPO" "${RELEASE_VERSION}RC*" | wc -l)" # This makes a 'number' of it. RC_COUNT=$((RC_COUNT)) fi @@ -163,60 +191,63 @@ function get_release_info { RC_COUNT=0 fi - export RELEASE_VERSION=$(read_config "RELEASE_VERSION" "$RELEASE_VERSION") - export NEXT_VERSION=$(read_config "NEXT_VERSION" "$NEXT_VERSION") - + RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")" + NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")" + export RELEASE_VERSION NEXT_VERSION - RC_COUNT=$(read_config "RC_COUNT" "$RC_COUNT") + RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")" + RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" + RELEASE_TAG="$(read_config "RELEASE_TAG" "$RELEASE_TAG")" # Check if the RC already exists, and if re-creating the RC, skip tag creation. - RELEASE_TAG="${RELEASE_VERSION}RC${RC_COUNT}" SKIP_TAG=0 if check_for_tag "$RELEASE_TAG"; then - read -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER + read -r -p "$RELEASE_TAG already exists. Continue anyway [y/n]? " ANSWER if [ "$ANSWER" != "y" ]; then - error "Exiting." + echo "Exiting." + exit 1 fi SKIP_TAG=1 fi - export RELEASE_TAG + export RELEASE_TAG SKIP_TAG GIT_REF="$RELEASE_TAG" if is_dry_run; then - echo "This is a dry run. Please confirm the ref that will be built for testing." - GIT_REF=$(read_config "GIT_REF" "$GIT_REF") + echo "This is a dry run. If tag does not actually exist, please confirm the ref that will be built for testing." + GIT_REF="$(read_config "GIT_REF" "$GIT_REF")" fi export GIT_REF - export PACKAGE_VERSION="$RELEASE_TAG" - export API_DIFF_TAG=$(get_api_diff_version $RELEASE_VERSION) + API_DIFF_TAG="$(get_api_diff_version "$RELEASE_VERSION")" # Gather some user information. - export ASF_USERNAME=$(read_config "ASF_USERNAME" "$LOGNAME") + ASF_USERNAME="$(read_config "ASF_USERNAME" "$LOGNAME")" - GIT_NAME=$(git config user.name || echo "") - export GIT_NAME=$(read_config "GIT_NAME" "$GIT_NAME") + GIT_NAME="$(git config user.name || echo "")" + GIT_NAME="$(read_config "GIT_NAME" "$GIT_NAME")" - export GIT_EMAIL="$ASF_USERNAME@apache.org" - export GPG_KEY=$(read_config "GPG_KEY" "$GIT_EMAIL") + GIT_EMAIL="$ASF_USERNAME@apache.org" + GPG_KEY="$(read_config "GPG_KEY" "$GIT_EMAIL")" + export API_DIFF_TAG ASF_USERNAME GIT_NAME GIT_EMAIL GPG_KEY cat < 0 )) && exit_with_usage + return 0 +} + +function init_locale { + local locale_value + OS="$(uname -s)" + case "${OS}" in + Darwin*) locale_value="en_US.UTF-8";; + Linux*) locale_value="C.UTF-8";; + *) error "unknown OS";; + esac + export LC_ALL="$locale_value" + export LANG="$locale_value" } # Initializes JAVA_VERSION to the version of the JVM in use. @@ -255,25 +327,66 @@ function init_java { function init_python { if ! [ -x "$(command -v python2)" ]; then - echo 'Error: python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' >&2 - exit 1 + error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2' fi - echo "python version: `python2 --version`" + echo "python version: $(python2 --version)" } # Set MVN function init_mvn { if [ -n "$MAVEN_HOME" ]; then - MVN=${MAVEN_HOME}/bin/mvn - elif [ $(type -P mvn) ]; then - MVN=mvn + MVN=("${MAVEN_HOME}/bin/mvn") + elif [ "$(type -P mvn)" ]; then + MVN=(mvn) else error "MAVEN_HOME is not set nor is mvn on the current path." fi - echo "mvn version: `$MVN --version`" - # Add timestamped logging. - MVN="${MVN} -B" + # Add batch mode. + MVN=("${MVN[@]}" -B) export MVN + echo -n "mvn version: " + "${MVN[@]}" --version + configure_maven +} + +function configure_maven { + # Add timestamps to mvn logs. + MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss ${MAVEN_OPTS}" + # Suppress gobs of "Download from central:" messages + MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn ${MAVEN_OPTS}" + MAVEN_LOCAL_REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}" + [[ -d "$MAVEN_LOCAL_REPO" ]] || mkdir -p "$MAVEN_LOCAL_REPO" + MAVEN_SETTINGS_FILE="${MAVEN_LOCAL_REPO}/tmp-settings.xml" + MVN=("${MVN[@]}" --settings "${MAVEN_SETTINGS_FILE}") + export MVN MAVEN_OPTS MAVEN_SETTINGS_FILE MAVEN_LOCAL_REPO + export ASF_USERNAME ASF_PASSWORD + # reference passwords from env rather than storing in the settings.xml file. + cat <<'EOF' > "$MAVEN_SETTINGS_FILE" + + + /${env.MAVEN_LOCAL_REPO} + + apache.snapshots.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + apache.releases.https${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + gpg.passphrase + ${env.GPG_PASSPHRASE} + + + + + true + + + ${env.GPG_KEY} + + + + +EOF } # Writes report into cwd! @@ -281,42 +394,63 @@ function generate_api_report { local project="$1" local previous_tag="$2" local release_tag="$3" + local previous_version # Generate api report. - ${project}/dev-support/checkcompatibility.py --annotation \ + "${project}"/dev-support/checkcompatibility.py --annotation \ org.apache.yetus.audience.InterfaceAudience.Public \ - $previous_tag $release_tag - local previous_version=$(echo ${previous_tag} | sed -e 's/rel\///') - cp ${project}/target/compat-check/report.html "./api_compare_${previous_version}_to_${release_tag}.html" + "$previous_tag" "$release_tag" + previous_version="$(echo "${previous_tag}" | sed -e 's/rel\///')" + cp "${project}/target/compat-check/report.html" "./api_compare_${previous_version}_to_${release_tag}.html" +} + +# Look up the Jira name associated with project. +# Currently all the 'hbase-*' projects share the same HBASE jira name. This works because, +# by convention, the HBASE jira "Fix Version" field values have the sub-project name pre-pended, +# as in "hbase-operator-tools-1.0.0". +# TODO: For non-hbase-related projects, enhance this to use Jira API query instead of text lookup. +function get_jira_name { + local project="$1" + local jira_name + case "${project}" in + hbase*) jira_name="HBASE";; + *) jira_name="";; + esac + if [[ -z "$jira_name" ]]; then + error "Sorry, can't determine the Jira name for project $project" + fi + echo "$jira_name" } # Update the CHANGES.md # DOES NOT DO COMMITS! Caller should do that. # yetus requires python2 to be on the path. function update_releasenotes { - local project="$1" - local release_version="$2" + local project_dir="$1" + local jira_fix_version="$2" local yetus="apache-yetus-${YETUS_VERSION}" + local jira_project + jira_project="$(get_jira_name "$(basename "$project_dir")")" wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/${yetus}-bin.tar.gz" | \ tar xvz -C . || exit - cd ./${yetus} || exit - ./bin/releasedocmaker -p HBASE --fileversions -v ${release_version} -l --sortorder=newer --skip-credits + cd "./${yetus}" || exit + ./bin/releasedocmaker -p "${jira_project}" --fileversions -v "${jira_fix_version}" -l --sortorder=newer --skip-credits # First clear out the changes written by previous RCs. pwd - sed -i -e "/^## Release ${release_version}/,/^## Release/ {//!d; /^## Release ${release_version}/d;}" \ - ${project}/CHANGES.md || true - sed -i -e "/^# HBASE ${release_version} Release Notes/,/^# HBASE/{//!d; /^# HBASE ${release_version} Release Notes/d;}" \ - ${project}/RELEASENOTES.md || true + sed -i -e "/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \ + "${project_dir}/CHANGES.md" || true + sed -i -e "/^# ${jira_project} ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project} ${jira_fix_version} Release Notes/d;}" \ + "${project_dir}/RELEASENOTES.md" || true # The above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md. - # To insert into project CHANGES.me...need to cut the top off the + # To insert into project's CHANGES.md...need to cut the top off the # CHANGELOG.X.X.X.md file removing license and first line and then # insert it after the license comment closing where we have a # DO NOT REMOVE marker text! - sed -i -e '/## Release/,$!d' CHANGELOG.${release_version}.md - sed -i -e "/DO NOT REMOVE/r CHANGELOG.${release_version}.md" ${project}/CHANGES.md + sed -i -e '/## Release/,$!d' "CHANGELOG.${jira_fix_version}.md" + sed -i -e "/DO NOT REMOVE/r CHANGELOG.${jira_fix_version}.md" "${project_dir}/CHANGES.md" # Similar for RELEASENOTES but slightly different. - sed -i -e '/Release Notes/,$!d' RELEASENOTES.${release_version}.md - sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${release_version}.md" ${project}/RELEASENOTES.md + sed -i -e '/Release Notes/,$!d' "RELEASENOTES.${jira_fix_version}.md" + sed -i -e "/DO NOT REMOVE/r RELEASENOTES.${jira_fix_version}.md" "${project_dir}/RELEASENOTES.md" cd .. || exit } @@ -332,14 +466,14 @@ function update_releasenotes { # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_src_release() { # Tar up the src and sign and hash it. - project="${1}" - version="${2}" - basename="${project}-${version}" - rm -rf "${basename}-src*" - tgz="${basename}-src.tar.gz" + local project="${1}" + local version="${2}" + local base_name="${project}-${version}" + rm -rf "${base_name}"-src* + tgz="${base_name}-src.tar.gz" cd "${project}" || exit git clean -d -f -x - git archive --format=tar.gz --output="../${tgz}" --prefix="${basename}/" "${GIT_REF:-master}" + git archive --format=tar.gz --output="../${tgz}" --prefix="${base_name}/" "${GIT_REF:-master}" cd .. || exit echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "${tgz}.asc" \ --detach-sig "${tgz}" @@ -354,15 +488,15 @@ make_src_release() { # - GPG needs to be defined, with the path to GPG: defaults 'gpg'. # - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file). # - GIT_REF which is the tag to create the tgz from: defaults to 'master'. -# - MVN Default is 'mvn'. +# - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE". # For example: # $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0 make_binary_release() { - project="${1}" - version="${2}" - basename="${project}-${version}" - rm -rf "${basename}-bin*" - cd $project || exit + local project="${1}" + local version="${2}" + local base_name="${project}-${version}" + rm -rf "${base_name}"-bin* + cd "$project" || exit git clean -d -f -x # Three invocations of maven. This seems to work. One to @@ -370,24 +504,92 @@ make_binary_release() { # a third to assemble the binary artifact. Trying to do # all in the one invocation fails; a problem in our # assembly spec to in maven. TODO. Meantime, three invocations. - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings clean install -DskipTests \ - -Dmaven.repo.local="${tmp_repo}" - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings site -DskipTests \ - -Dmaven.repo.local="${tmp_repo}" - MAVEN_OPTS="${MAVEN_OPTS}" ${MVN} --settings $tmp_settings install assembly:single -DskipTests \ - -Dcheckstyle.skip=true "${PUBLISH_PROFILES}" -Dmaven.repo.local="${tmp_repo}" + "${MVN[@]}" clean install -DskipTests + "${MVN[@]}" site -DskipTests + kick_gpg_agent + "${MVN[@]}" install assembly:single -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" # Check there is a bin gz output. The build may not produce one: e.g. hbase-thirdparty. - f_bin_tgz="./${PROJECT}-assembly/target/${basename}*-bin.tar.gz" - if ls ${f_bin_tgz} &>/dev/null; then - cp ${f_bin_tgz} .. + local f_bin_prefix="./${PROJECT}-assembly/target/${base_name}" + if ls "${f_bin_prefix}"*-bin.tar.gz &>/dev/null; then + cp "${f_bin_prefix}"*-bin.tar.gz .. cd .. || exit - for i in "${basename}"*-bin.tar.gz; do + for i in "${base_name}"*-bin.tar.gz; do echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i" echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${i}" > "$i.sha512" done else cd .. || exit - echo "No ${f_bin_tgz} product; expected?" + echo "No ${f_bin_prefix}*-bin.tar.gz product; expected?" + fi +} + +# "Wake up" the gpg agent so it responds properly to maven-gpg-plugin, and doesn't cause timeout. +# Specifically this is done between invocation of 'mvn site' and 'mvn assembly:single', because +# the 'site' build takes long enough that the gpg-agent does become unresponsive and the following +# 'assembly' build (where gpg signing occurs) experiences timeout, without this "kick". +function kick_gpg_agent { + # All that's needed is to run gpg on a random file + local i + i="$(mktemp)" + echo "This is a test file" > "$i" + echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i" + rm "$i" "$i.asc" +} + +# Do maven command to set version into local pom +function maven_set_version { #input: + local this_version="$1" + echo "${MVN[@]}" versions:set -DnewVersion="$this_version" + "${MVN[@]}" versions:set -DnewVersion="$this_version" | grep -v "no value" # silence logs +} + +# Do maven command to read version from local pom +function maven_get_version { + # shellcheck disable=SC2016 + "${MVN[@]}" -q -N -Dexec.executable="echo" -Dexec.args='${project.version}' exec:exec +} + +# Do maven deploy to snapshot or release artifact repository, with checks. +function maven_deploy { #inputs: + # Invoke with cwd=$PROJECT + local deploy_type="$1" + local mvn_log_file="$2" #secondary log file used later to extract staged_repo_id + if [[ "$deploy_type" != "snapshot" && "$deploy_type" != "release" ]]; then + error "unrecognized deploy type, must be 'snapshot'|'release'" + fi + if [[ -z "$mvn_log_file" ]] || ! touch "$mvn_log_file"; then + error "must provide writable maven log output filepath" + fi + # shellcheck disable=SC2153 + if [[ "$deploy_type" == "snapshot" ]] && ! [[ "$RELEASE_VERSION" =~ -SNAPSHOT$ ]]; then + error "Snapshots must have a version with suffix '-SNAPSHOT'; you gave version '$RELEASE_VERSION'" + elif [[ "$deploy_type" == "release" ]] && [[ "$RELEASE_VERSION" =~ SNAPSHOT ]]; then + error "Non-snapshot release version must not include the word 'SNAPSHOT'; you gave version '$RELEASE_VERSION'" + fi + # Publish ${PROJECT} to Maven repo + # shellcheck disable=SC2154 + echo "Publishing ${PROJECT} checkout at '$GIT_REF' ($git_hash)" + echo "Publish version is $RELEASE_VERSION" + # Coerce the requested version + maven_set_version "$RELEASE_VERSION" + # Prepare for signing + kick_gpg_agent + declare -a mvn_goals=(clean install) + if ! is_dry_run; then + mvn_goals=("${mvn_goals[@]}" deploy) + fi + echo "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + "${mvn_goals[@]}" + echo "Logging to ${mvn_log_file}. This will take a while..." + rm -f "$mvn_log_file" + # The tortuous redirect in the next command allows mvn's stdout and stderr to go to mvn_log_file, + # while also sending stderr back to the caller. + # shellcheck disable=SC2094 + if ! "${MVN[@]}" -DskipTests -Dcheckstyle.skip=true "${PUBLISH_PROFILES[@]}" \ + "${mvn_goals[@]}" 1>> "$mvn_log_file" 2> >( tee -a "$mvn_log_file" >&2 ); then + error "Deploy build failed, for details see log at '$mvn_log_file'." fi + echo "BUILD SUCCESS." + return 0 } diff --git a/dev-support/create-release/vote.tmpl b/dev-support/create-release/vote.tmpl index c9c4b742e561..7810cd22d281 100644 --- a/dev-support/create-release/vote.tmpl +++ b/dev-support/create-release/vote.tmpl @@ -8,23 +8,24 @@ The VOTE will remain open for at least 72 hours. The tag to be voted on is ${RELEASE_TAG}: -https://github.com/apache/${PROJECT}/tree/${RELEASE_TAG} + https://github.com/apache/${PROJECT}/tree/${RELEASE_TAG} The release files, including signatures, digests, as well as CHANGES.md and RELEASENOTES.md included in this RC can be found at: - https://dist.apache.org/repos/dist/dev/hbase/${RELEASE_TAG}/ + https://dist.apache.org/repos/dist/dev/hbase/${RELEASE_TAG}/ Maven artifacts are available in a staging repository at: - https://repository.apache.org/content/repositories/${staged_repo_id}/ + https://repository.apache.org/content/repositories/${staged_repo_id}/ Artifacts were signed with the ${GPG_KEY} key which can be found in: - https://dist.apache.org/repos/dist/release/hbase/KEYS + https://dist.apache.org/repos/dist/release/hbase/KEYS - To learn more about apache ${PROJECT_TEXT}, please see -http://hbase.apache.org/ +To learn more about Apache ${PROJECT_TEXT}, please see + + http://hbase.apache.org/ Thanks, Your HBase Release Manager