HBASE-24318 Create-release scripts fixes and enhancements

.gitignore

@@ -21,3 +21,5 @@ linklint/
 .checkstyle
 **/.checkstyle
 .java-version
+*.log
+**/*.log

dev-support/create-release/README.txt

@@ -5,9 +5,11 @@ For usage, pass '-h':
 
  $ ./do-release-docker.sh -h
 
-To run a build w/o invoking docker (not recommeneded!), use
-_do_release.sh_. It does not take parameters. It will ask
-you what commands to run with taking defaults from environment.
+To run a build w/o invoking docker (not recommended!), use _do_release.sh_.
+
+Both scripts will query interactively for needed parameters and passphrases.
+For explanation of the parameters, execute:
+ $ release-build.sh --help
 
 Before starting the RC build, run a reconciliation of what is in
 JIRA with what is in the commit log. Make sure they align and that
@@ -19,7 +21,6 @@ Running a build on GCE is easy enough. Here are some notes if of use.
 Create an instance. 4CPU/15G/10G disk seems to work well enough.
 Once up, run the below to make your machine fit for RC building:
 
-
 # Presuming debian-compatible OS
 $ sudo apt-get install -y git openjdk-8-jdk maven gnupg gnupg-agent
 # Install docker
@@ -37,13 +38,14 @@ $ sudo add-apt-repository -y \
 $ sudo apt-get update
 $ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
 $ sudo usermod -a -G docker $USERID
-# LOGOUT and then LOGIN again so $USERID shows as part of docker groupl
+# LOGOUT and then LOGIN again so $USERID shows as part of docker group
 # Copy up private key for $USERID export from laptop and import on gce.
 $ gpg --import stack.duboce.net.asc
 $ export GPG_TTY=$(tty) # https://github.com/keybase/keybase-issues/issues/2798
 $ eval $(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
-$ git clone https://github.com/apache/hbase.git
-$ cd hbase
+$ export PROJECT="${PROJECT:-hbase}"
+$ git clone https://github.com/apache/${PROJECT}.git
+$ cd "${PROJECT}"
 $ mkdir ~/build
 $ ./dev-resources/create-release/do-release-docker.sh -d ~/build
 # etc.

dev-support/create-release/do-release-docker.sh

@@ -47,15 +47,16 @@
 #
 set -e
 
-# Set this building other hbase repos: e.g. PROJECT=hbase-operator-tools
+# Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools
 export PROJECT="${PROJECT:-hbase}"
 
-SELF=$(cd $(dirname "$0") && pwd)
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
 
 function usage {
   local NAME
-  NAME="$(basename "$0")"
+  NAME="$(basename "${BASH_SOURCE[0]}")"
   cat <<EOF
 Usage: $NAME [options]
 
@@ -64,24 +65,26 @@ This script runs the release scripts inside a docker image.
 Options:
 
   -d [path]    required. working directory. output will be written to "output" in here.
-  -n           dry run mode. Checks and local builds, but does not upload anything.
+  -f           "force" -- actually publish this release. Unless you specify '-f', it will
+               default to dry run mode, which checks and does local builds, but does not upload anything.
   -t [tag]     tag for the hbase-rm docker image to use for building (default: "latest").
   -j [path]    path to local JDK installation to use building. By default the script will
                use openjdk8 installed in the docker image.
-  -p [project] project to build; default 'hbase'; alternatively, 'hbase-thirdparty', etc.
-  -s [step]    runs a single step of the process; valid steps are: tag, build, publish. if
-               none specified, runs tag, then build, and then publish.
+  -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+  -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
+               If none specified, runs tag, then publish-dist, and then publish-release.
+               'publish-snapshot' is also an allowed, less used, option.
 EOF
 }
 
 WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:hj:np:s:t:" opt; do
+while getopts "d:fhj:p:s:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
-    n) DRY_RUN=1 ;;
+    f) DRY_RUN=0 ;;
     t) IMGTAG="$OPTARG" ;;
     j) JAVA="$OPTARG" ;;
     p) PROJECT="$OPTARG" ;;
@@ -90,6 +93,10 @@ while getopts "d:hj:np:s:t:" opt; do
     ?) error "Invalid option. Run with -h for help." ;;
   esac
 done
+shift $((OPTIND-1))
+if (( $# > 0 )); then
+  error "Arguments can only be provided with option flags, invalid args: $*"
+fi
 
 if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
   error "Work directory (-d) must be defined and exist. Run with -h for help."
@@ -145,27 +152,27 @@ NEXT_VERSION=$NEXT_VERSION
 RELEASE_VERSION=$RELEASE_VERSION
 RELEASE_TAG=$RELEASE_TAG
 GIT_REF=$GIT_REF
-PACKAGE_VERSION=$PACKAGE_VERSION
 ASF_USERNAME=$ASF_USERNAME
 GIT_NAME=$GIT_NAME
 GIT_EMAIL=$GIT_EMAIL
 GPG_KEY=$GPG_KEY
 ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
 RELEASE_STEP=$RELEASE_STEP
-RELEASE_STEP=$RELEASE_STEP
 API_DIFF_TAG=$API_DIFF_TAG
 EOF
 
-JAVA_VOL=
+JAVA_VOL=()
 if [ -n "$JAVA" ]; then
   echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
-  JAVA_VOL="--volume $JAVA:/opt/hbase-java"
+  JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
 fi
 
 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
-docker run -ti \
+cmd=(docker run -ti \
   --env-file "$ENVFILE" \
   --volume "$WORKDIR:/opt/hbase-rm" \
-  $JAVA_VOL \
-  "hbase-rm:$IMGTAG"
+  "${JAVA_VOL[@]}" \
+  "hbase-rm:$IMGTAG")
+echo "${cmd[*]}"
+"${cmd[@]}"

dev-support/create-release/do-release.sh

@@ -23,24 +23,32 @@
 # and passwords to use building.
 export PROJECT="${PROJECT:-hbase}"
 
-SELF=$(cd $(dirname $0) && pwd)
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
 
-while getopts "bn" opt; do
+while getopts "b:fs:" opt; do
   case $opt in
-    b) GIT_BRANCH=$OPTARG ;;
-    n) DRY_RUN=1 ;;
+    b) export GIT_BRANCH=$OPTARG ;;
+    f) export DRY_RUN=0 ;;  # "force", ie actually publish this release (otherwise defaults to dry run)
+    s) RELEASE_STEP="$OPTARG" ;;
     ?) error "Invalid option: $OPTARG" ;;
   esac
 done
+shift $((OPTIND-1))
+if (( $# > 0 )); then
+  error "Arguments can only be provided with option flags, invalid args: $*"
+fi
 
 # If running in docker, import and then cache keys.
 if [ "$RUNNING_IN_DOCKER" = "1" ]; then
   # Run gpg agent.
-  eval $(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
-  echo "GPG Version: `gpg --version`"
-  # Inside docker, need to import the GPG key stored in the current directory.
-  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
+  eval "$(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase \
+          --default-cache-ttl=86400 --max-cache-ttl=86400)"
+  echo "GPG Version: $(gpg --version)"
+  # Inside docker, need to import the GPG keyfile stored in the current directory.
+  # (On workstation, assume GPG has access to keychain/cache with key_id already imported.)
+  echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
 
   # We may need to adjust the path since JAVA_HOME may be overridden by the driver script.
   if [ -n "$JAVA_HOME" ]; then
@@ -53,7 +61,15 @@ else
   # Outside docker, need to ask for information about the release.
   get_release_info
 fi
-export GPG_TTY=$(tty)
+GPG_TTY="$(tty)"
+export GPG_TTY
+
+if [[ -z "$RELEASE_STEP" ]]; then
+  # If doing all stages, leave out 'publish-snapshot'
+  RELEASE_STEP="tag_publish-dist_publish-release"
+  # and use shared maven local repo for efficiency
+  export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}"
+fi
 
 function should_build {
   local WHAT=$1
@@ -66,26 +82,30 @@ function should_build {
   fi
 }
 
-if should_build "tag" && [ $SKIP_TAG = 0 ]; then
+if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then
   run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
-    "$SELF/release-tag.sh"
-  echo "It may take some time for the tag to be synchronized to github."
-  echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available."
-  read
+    "$SELF/release-build.sh" tag
+  if is_dry_run; then
+    export TAG_SAME_DRY_RUN="true";
+  fi
 else
   echo "Skipping tag creation for $RELEASE_TAG."
 fi
 
-if should_build "build"; then
-  run_silent "Building ${PROJECT}..." "build.log" \
-    "$SELF/release-build.sh" build
+if should_build "publish-dist"; then
+  run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \
+    "$SELF/release-build.sh" publish-dist
 else
-  echo "Skipping build step."
+  echo "Skipping publish-dist step."
 fi
 
-if should_build "publish"; then
-  run_silent "Publishing release" "publish.log" \
+if should_build "publish-snapshot"; then
+  run_silent "Publishing snapshot" "publish-snapshot.log" \
+    "$SELF/release-build.sh" publish-snapshot
+
+elif should_build "publish-release"; then
+  run_silent "Publishing release" "publish-release.log" \
     "$SELF/release-build.sh" publish-release
 else
-  echo "Skipping publish step."
+  echo "Skipping publish-release step."
 fi