HDDS-4741. Modularize upgrade test

hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh

@@ -17,7 +17,7 @@
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
 ALL_RESULT_DIR="$SCRIPT_DIR/result"
 mkdir -p "$ALL_RESULT_DIR"
-rm "$ALL_RESULT_DIR/*" || true
+rm "$ALL_RESULT_DIR"/* || true
 source "$SCRIPT_DIR/../testlib.sh"
 
 tests=$(find_tests)

hadoop-ozone/dist/src/main/compose/restart/test.sh

@@ -23,19 +23,12 @@ export COMPOSE_DIR
 
 export OZONE_VOLUME
 
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
-  current_user=$(whoami)
-  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
-    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
-      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
-  fi
-fi
-
 # shellcheck source=/dev/null
 source "${COMPOSE_DIR}/../testlib.sh"
 
+mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+fix_data_dir_permissions
+
 # prepare pre-upgrade cluster
 start_docker_env
 execute_robot_test scm -v PREFIX:pre freon/generate.robot

hadoop-ozone/dist/src/main/compose/test-all.sh

@@ -23,7 +23,7 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
 ALL_RESULT_DIR="$SCRIPT_DIR/result"
 PROJECT_DIR="$SCRIPT_DIR/.."
 mkdir -p "$ALL_RESULT_DIR"
-rm "$ALL_RESULT_DIR/*" || true
+rm "$ALL_RESULT_DIR"/* || true
 
 source "$SCRIPT_DIR"/testlib.sh
 

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -16,6 +16,9 @@
 # limitations under the License.
 set -e
 
+_testlib_this="${BASH_SOURCE[0]}"
+_testlib_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
 RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"}
 RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
@@ -280,12 +283,14 @@ cleanup_docker_images() {
 
 ## @description  Generate robot framework reports based on the saved results.
 generate_report(){
+  local title="${1:-${COMPOSE_ENV_NAME}}"
+  local dir="${2:-${RESULT_DIR}}"
 
   if command -v rebot > /dev/null 2>&1; then
      #Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output)
-     rebot --reporttitle "${COMPOSE_ENV_NAME}" -N "${COMPOSE_ENV_NAME}" -d "$RESULT_DIR" "$RESULT_DIR/robot-*.xml"
+     rebot --reporttitle "${title}" -N "${title}" -d "${dir}" "${dir}/*.xml"
   else
-     echo "Robot framework is not installed, the reports can be generated (sudo pip install robotframework)."
+     echo "Robot framework is not installed, the reports cannot be generated (sudo pip install robotframework)."
      exit 1
   fi
 }
@@ -298,7 +303,7 @@ copy_results() {
   local result_dir="${test_dir}/result"
   local test_dir_name=$(basename ${test_dir})
   if [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then
-    rebot --nostatusrc -N "${test_dir_name}" -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml"
+    rebot --nostatusrc -N "${test_dir_name}" -l NONE -r NONE -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml"
   fi
 
   cp "${result_dir}"/docker-*.log "${all_result_dir}"/
@@ -325,3 +330,63 @@ run_test_script() {
 
   return ${ret}
 }
+
+## @description Make `OZONE_VOLUME_OWNER` the owner of the `OZONE_VOLUME`
+##   directory tree (required in Github Actions runner environment)
+fix_data_dir_permissions() {
+  if [[ -n "${OZONE_VOLUME}" ]] && [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+    current_user=$(whoami)
+    if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+      chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+        || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+    fi
+  fi
+}
+
+## @description Define variables required for using Ozone docker image which
+##   includes binaries for a specific release
+## @param `ozone` image version
+prepare_for_binary_image() {
+  local v=$1
+
+  export OZONE_DIR=/opt/ozone
+  export OZONE_IMAGE="apache/ozone:${v}"
+}
+
+## @description Define variables required for using `ozone-runner` docker image
+##   (no binaries included)
+## @param `ozone-runner` image version (optional)
+prepare_for_runner_image() {
+  local default_version=${docker.ozone-runner.version} # set at build-time from Maven property
+  local runner_version=${OZONE_RUNNER_VERSION:-${default_version}} # may be specified by user running the test
+  local v=${1:-${runner_version}} # prefer explicit argument
+
+  export OZONE_DIR=/opt/hadoop
+  export OZONE_IMAGE="apache/ozone-runner:${v}"
+}
+
+## @description Print the logical version for a specific release
+## @param the release for which logical version should be printed
+get_logical_version() {
+  local v="$1"
+
+  # shellcheck source=/dev/null
+  echo $(source "${_testlib_dir}/versions/${v}.sh" && ozone_logical_version)
+}
+
+## @description Activate the version-specific behavior for a given release
+## @param the release for which definitions should be loaded
+load_version_specifics() {
+  local v="$1"
+
+  # shellcheck source=/dev/null
+  source "${_testlib_dir}/versions/${v}.sh"
+
+  ozone_version_load
+}
+
+## @description Deactivate the previously version-specific behavior,
+##   reverting to the current version's definitions
+unload_version_specifics() {
+  ozone_version_unload
+}

hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh → hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh

@@ -15,5 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=scmcli
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status'
+_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+export OZONE_UPGRADE_FROM="$(basename ${_mydir})"
+export RESULT_DIR="${_mydir}/result"
+
+cd "${_mydir}/.."
+source upgrade_to_current.sh

hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh → hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh

@@ -15,5 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OZONE_ADMIN_COMMAND=admin
-export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose'
+_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+export OZONE_UPGRADE_FROM="$(basename ${_mydir})"
+export RESULT_DIR="${_mydir}/result"
+
+cd "${_mydir}/.."
+source upgrade_to_current.sh

hadoop-ozone/dist/src/main/compose/upgrade/test.sh

@@ -15,59 +15,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-: "${OZONE_REPLICATION_FACTOR:=3}"
-: "${OZONE_UPGRADE_FROM:="0.5.0"}"
-: "${OZONE_UPGRADE_TO:="1.0.0"}"
-: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
-
-export OZONE_VOLUME
-
-mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-
-if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
-  current_user=$(whoami)
-  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
-    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
-      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
+ALL_RESULT_DIR="$SCRIPT_DIR/result"
+mkdir -p "$ALL_RESULT_DIR"
+rm "$ALL_RESULT_DIR"/* || true
+source "$SCRIPT_DIR/../testlib.sh"
+
+tests=$(find_tests)
+cd "$SCRIPT_DIR"
+
+RESULT=0
+# shellcheck disable=SC2044
+for t in ${tests}; do
+  d="$(dirname "${t}")"
+
+  if ! run_test_script "${d}"; then
+    RESULT=1
   fi
-fi
-
-# define version-specifics
-export OZONE_DIR=/opt/ozone
-export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_FROM}.sh"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
-
-# prepare pre-upgrade cluster
-start_docker_env
-execute_robot_test scm -v PREFIX:pre freon/generate.robot
-execute_robot_test scm -v PREFIX:pre freon/validate.robot
-KEEP_RUNNING=false stop_docker_env
-
-# run upgrade scripts
-SCRIPT_DIR=../../libexec/upgrade
-[[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh"
 
-# update version-specifics
-export OZONE_DIR=/opt/hadoop
-unset OZONE_IMAGE # use apache/ozone-runner defined in docker-compose.yaml
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_TO}.sh"
-# shellcheck source=/dev/null
-source "${COMPOSE_DIR}/../testlib.sh"
+  copy_results "${d}" "${ALL_RESULT_DIR}"
+done
 
-# re-start cluster with new version and check after upgrade
-export OZONE_KEEP_RESULTS=true
-start_docker_env
-execute_robot_test scm -v PREFIX:pre freon/validate.robot
-# test write key to old bucket after upgrade
-execute_robot_test scm -v PREFIX:post freon/generate.robot
-execute_robot_test scm -v PREFIX:post freon/validate.robot
-stop_docker_env
+generate_report "upgrade" "${ALL_RESULT_DIR}"
 
-generate_report
+exit ${RESULT}

hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -o pipefail
+
+_upgrade_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+: "${OZONE_VOLUME_OWNER:=}"
+
+source "${_upgrade_dir}/../testlib.sh"
+
+## @description Create the directory tree required for persisting data between
+##   compose cluster restarts
+create_data_dir() {
+  if [[ -z "${OZONE_VOLUME}" ]]; then
+    return 1
+  fi
+
+  rm -fr "${OZONE_VOLUME}" 2> /dev/null || sudo rm -fr "${OZONE_VOLUME}"
+  mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+  fix_data_dir_permissions
+}
+
+## @description Run upgrade steps required for going from one logical version to another.
+## @param Starting logical version
+## @param Target logical version
+execute_upgrade_steps() {
+  local -i from=$1
+  local -i to=$2
+
+  if [[ ${from} -ge ${to} ]]; then
+    return
+  fi
+
+  pushd ${_testlib_dir}/../libexec/upgrade
+
+  local v
+  for v in $(seq ${from} $((to-1))); do
+    if [[ -e "v$v.sh" ]]; then
+      source "v$v.sh"
+    fi
+  done
+
+  popd
+}
+
+## @description Pre-upgrade test steps
+first_run() {
+  start_docker_env
+  execute_robot_test scm -v PREFIX:pre freon/generate.robot
+  execute_robot_test scm -v PREFIX:pre freon/validate.robot
+  KEEP_RUNNING=false stop_docker_env
+}
+
+## @description Post-upgrade test steps
+second_run() {
+  export OZONE_KEEP_RESULTS=true
+  start_docker_env
+  execute_robot_test scm -v PREFIX:pre freon/validate.robot
+  # test write key to old bucket after upgrade
+  execute_robot_test scm -v PREFIX:post freon/generate.robot
+  execute_robot_test scm -v PREFIX:post freon/validate.robot
+  stop_docker_env
+}

hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script tests upgrade from a previous release to the current
+# binaries.  Docker image with Ozone binaries is required for the
+# initial version, while the snapshot version uses Ozone runner image.
+
+set -e -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${OZONE_REPLICATION_FACTOR:=3}"
+: "${OZONE_UPGRADE_FROM:="0.5.0"}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_REPLICATION_FACTOR OZONE_UPGRADE_FROM OZONE_VOLUME
+
+current_version=1.1.0
+
+source "${COMPOSE_DIR}/testlib.sh"
+
+create_data_dir
+
+prepare_for_binary_image "${OZONE_UPGRADE_FROM}"
+load_version_specifics "${OZONE_UPGRADE_FROM}"
+first_run
+unload_version_specifics
+
+from=$(get_logical_version "${OZONE_UPGRADE_FROM}")
+to=$(get_logical_version "${current_version}")
+execute_upgrade_steps "$from" "$to"
+
+prepare_for_runner_image
+second_run
+
+generate_report