diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh index c52034899593..4e0151be9b96 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh @@ -17,7 +17,7 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd ) ALL_RESULT_DIR="$SCRIPT_DIR/result" mkdir -p "$ALL_RESULT_DIR" -rm "$ALL_RESULT_DIR/*" || true +rm "$ALL_RESULT_DIR"/* || true source "$SCRIPT_DIR/../testlib.sh" tests=$(find_tests) diff --git a/hadoop-ozone/dist/src/main/compose/restart/test.sh b/hadoop-ozone/dist/src/main/compose/restart/test.sh index 41e06c5ac816..cf0f53242da5 100644 --- a/hadoop-ozone/dist/src/main/compose/restart/test.sh +++ b/hadoop-ozone/dist/src/main/compose/restart/test.sh @@ -23,19 +23,12 @@ export COMPOSE_DIR export OZONE_VOLUME -mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm} - -if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then - current_user=$(whoami) - if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then - chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \ - || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" - fi -fi - # shellcheck source=/dev/null source "${COMPOSE_DIR}/../testlib.sh" +mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm} +fix_data_dir_permissions + # prepare pre-upgrade cluster start_docker_env execute_robot_test scm -v PREFIX:pre freon/generate.robot diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh index c51a96935200..3ef2dcfb9d81 100755 --- a/hadoop-ozone/dist/src/main/compose/test-all.sh +++ b/hadoop-ozone/dist/src/main/compose/test-all.sh @@ -23,7 +23,7 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd ) ALL_RESULT_DIR="$SCRIPT_DIR/result" PROJECT_DIR="$SCRIPT_DIR/.." mkdir -p "$ALL_RESULT_DIR" -rm "$ALL_RESULT_DIR/*" || true +rm "$ALL_RESULT_DIR"/* || true source "$SCRIPT_DIR"/testlib.sh diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 0e12fde032d7..3ba5bcd12723 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -16,6 +16,9 @@ # limitations under the License. set -e +_testlib_this="${BASH_SOURCE[0]}" +_testlib_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR") RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"} RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result" @@ -280,12 +283,14 @@ cleanup_docker_images() { ## @description Generate robot framework reports based on the saved results. generate_report(){ + local title="${1:-${COMPOSE_ENV_NAME}}" + local dir="${2:-${RESULT_DIR}}" if command -v rebot > /dev/null 2>&1; then #Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output) - rebot --reporttitle "${COMPOSE_ENV_NAME}" -N "${COMPOSE_ENV_NAME}" -d "$RESULT_DIR" "$RESULT_DIR/robot-*.xml" + rebot --reporttitle "${title}" -N "${title}" -d "${dir}" "${dir}/*.xml" else - echo "Robot framework is not installed, the reports can be generated (sudo pip install robotframework)." + echo "Robot framework is not installed, the reports cannot be generated (sudo pip install robotframework)." exit 1 fi } @@ -298,7 +303,7 @@ copy_results() { local result_dir="${test_dir}/result" local test_dir_name=$(basename ${test_dir}) if [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then - rebot --nostatusrc -N "${test_dir_name}" -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml" + rebot --nostatusrc -N "${test_dir_name}" -l NONE -r NONE -o "${all_result_dir}/${test_dir_name}.xml" "${result_dir}/*.xml" fi cp "${result_dir}"/docker-*.log "${all_result_dir}"/ @@ -325,3 +330,63 @@ run_test_script() { return ${ret} } + +## @description Make `OZONE_VOLUME_OWNER` the owner of the `OZONE_VOLUME` +## directory tree (required in Github Actions runner environment) +fix_data_dir_permissions() { + if [[ -n "${OZONE_VOLUME}" ]] && [[ -n "${OZONE_VOLUME_OWNER}" ]]; then + current_user=$(whoami) + if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then + chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \ + || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" + fi + fi +} + +## @description Define variables required for using Ozone docker image which +## includes binaries for a specific release +## @param `ozone` image version +prepare_for_binary_image() { + local v=$1 + + export OZONE_DIR=/opt/ozone + export OZONE_IMAGE="apache/ozone:${v}" +} + +## @description Define variables required for using `ozone-runner` docker image +## (no binaries included) +## @param `ozone-runner` image version (optional) +prepare_for_runner_image() { + local default_version=${docker.ozone-runner.version} # set at build-time from Maven property + local runner_version=${OZONE_RUNNER_VERSION:-${default_version}} # may be specified by user running the test + local v=${1:-${runner_version}} # prefer explicit argument + + export OZONE_DIR=/opt/hadoop + export OZONE_IMAGE="apache/ozone-runner:${v}" +} + +## @description Print the logical version for a specific release +## @param the release for which logical version should be printed +get_logical_version() { + local v="$1" + + # shellcheck source=/dev/null + echo $(source "${_testlib_dir}/versions/${v}.sh" && ozone_logical_version) +} + +## @description Activate the version-specific behavior for a given release +## @param the release for which definitions should be loaded +load_version_specifics() { + local v="$1" + + # shellcheck source=/dev/null + source "${_testlib_dir}/versions/${v}.sh" + + ozone_version_load +} + +## @description Deactivate the previously version-specific behavior, +## reverting to the current version's definitions +unload_version_specifics() { + ozone_version_unload +} diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh old mode 100644 new mode 100755 similarity index 79% rename from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh rename to hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh index 667ce959026a..11530c7f3d0c --- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/0.5.0/test.sh @@ -15,5 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -export OZONE_ADMIN_COMMAND=scmcli -export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status' +_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +export OZONE_UPGRADE_FROM="$(basename ${_mydir})" +export RESULT_DIR="${_mydir}/result" + +cd "${_mydir}/.." +source upgrade_to_current.sh diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh old mode 100644 new mode 100755 similarity index 79% rename from hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh rename to hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh index 3ff23e0441bc..11530c7f3d0c --- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-1.0.0.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/1.0.0/test.sh @@ -15,5 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -export OZONE_ADMIN_COMMAND=admin -export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose' +_mydir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +export OZONE_UPGRADE_FROM="$(basename ${_mydir})" +export RESULT_DIR="${_mydir}/result" + +cd "${_mydir}/.." +source upgrade_to_current.sh diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh old mode 100644 new mode 100755 index 1c16c81ab921..929be05bcc79 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -15,59 +15,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export COMPOSE_DIR - -: "${OZONE_REPLICATION_FACTOR:=3}" -: "${OZONE_UPGRADE_FROM:="0.5.0"}" -: "${OZONE_UPGRADE_TO:="1.0.0"}" -: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" - -export OZONE_VOLUME - -mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm} - -if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then - current_user=$(whoami) - if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then - chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \ - || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd ) +ALL_RESULT_DIR="$SCRIPT_DIR/result" +mkdir -p "$ALL_RESULT_DIR" +rm "$ALL_RESULT_DIR"/* || true +source "$SCRIPT_DIR/../testlib.sh" + +tests=$(find_tests) +cd "$SCRIPT_DIR" + +RESULT=0 +# shellcheck disable=SC2044 +for t in ${tests}; do + d="$(dirname "${t}")" + + if ! run_test_script "${d}"; then + RESULT=1 fi -fi - -# define version-specifics -export OZONE_DIR=/opt/ozone -export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}" -# shellcheck source=/dev/null -source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_FROM}.sh" -# shellcheck source=/dev/null -source "${COMPOSE_DIR}/../testlib.sh" - -# prepare pre-upgrade cluster -start_docker_env -execute_robot_test scm -v PREFIX:pre freon/generate.robot -execute_robot_test scm -v PREFIX:pre freon/validate.robot -KEEP_RUNNING=false stop_docker_env - -# run upgrade scripts -SCRIPT_DIR=../../libexec/upgrade -[[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" -# update version-specifics -export OZONE_DIR=/opt/hadoop -unset OZONE_IMAGE # use apache/ozone-runner defined in docker-compose.yaml -# shellcheck source=/dev/null -source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_TO}.sh" -# shellcheck source=/dev/null -source "${COMPOSE_DIR}/../testlib.sh" + copy_results "${d}" "${ALL_RESULT_DIR}" +done -# re-start cluster with new version and check after upgrade -export OZONE_KEEP_RESULTS=true -start_docker_env -execute_robot_test scm -v PREFIX:pre freon/validate.robot -# test write key to old bucket after upgrade -execute_robot_test scm -v PREFIX:post freon/generate.robot -execute_robot_test scm -v PREFIX:post freon/validate.robot -stop_docker_env +generate_report "upgrade" "${ALL_RESULT_DIR}" -generate_report +exit ${RESULT} diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh new file mode 100755 index 000000000000..1c6cfe6d2026 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +_upgrade_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +: "${OZONE_REPLICATION_FACTOR:=3}" +: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" +: "${OZONE_VOLUME_OWNER:=}" + +source "${_upgrade_dir}/../testlib.sh" + +## @description Create the directory tree required for persisting data between +## compose cluster restarts +create_data_dir() { + if [[ -z "${OZONE_VOLUME}" ]]; then + return 1 + fi + + rm -fr "${OZONE_VOLUME}" 2> /dev/null || sudo rm -fr "${OZONE_VOLUME}" + mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm} + fix_data_dir_permissions +} + +## @description Run upgrade steps required for going from one logical version to another. +## @param Starting logical version +## @param Target logical version +execute_upgrade_steps() { + local -i from=$1 + local -i to=$2 + + if [[ ${from} -ge ${to} ]]; then + return + fi + + pushd ${_testlib_dir}/../libexec/upgrade + + local v + for v in $(seq ${from} $((to-1))); do + if [[ -e "v$v.sh" ]]; then + source "v$v.sh" + fi + done + + popd +} + +## @description Pre-upgrade test steps +first_run() { + start_docker_env + execute_robot_test scm -v PREFIX:pre freon/generate.robot + execute_robot_test scm -v PREFIX:pre freon/validate.robot + KEEP_RUNNING=false stop_docker_env +} + +## @description Post-upgrade test steps +second_run() { + export OZONE_KEEP_RESULTS=true + start_docker_env + execute_robot_test scm -v PREFIX:pre freon/validate.robot + # test write key to old bucket after upgrade + execute_robot_test scm -v PREFIX:post freon/generate.robot + execute_robot_test scm -v PREFIX:post freon/validate.robot + stop_docker_env +} diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh new file mode 100755 index 000000000000..872885196e19 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_current.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script tests upgrade from a previous release to the current +# binaries. Docker image with Ozone binaries is required for the +# initial version, while the snapshot version uses Ozone runner image. + +set -e -o pipefail + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +: "${OZONE_REPLICATION_FACTOR:=3}" +: "${OZONE_UPGRADE_FROM:="0.5.0"}" +: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" + +export OZONE_REPLICATION_FACTOR OZONE_UPGRADE_FROM OZONE_VOLUME + +current_version=1.1.0 + +source "${COMPOSE_DIR}/testlib.sh" + +create_data_dir + +prepare_for_binary_image "${OZONE_UPGRADE_FROM}" +load_version_specifics "${OZONE_UPGRADE_FROM}" +first_run +unload_version_specifics + +from=$(get_logical_version "${OZONE_UPGRADE_FROM}") +to=$(get_logical_version "${current_version}") +execute_upgrade_steps "$from" "$to" + +prepare_for_runner_image +second_run + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh new file mode 100755 index 000000000000..5a9b4028a2e3 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrade_to_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script tests upgrade from one release to a later one. Docker +# image with Ozone binaries are required for both versions. + +set -e -o pipefail + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +: "${OZONE_REPLICATION_FACTOR:=3}" +: "${OZONE_UPGRADE_FROM:="0.5.0"}" +: "${OZONE_UPGRADE_TO:="1.0.0"}" +: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" + +export OZONE_REPLICATION_FACTOR OZONE_UPGRADE_FROM OZONE_UPGRADE_TO OZONE_VOLUME + +source "${COMPOSE_DIR}/testlib.sh" + +create_data_dir + +prepare_for_binary_image "${OZONE_UPGRADE_FROM}" +load_version_specifics "${OZONE_UPGRADE_FROM}" +first_run +unload_version_specifics + +from=$(get_logical_version "${OZONE_UPGRADE_FROM}") +to=$(get_logical_version "${OZONE_UPGRADE_TO}") +execute_upgrade_steps "$from" "$to" + +prepare_for_binary_image "${OZONE_UPGRADE_TO}" +load_version_specifics "${OZONE_UPGRADE_TO}" +second_run +unload_version_specifics + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md deleted file mode 100644 index 24cd113469a6..000000000000 --- a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md +++ /dev/null @@ -1,15 +0,0 @@ - - -The scripts in this directory define version-specific behavior required for [`testlib.sh`](../../testlib.sh). For example the `ozone admin` command was renamed from `ozone scmcli` in 1.0.0. diff --git a/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh b/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh new file mode 100644 index 000000000000..f973a04b73cb --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ozone_logical_version() { + echo 1 +} + +ozone_version_load() { + export OZONE_ADMIN_COMMAND=scmcli + export OZONE_SAFEMODE_STATUS_COMMAND='ozone scmcli safemode status' +} + +ozone_version_unload() { + unset OZONE_ADMIN_COMMAND + unset OZONE_SAFEMODE_STATUS_COMMAND +} diff --git a/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh b/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh new file mode 100644 index 000000000000..c56aa3319836 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ozone_logical_version() { + echo 2 +} + +ozone_version_load() { + export OZONE_ADMIN_COMMAND=admin + export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose' +} + +ozone_version_unload() { + unset OZONE_ADMIN_COMMAND + unset OZONE_SAFEMODE_STATUS_COMMAND +} diff --git a/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh b/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh new file mode 100644 index 000000000000..c56aa3319836 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ozone_logical_version() { + echo 2 +} + +ozone_version_load() { + export OZONE_ADMIN_COMMAND=admin + export OZONE_SAFEMODE_STATUS_COMMAND='ozone admin safemode status --verbose' +} + +ozone_version_unload() { + unset OZONE_ADMIN_COMMAND + unset OZONE_SAFEMODE_STATUS_COMMAND +} diff --git a/hadoop-ozone/dist/src/main/compose/versions/README.md b/hadoop-ozone/dist/src/main/compose/versions/README.md new file mode 100644 index 000000000000..f6dc62cb335a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/versions/README.md @@ -0,0 +1,23 @@ + + +The scripts in this directory define version-specific behavior required for [`testlib.sh`](../testlib.sh). For example the `ozone admin` command was renamed from `ozone scmcli` in 1.0.0. + +For each release a logical version is defined, starting from 1. It is incremented only if the new version needs to execute some scripts to handle some (unintended) incompatibility. For example 1.0.0 or later need to rename some RocksDB column-families when upgrading from 0.5.0. + +Interface: + + * `ozone_logical_version`: print the logical version + * `ozone_version_load`: define version-specific variables for the test library + * `ozone_version_unload`: unset version-specific variables; this reverts test library behavior to the "current" one. diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh index 27c5ef4bfa96..7d7ee1da4f1d 100755 --- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh +++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh @@ -21,7 +21,7 @@ basename=$(basename ${COMPOSE_DIR}) current_version=1.1.0 -# shellcheck source=/dev/null +# shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh source "${COMPOSE_DIR}/../testlib.sh" old_client() { @@ -81,13 +81,12 @@ create_results_dir COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${current_version} test_cross_compatibility for cluster_version in 1.0.0; do - # shellcheck source=/dev/null - source "${COMPOSE_DIR}/../upgrade/versions/ozone-${cluster_version}.sh" - # shellcheck source=/dev/null - source "${COMPOSE_DIR}/../testlib.sh" + load_version_specifics ${cluster_version} export OZONE_VERSION=${cluster_version} COMPOSE_FILE=old-cluster.yaml:clients.yaml test_cross_compatibility + + unload_version_specifics done generate_report diff --git a/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh b/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh index ae810c9f679c..5d866514be9f 100755 --- a/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh +++ b/hadoop-ozone/dist/src/main/k8s/examples/test-all.sh @@ -24,7 +24,7 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd ) set -ex ALL_RESULT_DIR="$SCRIPT_DIR/result" -rm "$ALL_RESULT_DIR/*" || true +rm "$ALL_RESULT_DIR"/* || true mkdir -p "$ALL_RESULT_DIR" RESULT=0 diff --git a/hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh b/hadoop-ozone/dist/src/shell/upgrade/v1.sh similarity index 85% rename from hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh rename to hadoop-ozone/dist/src/shell/upgrade/v1.sh index 65739787ee67..1442ffdf799e 100755 --- a/hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh +++ b/hadoop-ozone/dist/src/shell/upgrade/v1.sh @@ -20,4 +20,4 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" : "${SCM_DIR:="${OZONE_VOLUME}/scm"}" : "${OZONE_RUNNER_VERSION:="20200625-1"}" -docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/1.0.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh +docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/v1":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh diff --git a/hadoop-ozone/dist/src/shell/upgrade/1.0.0/01-migrate-scm-db.sh b/hadoop-ozone/dist/src/shell/upgrade/v1/01-migrate-scm-db.sh similarity index 100% rename from hadoop-ozone/dist/src/shell/upgrade/1.0.0/01-migrate-scm-db.sh rename to hadoop-ozone/dist/src/shell/upgrade/v1/01-migrate-scm-db.sh