From dd535966e9e8a2922204a28be558abfdef859480 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Tue, 23 Jun 2020 16:15:31 +0200
Subject: [PATCH 01/21] HDDS-3855. Add upgrade smoketest

---
 hadoop-ozone/dev-support/upgrade/0.6.0.sh     |  24 ++++
 .../upgrade/0.6.0/01-migrate-scm-db.sh        |  24 ++++
 hadoop-ozone/dist/pom.xml                     |   2 +-
 hadoop-ozone/dist/src/main/compose/testlib.sh |  39 +++---
 .../dist/src/main/compose/upgrade/.env        |  21 +++
 .../dist/src/main/compose/upgrade/README.md   |  29 ++++
 .../main/compose/upgrade/docker-compose.yaml  | 127 ++++++++++++++++++
 .../src/main/compose/upgrade/docker-config    |  33 +++++
 .../dist/src/main/compose/upgrade/test.sh     |  61 +++++++++
 .../main/compose/upgrade/versions/README.md   |  15 +++
 .../compose/upgrade/versions/ozone-0.5.0.sh   |  18 +++
 .../compose/upgrade/versions/ozone-0.6.0.sh   |  18 +++
 12 files changed, 390 insertions(+), 21 deletions(-)
 create mode 100755 hadoop-ozone/dev-support/upgrade/0.6.0.sh
 create mode 100755 hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/.env
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/README.md
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/docker-config
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/test.sh
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
 create mode 100644 hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh

diff --git a/hadoop-ozone/dev-support/upgrade/0.6.0.sh b/hadoop-ozone/dev-support/upgrade/0.6.0.sh
new file mode 100755
index 000000000000..0759d2fb80f9
--- /dev/null
+++ b/hadoop-ozone/dev-support/upgrade/0.6.0.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+: ${OZONE_VOLUME:="/tmp/ozone-data"}
+: ${SCM_DIR:="${OZONE_VOLUME}/scm"}
+: ${OZONE_RUNNER_VERSION:="20200625-1"}
+
+docker run -it --rm -v ${SCM_DIR}:/scm -v ${SCRIPT_DIR}/0.6.0:/upgrade -w /scm/metadata apache/ozone-runner:${OZONE_RUNNER_VERSION} /upgrade/01-migrate-scm-db.sh
diff --git a/hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh b/hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh
new file mode 100755
index 000000000000..dee676841972
--- /dev/null
+++ b/hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo Running upgrade script for HDDS-3499
+
+ldb --db=scm.db create_column_family containers
+ldb --db=scm.db create_column_family pipelines
+
+ldb --db=scm-container.db --key_hex --value_hex dump | ldb --db=scm.db --key_hex --value_hex --column_family=containers load
+ldb --db=scm-pipeline.db --key_hex --value_hex dump | ldb --db=scm.db --key_hex --value_hex --column_family=pipelines load
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index 0929b2410021..db7add34a309 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -28,7 +28,7 @@
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
-    <docker.ozone-runner.version>20200420-1</docker.ozone-runner.version>
+    <docker.ozone-runner.version>20200625-1</docker.ozone-runner.version>
   </properties>
 
   <build>
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 15d1664ed80c..56c35c186243 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -17,7 +17,6 @@
 set -e
 
 COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
-COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml
 RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"}
 RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
 SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
@@ -32,7 +31,7 @@ fi
 ## @description create results directory, purging any prior data
 create_results_dir() {
   #delete previous results
-  rm -rf "$RESULT_DIR"
+  [[ "${OZONE_KEEP_RESULTS:-}" == "true" ]] || rm -rf "$RESULT_DIR"
   mkdir -p "$RESULT_DIR"
   #Should be writeable from the docker containers where user is different.
   chmod ogu+w "$RESULT_DIR"
@@ -40,9 +39,9 @@ create_results_dir() {
 
 
 ## @description wait until safemode exit (or 180 seconds)
-## @param the docker-compose file
 wait_for_safemode_exit(){
-  local compose_file=$1
+  # version-dependent
+  : ${OZONE_ADMIN_COMMAND:=admin}
 
   #Reset the timer
   SECONDS=0
@@ -51,11 +50,11 @@ wait_for_safemode_exit(){
   while [[ $SECONDS -lt 180 ]]; do
 
      #This line checks the safemode status in scm
-     local command="ozone admin safemode status"
+     local command="ozone ${OZONE_ADMIN_COMMAND} safemode status"
      if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
-         status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab && $command" || true)
+         status=$(docker-compose exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab && $command" || true)
      else
-         status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "$command")
+         status=$(docker-compose exec -T scm bash -c "$command")
      fi
 
      echo $status
@@ -80,9 +79,9 @@ start_docker_env(){
 
   create_results_dir
   export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
-  docker-compose -f "$COMPOSE_FILE" --no-ansi down
-  if ! { docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}" \
-      && wait_for_safemode_exit "$COMPOSE_FILE"; }; then
+  docker-compose --no-ansi down
+  if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \
+      && wait_for_safemode_exit ; }; then
     OUTPUT_NAME="$COMPOSE_ENV_NAME"
     stop_docker_env
     return 1
@@ -114,11 +113,11 @@ execute_robot_test(){
 
   OUTPUT_PATH="$RESULT_DIR_INSIDE/${OUTPUT_FILE}"
   # shellcheck disable=SC2068
-  docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
-    && docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
+  docker-compose exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
+    && docker-compose exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   copy_daemon_logs
@@ -135,7 +134,7 @@ execute_robot_test(){
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose -f "$COMPOSE_FILE" ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop | grep -F '.out'); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done
@@ -149,7 +148,7 @@ copy_daemon_logs() {
 execute_command_in_container(){
   set -e
   # shellcheck disable=SC2068
-  docker-compose -f "$COMPOSE_FILE" exec -T "$@"
+  docker-compose exec -T "$@"
   set +e
 }
 
@@ -157,7 +156,7 @@ execute_command_in_container(){
 ## @param       List of container names, eg datanode_1 datanode_2
 stop_containers() {
   set -e
-  docker-compose -f "$COMPOSE_FILE" --no-ansi stop $@
+  docker-compose --no-ansi stop $@
   set +e
 }
 
@@ -166,7 +165,7 @@ stop_containers() {
 ## @param       List of container names, eg datanode_1 datanode_2
 start_containers() {
   set -e
-  docker-compose -f "$COMPOSE_FILE" --no-ansi start $@
+  docker-compose --no-ansi start $@
   set +e
 }
 
@@ -185,7 +184,7 @@ wait_for_port(){
 
   while [[ $SECONDS -lt $timeout ]]; do
      set +e
-     docker-compose -f "${COMPOSE_FILE}" exec -T scm /bin/bash -c "nc -z $host $port"
+     docker-compose exec -T scm /bin/bash -c "nc -z $host $port"
      status=$?
      set -e
      if [ $status -eq 0 ] ; then
@@ -202,9 +201,9 @@ wait_for_port(){
 
 ## @description  Stops a docker-compose based test environment (with saving the logs)
 stop_docker_env(){
-  docker-compose -f "$COMPOSE_FILE" --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+  docker-compose --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
   if [ "${KEEP_RUNNING:-false}" = false ]; then
-     docker-compose -f "$COMPOSE_FILE" --no-ansi down
+     docker-compose --no-ansi down
   fi
 }
 
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/.env b/hadoop-ozone/dist/src/main/compose/upgrade/.env
new file mode 100644
index 000000000000..fe72aec712a4
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/.env
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_DIR=/opt/hadoop
+OZONE_VOLUME=/tmp/ozone-data
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
new file mode 100644
index 000000000000..8b515086b4e2
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
@@ -0,0 +1,29 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Compose file for upgrade
+
+This directory contains a sample cluster definition and script for
+testing upgrade from previous version to the current one.
+
+Data for each container is persisted in mounted volume
+(`/tmp/ozone-data` by default, but can be overridden via `OZONE_VOLUME`
+environment variable).
+
+Prior version is run using an official `apache/ozone` image, while the
+current version is run with the `ozone-runner` image using locally built
+source code.
+
+Currently the test script only supports a single version upgrade (eg.
+from 0.5.0 to 0.6.0).
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
new file mode 100644
index 000000000000..ae527593dcd4
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
+x-common-config:
+  &common-config
+  env_file:
+    - docker-config
+  image: ${OZONE_IMAGE}
+
+x-replication:
+  &replication
+  OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
+
+x-datanode:
+  &datanode
+  command: ["ozone","datanode"]
+  <<: *common-config
+  environment:
+    <<: *replication
+  ports:
+    - 9864
+    - 9882
+
+services:
+  dn1:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.11
+    volumes:
+      - ${OZONE_VOLUME}/dn1:/data
+      - ../..:${OZONE_DIR}
+  dn2:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.12
+    volumes:
+      - ${OZONE_VOLUME}/dn2:/data
+      - ../..:${OZONE_DIR}
+  dn3:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.13
+    volumes:
+      - ${OZONE_VOLUME}/dn3:/data
+      - ../..:${OZONE_DIR}
+  om:
+    command: ["ozone","om"]
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.14
+    ports:
+      - 9874:9874
+      - 9862:9862
+    volumes:
+      - ${OZONE_VOLUME}/om:/data
+      - ../..:${OZONE_DIR}
+  recon:
+    command: ["ozone","recon"]
+    <<: *common-config
+    environment:
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.15
+    ports:
+      - 9888:9888
+    volumes:
+      - ${OZONE_VOLUME}/recon:/data
+      - ../..:${OZONE_DIR}
+  s3g:
+    command: ["ozone","s3g"]
+    <<: *common-config
+    environment:
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.16
+    ports:
+      - 9878:9878
+    volumes:
+      - ${OZONE_VOLUME}/s3g:/data
+      - ../..:${OZONE_DIR}
+  scm:
+    command: ["ozone","scm"]
+    <<: *common-config
+    environment:
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.17
+    ports:
+      - 9876:9876
+    volumes:
+      - ${OZONE_VOLUME}/scm:/data
+      - ../..:${OZONE_DIR}
+
+networks:
+  net:
+    driver: bridge
+    ipam:
+      config:
+      - subnet: 10.9.0.0/16
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/docker-config
new file mode 100644
index 000000000000..ecc994b4ac26
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-config
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
+CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
+
+no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
new file mode 100644
index 000000000000..9ac94c2b89a4
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+: "${OZONE_UPGRADE_FROM:="0.5.0"}"
+: "${OZONE_UPGRADE_TO:="0.6.0"}"
+
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+
+export OZONE_VOLUME=/tmp/ozone-data
+rm -fr ${OZONE_VOLUME}
+
+# define version-specifics
+export OZONE_DIR=/opt/ozone
+export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}"
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_FROM}.sh"
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/../testlib.sh"
+
+# prepare pre-upgrade cluster
+start_docker_env
+execute_robot_test scm topology/loaddata.robot
+stop_docker_env
+
+# run upgrade scripts
+SCRIPT_DIR=../../../../../dev-support/upgrade
+[[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh"
+
+# update version-specifics
+export OZONE_DIR=/opt/hadoop
+unset OZONE_IMAGE # use apache/ozone-runner defined in docker-compose.yaml
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/versions/ozone-${OZONE_UPGRADE_TO}.sh"
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/../testlib.sh"
+
+# re-start cluster with new version and check after upgrade
+export OZONE_KEEP_RESULTS=true
+start_docker_env
+execute_robot_test scm topology/readdata.robot
+stop_docker_env
+
+generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md
new file mode 100644
index 000000000000..c662c2f286be
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/README.md
@@ -0,0 +1,15 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+The scripts in this directory define version-specific behavior required for [`testlib.sh`](../../testlib.sh).  For example the `ozone admin` command was renamed from `ozone scmcli` in 0.6.0.
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
new file mode 100644
index 000000000000..68f6c75764df
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.5.0.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export OZONE_ADMIN_COMMAND=scmcli
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh
new file mode 100644
index 000000000000..bab1dba91baf
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/versions/ozone-0.6.0.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export OZONE_ADMIN_COMMAND=admin

From ea07c03d65dafa124666c7bb48411743be72bc7e Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 26 Jun 2020 09:56:10 +0200
Subject: [PATCH 02/21] trigger new CI check


From 60a69e9a589dd67b7043eebb5bc4104697e10c82 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 26 Jun 2020 12:14:07 +0200
Subject: [PATCH 03/21] only run upgrade test

---
 .github/workflows/post-commit.yml             | 143 ------------------
 .../dist/src/main/compose/test-all.sh         |   2 +-
 2 files changed, 1 insertion(+), 144 deletions(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 35636c27419c..3c7cf23377ef 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -19,92 +19,9 @@ on:
   schedule:
     - cron: 30 0,12 * * *
 jobs:
-  build:
-    name: compile
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@master
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.pnpm-store
-            **/node_modules
-          key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-
-      - uses: ./.github/buildenv
-        with:
-          args: ./hadoop-ozone/dev-support/checks/build.sh
-  rat:
-    name: rat
-    runs-on: ubuntu-18.04
-    steps:
-        - uses: actions/checkout@master
-        - uses: ./.github/buildenv
-          with:
-             args: ./hadoop-ozone/dev-support/checks/rat.sh
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: rat
-            path: target/rat
-  author:
-    name: author
-    runs-on: ubuntu-18.04
-    steps:
-        - uses: actions/checkout@master
-        - uses: ./.github/buildenv
-          with:
-             args: ./hadoop-ozone/dev-support/checks/author.sh
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: author
-            path: target/author
-  unit:
-    name: unit
-    runs-on: ubuntu-18.04
-    steps:
-        - uses: actions/checkout@master
-        - uses: ./.github/buildenv
-          with:
-             args: ./hadoop-ozone/dev-support/checks/unit.sh
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: unit
-            path: target/unit
-  checkstyle:
-    name: checkstyle
-    runs-on: ubuntu-18.04
-    steps:
-        - uses: actions/checkout@master
-        - uses: ./.github/buildenv
-          with:
-             args: ./hadoop-ozone/dev-support/checks/checkstyle.sh
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: checkstyle
-            path: target/checkstyle
-  findbugs:
-    name: findbugs
-    runs-on: ubuntu-18.04
-    steps:
-        - uses: actions/checkout@master
-        - uses: ./.github/buildenv
-          with:
-             args: ./hadoop-ozone/dev-support/checks/findbugs.sh
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: findbugs
-            path: target/findbugs
   acceptance:
     name: acceptance
     runs-on: ubuntu-18.04
-    needs:
-      - build
     steps:
         - uses: actions/checkout@master
         - uses: actions/cache@v2
@@ -133,63 +50,3 @@ jobs:
           with:
             name: acceptance
             path: /mnt/ozone/compose/result
-  integration:
-    name: integration
-    runs-on: ubuntu-18.04
-    needs:
-        - build
-    strategy:
-      matrix:
-        profile:
-          - client
-          - filesystem
-          - filesystem-contract
-          - freon
-          - hdds-om
-          - ozone
-      fail-fast: false
-    steps:
-        - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
-        - uses: actions/checkout@master
-          with:
-            path: mnt/ozone
-        - uses: ./mnt/ozone/.github/buildenv
-          with:
-             args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }}
-        - uses: actions/upload-artifact@master
-          if: always()
-          with:
-            name: it-${{ matrix.profile }}
-            path: mnt/ozone/target/integration
-  coverage:
-    name: coverage
-    runs-on: ubuntu-18.04
-    needs:
-       - integration
-       - unit
-    steps:
-       - uses: actions/checkout@v2
-       - uses: ./.github/buildenv
-         with:
-            args: ./hadoop-ozone/dev-support/checks/build.sh
-       - uses: actions/download-artifact@v2
-         with:
-            path: target/artifacts
-       - run: ./hadoop-ozone/dev-support/checks/coverage.sh
-       - uses: ./.github/buildenv
-         if: github.repository == 'apache/hadoop-ozone' && github.event_name != 'pull_request'
-         with:
-           args: ./hadoop-ozone/dev-support/checks/sonar.sh
-         env:
-           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
-           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-       - name: Upload coverage to Codecov
-         uses: codecov/codecov-action@v1
-         with:
-           file: ./target/coverage/all.xml
-           name: codecov-umbrella
-           fail_ci_if_error: true
-       - uses: actions/upload-artifact@master
-         with:
-           name: coverage
-           path: target/coverage
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index 809123b5e0ac..594b7a6e8897 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -29,7 +29,7 @@ rm "$ALL_RESULT_DIR/*"
 RESULT=0
 IFS=$'\n'
 # shellcheck disable=SC2044
-for test in $(find "$SCRIPT_DIR" -name test.sh | sort); do
+for test in $(find "$SCRIPT_DIR" -path '*/upgrade/test.sh' | sort); do
   echo "Executing test in $(dirname "$test")"
 
   #required to read the .env file from the right location

From 9749205b2902b1b62a8c018283769e4b94c39a69 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 26 Jun 2020 12:14:18 +0200
Subject: [PATCH 04/21] create volume root

---
 hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 9ac94c2b89a4..364b4ad0032a 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -26,6 +26,7 @@ export OZONE_REPLICATION_FACTOR=3
 
 export OZONE_VOLUME=/tmp/ozone-data
 rm -fr ${OZONE_VOLUME}
+mkdir -p ${OZONE_VOLUME}
 
 # define version-specifics
 export OZONE_DIR=/opt/ozone

From ffb36d04be25a40086c4ab691e602900ea16b18c Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 26 Jun 2020 13:18:36 +0200
Subject: [PATCH 05/21] avoid /tmp as default

---
 hadoop-ozone/dev-support/upgrade/0.6.0.sh            | 7 +++----
 hadoop-ozone/dist/src/main/compose/upgrade/README.md | 6 +++---
 hadoop-ozone/dist/src/main/compose/upgrade/test.sh   | 9 +++------
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/hadoop-ozone/dev-support/upgrade/0.6.0.sh b/hadoop-ozone/dev-support/upgrade/0.6.0.sh
index 0759d2fb80f9..b53b76df394a 100755
--- a/hadoop-ozone/dev-support/upgrade/0.6.0.sh
+++ b/hadoop-ozone/dev-support/upgrade/0.6.0.sh
@@ -17,8 +17,7 @@
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-: ${OZONE_VOLUME:="/tmp/ozone-data"}
-: ${SCM_DIR:="${OZONE_VOLUME}/scm"}
-: ${OZONE_RUNNER_VERSION:="20200625-1"}
+: "${SCM_DIR:="${OZONE_VOLUME}/scm"}"
+: "${OZONE_RUNNER_VERSION:="20200625-1"}"
 
-docker run -it --rm -v ${SCM_DIR}:/scm -v ${SCRIPT_DIR}/0.6.0:/upgrade -w /scm/metadata apache/ozone-runner:${OZONE_RUNNER_VERSION} /upgrade/01-migrate-scm-db.sh
+docker run -it --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/0.6.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
index 8b515086b4e2..2a832f4b9e29 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/README.md
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md
@@ -17,9 +17,9 @@
 This directory contains a sample cluster definition and script for
 testing upgrade from previous version to the current one.
 
-Data for each container is persisted in mounted volume
-(`/tmp/ozone-data` by default, but can be overridden via `OZONE_VOLUME`
-environment variable).
+Data for each container is persisted in mounted volume (by default it's
+`data` under the `compose/upgrade` directory, but can be overridden via
+`OZONE_VOLUME` environment variable).
 
 Prior version is run using an official `apache/ozone` image, while the
 current version is run with the `ozone-runner` image using locally built
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 364b4ad0032a..a0b10711e99c 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -18,15 +18,12 @@
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
 
+: "${OZONE_REPLICATION_FACTOR:=3}"
 : "${OZONE_UPGRADE_FROM:="0.5.0"}"
 : "${OZONE_UPGRADE_TO:="0.6.0"}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
 
-export SECURITY_ENABLED=false
-export OZONE_REPLICATION_FACTOR=3
-
-export OZONE_VOLUME=/tmp/ozone-data
-rm -fr ${OZONE_VOLUME}
-mkdir -p ${OZONE_VOLUME}
+export OZONE_VOLUME
 
 # define version-specifics
 export OZONE_DIR=/opt/ozone

From 87649d078703457bbda16a951487c95143a44d80 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 26 Jun 2020 13:33:28 +0200
Subject: [PATCH 06/21] create data directory

---
 .github/workflows/post-commit.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 3c7cf23377ef..cc04d7e61123 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -41,10 +41,11 @@ jobs:
         - run: docker image rm apache/ozone-build || true
         - run: sudo pip install robotframework
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
-        - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
+        - run: cd /mnt/ozone && mkdir .aws data && sudo chown 1000 .aws data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
+            OZONE_VOLUME: /mnt/ozone/data
         - uses: actions/upload-artifact@master
           if: always()
           with:

From 5c160126feaa7329c8a4b51815279cbb22708608 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 07:26:02 +0200
Subject: [PATCH 07/21] fixup for ffb36d04b

---
 hadoop-ozone/dist/src/main/compose/upgrade/.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/.env b/hadoop-ozone/dist/src/main/compose/upgrade/.env
index fe72aec712a4..6f757c552b2e 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/.env
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/.env
@@ -18,4 +18,4 @@ HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
 OZONE_DIR=/opt/hadoop
-OZONE_VOLUME=/tmp/ozone-data
+OZONE_VOLUME=.

From b8e0aaa4532379ab31c2c2c9168000f931e2dbc3 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 07:52:08 +0200
Subject: [PATCH 08/21] debug

---
 .github/workflows/post-commit.yml                         | 4 +++-
 .../dist/src/main/compose/upgrade/docker-compose.yaml     | 8 ++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index cc04d7e61123..fa6aced0f1bd 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -41,7 +41,9 @@ jobs:
         - run: docker image rm apache/ozone-build || true
         - run: sudo pip install robotframework
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
-        - run: cd /mnt/ozone && mkdir .aws data && sudo chown 1000 .aws data
+        - run: id
+        - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
+        - run: cd /mnt/ozone && mkdir data && sudo chown runner data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
index ae527593dcd4..a74d0592c48f 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
@@ -103,6 +103,14 @@ services:
     volumes:
       - ${OZONE_VOLUME}/s3g:/data
       - ../..:${OZONE_DIR}
+  debug:
+    command: "bash -c 'id; echo ===; mount; echo ===; df -k; echo ===; ls -la /data'"
+    <<: *common-config
+    networks:
+      net:
+    volumes:
+      - ${OZONE_VOLUME}/scm:/data
+      - ../..:${OZONE_DIR}
   scm:
     command: ["ozone","scm"]
     <<: *common-config

From 7562d5da923b459a2f8b2068209f6551cf8580f2 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 08:08:50 +0200
Subject: [PATCH 09/21] own 1000

---
 .github/workflows/post-commit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index fa6aced0f1bd..e837bcf9b5d1 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -43,7 +43,7 @@ jobs:
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
         - run: id
         - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
-        - run: cd /mnt/ozone && mkdir data && sudo chown runner data
+        - run: cd /mnt/ozone && mkdir data && sudo chown 1000 data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false

From 89869095514a811c1b324b154c402f3d71cf81e2 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 08:09:58 +0200
Subject: [PATCH 10/21] use default OZONE_VOLUME

---
 .github/workflows/post-commit.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index e837bcf9b5d1..fef54ee69669 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -43,11 +43,9 @@ jobs:
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
         - run: id
         - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
-        - run: cd /mnt/ozone && mkdir data && sudo chown 1000 data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
-            OZONE_VOLUME: /mnt/ozone/data
         - uses: actions/upload-artifact@master
           if: always()
           with:

From f1fa5ee8d2d632730b448fd68754f15a49ec6c50 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 08:28:52 +0200
Subject: [PATCH 11/21] Revert "use default OZONE_VOLUME"

This reverts commit 89869095514a811c1b324b154c402f3d71cf81e2.
---
 .github/workflows/post-commit.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index fef54ee69669..e837bcf9b5d1 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -43,9 +43,11 @@ jobs:
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
         - run: id
         - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
+        - run: cd /mnt/ozone && mkdir data && sudo chown 1000 data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
+            OZONE_VOLUME: /mnt/ozone/data
         - uses: actions/upload-artifact@master
           if: always()
           with:

From e20b770595acbfc24643c7cd4c9001371055ea53 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 08:32:55 +0200
Subject: [PATCH 12/21] more debug

---
 .github/workflows/post-commit.yml                   |  3 +--
 .../src/main/compose/upgrade/docker-compose.yaml    |  2 +-
 hadoop-ozone/dist/src/main/compose/upgrade/test.sh  | 13 +++++++++++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index e837bcf9b5d1..20ab9b73b43f 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -43,11 +43,10 @@ jobs:
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
         - run: id
         - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
-        - run: cd /mnt/ozone && mkdir data && sudo chown 1000 data
+        - run: cd /mnt/ozone/compose && mkdir data && sudo chown 1000 data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
-            OZONE_VOLUME: /mnt/ozone/data
         - uses: actions/upload-artifact@master
           if: always()
           with:
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
index a74d0592c48f..7f87315498c1 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
@@ -109,8 +109,8 @@ services:
     networks:
       net:
     volumes:
-      - ${OZONE_VOLUME}/scm:/data
       - ../..:${OZONE_DIR}
+      - ${OZONE_VOLUME}/debug:/data
   scm:
     command: ["ozone","scm"]
     <<: *common-config
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index a0b10711e99c..4f8e806a2cc0 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -25,6 +25,19 @@ export COMPOSE_DIR
 
 export OZONE_VOLUME
 
+echo ===
+echo ${OZONE_VOLUME}
+echo ===
+mkdir -pv ${OZONE_VOLUME}/{dn1,dn2,dn3,om,recon,s3g,scm}
+mkdir -pv ${OZONE_VOLUME}/debug
+echo ===
+df -k
+echo ===
+mount
+echo ===
+ls -la ${OZONE_VOLUME}
+echo ===
+
 # define version-specifics
 export OZONE_DIR=/opt/ozone
 export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}"

From 5f53ec02d7bd9e079c4985206517f472405c811b Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 09:12:19 +0200
Subject: [PATCH 13/21] chown -R OZONE_VOLUME_OWNER

---
 .github/workflows/post-commit.yml             |  2 +-
 .../dist/src/main/compose/upgrade/test.sh     | 21 +++++++++++--------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 20ab9b73b43f..d355104759cd 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -43,10 +43,10 @@ jobs:
         - run: sudo mv ./hadoop-ozone/dist/target/ozone-* /mnt/ozone && sudo chown runner -R /mnt/ozone
         - run: id
         - run: cd /mnt/ozone && mkdir .aws && sudo chown 1000 .aws
-        - run: cd /mnt/ozone/compose && mkdir data && sudo chown 1000 data
         - run: cd /mnt/ozone/compose && ./test-all.sh
           env:
             KEEP_IMAGE: false
+            OZONE_VOLUME_OWNER: 1000
         - uses: actions/upload-artifact@master
           if: always()
           with:
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 4f8e806a2cc0..ab559edbe13d 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -26,16 +26,19 @@ export COMPOSE_DIR
 export OZONE_VOLUME
 
 echo ===
-echo ${OZONE_VOLUME}
-echo ===
-mkdir -pv ${OZONE_VOLUME}/{dn1,dn2,dn3,om,recon,s3g,scm}
-mkdir -pv ${OZONE_VOLUME}/debug
-echo ===
-df -k
-echo ===
-mount
+mkdir -pv "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+mkdir -pv "${OZONE_VOLUME}/debug"
+
+if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+  current_user=$(whoami)
+  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+  fi
+fi
+
 echo ===
-ls -la ${OZONE_VOLUME}
+ls -la "${OZONE_VOLUME}"
 echo ===
 
 # define version-specifics

From fffe8dec55b26ff559b196717a2ad94eb9fb0916 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 13:53:38 +0200
Subject: [PATCH 14/21] debug upgrade script

---
 hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index ab559edbe13d..51d6eaeb5ab9 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -25,9 +25,8 @@ export COMPOSE_DIR
 
 export OZONE_VOLUME
 
-echo ===
-mkdir -pv "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
-mkdir -pv "${OZONE_VOLUME}/debug"
+mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+mkdir -p "${OZONE_VOLUME}/debug"
 
 if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
   current_user=$(whoami)
@@ -37,10 +36,6 @@ if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
   fi
 fi
 
-echo ===
-ls -la "${OZONE_VOLUME}"
-echo ===
-
 # define version-specifics
 export OZONE_DIR=/opt/ozone
 export OZONE_IMAGE="apache/ozone:${OZONE_UPGRADE_FROM}"
@@ -54,6 +49,8 @@ start_docker_env
 execute_robot_test scm topology/loaddata.robot
 stop_docker_env
 
+set -x
+
 # run upgrade scripts
 SCRIPT_DIR=../../../../../dev-support/upgrade
 [[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh"

From fa8651242f9a52ad7e8264818b80feffeb91ef73 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 15:36:34 +0200
Subject: [PATCH 15/21] move upgrade scripts to libexec

---
 hadoop-ozone/dist/dev-support/bin/dist-layout-stitching       | 1 +
 hadoop-ozone/dist/src/main/compose/upgrade/test.sh            | 4 +---
 hadoop-ozone/{dev-support => dist/src/shell}/upgrade/0.6.0.sh | 0
 .../src/shell}/upgrade/0.6.0/01-migrate-scm-db.sh             | 0
 4 files changed, 2 insertions(+), 3 deletions(-)
 rename hadoop-ozone/{dev-support => dist/src/shell}/upgrade/0.6.0.sh (100%)
 rename hadoop-ozone/{dev-support => dist/src/shell}/upgrade/0.6.0/01-migrate-scm-db.sh (100%)

diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index e1f5c7e55c5e..80455a62953a 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -108,6 +108,7 @@ run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-config.cmd" "libexec/"
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh" "libexec/"
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/"
 run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/shellprofile.d" "libexec/"
+run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/upgrade" "libexec/"
 
 
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-daemons.sh" "sbin/"
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 51d6eaeb5ab9..0babd17676d5 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -49,10 +49,8 @@ start_docker_env
 execute_robot_test scm topology/loaddata.robot
 stop_docker_env
 
-set -x
-
 # run upgrade scripts
-SCRIPT_DIR=../../../../../dev-support/upgrade
+SCRIPT_DIR=../../libexec/upgrade
 [[ -f "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh" ]] && "${SCRIPT_DIR}/${OZONE_UPGRADE_TO}.sh"
 
 # update version-specifics
diff --git a/hadoop-ozone/dev-support/upgrade/0.6.0.sh b/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh
similarity index 100%
rename from hadoop-ozone/dev-support/upgrade/0.6.0.sh
rename to hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh
diff --git a/hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh b/hadoop-ozone/dist/src/shell/upgrade/0.6.0/01-migrate-scm-db.sh
similarity index 100%
rename from hadoop-ozone/dev-support/upgrade/0.6.0/01-migrate-scm-db.sh
rename to hadoop-ozone/dist/src/shell/upgrade/0.6.0/01-migrate-scm-db.sh

From 8ef93dc917f12c2d7e9b34d3135226aa0a2538c2 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 15:54:51 +0200
Subject: [PATCH 16/21] no -it for docker

---
 hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh b/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh
index b53b76df394a..58c78dbdfb64 100755
--- a/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh
+++ b/hadoop-ozone/dist/src/shell/upgrade/0.6.0.sh
@@ -20,4 +20,4 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 : "${SCM_DIR:="${OZONE_VOLUME}/scm"}"
 : "${OZONE_RUNNER_VERSION:="20200625-1"}"
 
-docker run -it --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/0.6.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh
+docker run --rm -v "${SCM_DIR}":/scm -v "${SCRIPT_DIR}/0.6.0":/upgrade -w /scm/metadata apache/ozone-runner:"${OZONE_RUNNER_VERSION}" /upgrade/01-migrate-scm-db.sh

From c8d3d533576d1d4a44f3ad25b7d815a2b7f42128 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 16:47:09 +0200
Subject: [PATCH 17/21] Revert "only run upgrade test"

This reverts commit 60a69e9a589dd67b7043eebb5bc4104697e10c82.
---
 .github/workflows/post-commit.yml             | 143 ++++++++++++++++++
 .../dist/src/main/compose/test-all.sh         |   2 +-
 2 files changed, 144 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index d355104759cd..fe37da452db0 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -19,9 +19,92 @@ on:
   schedule:
     - cron: 30 0,12 * * *
 jobs:
+  build:
+    name: compile
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.pnpm-store
+            **/node_modules
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+      - uses: ./.github/buildenv
+        with:
+          args: ./hadoop-ozone/dev-support/checks/build.sh
+  rat:
+    name: rat
+    runs-on: ubuntu-18.04
+    steps:
+        - uses: actions/checkout@master
+        - uses: ./.github/buildenv
+          with:
+             args: ./hadoop-ozone/dev-support/checks/rat.sh
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: rat
+            path: target/rat
+  author:
+    name: author
+    runs-on: ubuntu-18.04
+    steps:
+        - uses: actions/checkout@master
+        - uses: ./.github/buildenv
+          with:
+             args: ./hadoop-ozone/dev-support/checks/author.sh
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: author
+            path: target/author
+  unit:
+    name: unit
+    runs-on: ubuntu-18.04
+    steps:
+        - uses: actions/checkout@master
+        - uses: ./.github/buildenv
+          with:
+             args: ./hadoop-ozone/dev-support/checks/unit.sh
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: unit
+            path: target/unit
+  checkstyle:
+    name: checkstyle
+    runs-on: ubuntu-18.04
+    steps:
+        - uses: actions/checkout@master
+        - uses: ./.github/buildenv
+          with:
+             args: ./hadoop-ozone/dev-support/checks/checkstyle.sh
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: checkstyle
+            path: target/checkstyle
+  findbugs:
+    name: findbugs
+    runs-on: ubuntu-18.04
+    steps:
+        - uses: actions/checkout@master
+        - uses: ./.github/buildenv
+          with:
+             args: ./hadoop-ozone/dev-support/checks/findbugs.sh
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: findbugs
+            path: target/findbugs
   acceptance:
     name: acceptance
     runs-on: ubuntu-18.04
+    needs:
+      - build
     steps:
         - uses: actions/checkout@master
         - uses: actions/cache@v2
@@ -52,3 +135,63 @@ jobs:
           with:
             name: acceptance
             path: /mnt/ozone/compose/result
+  integration:
+    name: integration
+    runs-on: ubuntu-18.04
+    needs:
+        - build
+    strategy:
+      matrix:
+        profile:
+          - client
+          - filesystem
+          - filesystem-contract
+          - freon
+          - hdds-om
+          - ozone
+      fail-fast: false
+    steps:
+        - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
+        - uses: actions/checkout@master
+          with:
+            path: mnt/ozone
+        - uses: ./mnt/ozone/.github/buildenv
+          with:
+             args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }}
+        - uses: actions/upload-artifact@master
+          if: always()
+          with:
+            name: it-${{ matrix.profile }}
+            path: mnt/ozone/target/integration
+  coverage:
+    name: coverage
+    runs-on: ubuntu-18.04
+    needs:
+       - integration
+       - unit
+    steps:
+       - uses: actions/checkout@v2
+       - uses: ./.github/buildenv
+         with:
+            args: ./hadoop-ozone/dev-support/checks/build.sh
+       - uses: actions/download-artifact@v2
+         with:
+            path: target/artifacts
+       - run: ./hadoop-ozone/dev-support/checks/coverage.sh
+       - uses: ./.github/buildenv
+         if: github.repository == 'apache/hadoop-ozone' && github.event_name != 'pull_request'
+         with:
+           args: ./hadoop-ozone/dev-support/checks/sonar.sh
+         env:
+           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+       - name: Upload coverage to Codecov
+         uses: codecov/codecov-action@v1
+         with:
+           file: ./target/coverage/all.xml
+           name: codecov-umbrella
+           fail_ci_if_error: true
+       - uses: actions/upload-artifact@master
+         with:
+           name: coverage
+           path: target/coverage
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index 594b7a6e8897..809123b5e0ac 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -29,7 +29,7 @@ rm "$ALL_RESULT_DIR/*"
 RESULT=0
 IFS=$'\n'
 # shellcheck disable=SC2044
-for test in $(find "$SCRIPT_DIR" -path '*/upgrade/test.sh' | sort); do
+for test in $(find "$SCRIPT_DIR" -name test.sh | sort); do
   echo "Executing test in $(dirname "$test")"
 
   #required to read the .env file from the right location

From 03cbcd07b8587c52eeee22c32a97985973394ce4 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 27 Jun 2020 20:19:20 +0200
Subject: [PATCH 18/21] trigger new CI check


From 9123592b6f0816c352f9e4523c5d8cdccf24ab1b Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sun, 28 Jun 2020 08:34:09 +0200
Subject: [PATCH 19/21] trigger new CI check


From 3687fe3ba590aff98be3f44979a62094195825f3 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sun, 28 Jun 2020 11:56:17 +0200
Subject: [PATCH 20/21] trigger new CI check


From 9dbbcbb7e1719a5bba94533e49674556d4b1e362 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sun, 28 Jun 2020 14:50:49 +0200
Subject: [PATCH 21/21] Remove leftover debug

---
 .../dist/src/main/compose/upgrade/docker-compose.yaml     | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
index 7f87315498c1..ae527593dcd4 100644
--- a/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml
@@ -103,14 +103,6 @@ services:
     volumes:
       - ${OZONE_VOLUME}/s3g:/data
       - ../..:${OZONE_DIR}
-  debug:
-    command: "bash -c 'id; echo ===; mount; echo ===; df -k; echo ===; ls -la /data'"
-    <<: *common-config
-    networks:
-      net:
-    volumes:
-      - ../..:${OZONE_DIR}
-      - ${OZONE_VOLUME}/debug:/data
   scm:
     command: ["ozone","scm"]
     <<: *common-config