HDDS-3855. Add upgrade smoketest

.github/workflows/post-commit.yml

@@ -135,6 +135,7 @@ jobs:
         env:
           KEEP_IMAGE: false
           OZONE_WITH_COVERAGE: true
+          OZONE_VOLUME_OWNER: 1000
       - uses: actions/upload-artifact@master
         if: always()
         with:

hadoop-ozone/dist/dev-support/bin/dist-layout-stitching

@@ -108,6 +108,7 @@ run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-config.cmd" "libexec/"
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-functions.sh" "libexec/"
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/ozone/ozone-config.sh" "libexec/"
 run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/shellprofile.d" "libexec/"
+run cp -r "${ROOT}/hadoop-ozone/dist/src/shell/upgrade" "libexec/"
 
 
 run cp "${ROOT}/hadoop-ozone/dist/src/shell/hdds/hadoop-daemons.sh" "sbin/"

hadoop-ozone/dist/pom.xml

@@ -28,7 +28,7 @@
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
-    <docker.ozone-runner.version>20200420-1</docker.ozone-runner.version>
+    <docker.ozone-runner.version>20200625-1</docker.ozone-runner.version>
   </properties>
 
   <build>

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -17,7 +17,6 @@
 set -e
 
 COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
-COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml
 RESULT_DIR=${RESULT_DIR:-"$COMPOSE_DIR/result"}
 RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
 SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
@@ -32,17 +31,17 @@ fi
 ## @description create results directory, purging any prior data
 create_results_dir() {
   #delete previous results
-  rm -rf "$RESULT_DIR"
+  [[ "${OZONE_KEEP_RESULTS:-}" == "true" ]] || rm -rf "$RESULT_DIR"
   mkdir -p "$RESULT_DIR"
   #Should be writeable from the docker containers where user is different.
   chmod ogu+w "$RESULT_DIR"
 }
 
 
 ## @description wait until safemode exit (or 180 seconds)
-## @param the docker-compose file
 wait_for_safemode_exit(){
-  local compose_file=$1
+  # version-dependent
+  : ${OZONE_ADMIN_COMMAND:=admin}
 
   #Reset the timer
   SECONDS=0
@@ -51,11 +50,11 @@ wait_for_safemode_exit(){
   while [[ $SECONDS -lt 180 ]]; do
 
      #This line checks the safemode status in scm
-     local command="ozone admin safemode status"
+     local command="ozone ${OZONE_ADMIN_COMMAND} safemode status"
      if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
-         status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/[email protected] -t /etc/security/keytabs/HTTP.keytab && $command" || true)
+         status=$(docker-compose exec -T scm bash -c "kinit -k HTTP/[email protected] -t /etc/security/keytabs/HTTP.keytab && $command" || true)
      else
-         status=$(docker-compose -f "${compose_file}" exec -T scm bash -c "$command")
+         status=$(docker-compose exec -T scm bash -c "$command")
      fi
 
      echo $status
@@ -80,9 +79,9 @@ start_docker_env(){
 
   create_results_dir
   export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
-  docker-compose -f "$COMPOSE_FILE" --no-ansi down
-  if ! { docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}" \
-      && wait_for_safemode_exit "$COMPOSE_FILE"; }; then
+  docker-compose --no-ansi down
+  if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \
+      && wait_for_safemode_exit ; }; then
     OUTPUT_NAME="$COMPOSE_ENV_NAME"
     stop_docker_env
     return 1
@@ -114,11 +113,11 @@ execute_robot_test(){
 
   OUTPUT_PATH="$RESULT_DIR_INSIDE/${OUTPUT_FILE}"
   # shellcheck disable=SC2068
-  docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
-    && docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
+  docker-compose exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
+    && docker-compose exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
-  FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+  FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
   copy_daemon_logs
@@ -135,7 +134,7 @@ execute_robot_test(){
 ## @description Copy any 'out' files for daemon processes to the result dir
 copy_daemon_logs() {
   local c f
-  for c in $(docker-compose -f "$COMPOSE_FILE" ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
+  for c in $(docker-compose ps | grep "^${COMPOSE_ENV_NAME}_" | awk '{print $1}'); do
     for f in $(docker exec "${c}" ls -1 /var/log/hadoop | grep -F '.out'); do
       docker cp "${c}:/var/log/hadoop/${f}" "$RESULT_DIR/"
     done
@@ -149,15 +148,15 @@ copy_daemon_logs() {
 execute_command_in_container(){
   set -e
   # shellcheck disable=SC2068
-  docker-compose -f "$COMPOSE_FILE" exec -T "$@"
+  docker-compose exec -T "$@"
   set +e
 }
 
 ## @description Stop a list of named containers
 ## @param       List of container names, eg datanode_1 datanode_2
 stop_containers() {
   set -e
-  docker-compose -f "$COMPOSE_FILE" --no-ansi stop $@
+  docker-compose --no-ansi stop $@
   set +e
 }
 
@@ -166,7 +165,7 @@ stop_containers() {
 ## @param       List of container names, eg datanode_1 datanode_2
 start_containers() {
   set -e
-  docker-compose -f "$COMPOSE_FILE" --no-ansi start $@
+  docker-compose --no-ansi start $@
   set +e
 }
 
@@ -185,7 +184,7 @@ wait_for_port(){
 
   while [[ $SECONDS -lt $timeout ]]; do
      set +e
-     docker-compose -f "${COMPOSE_FILE}" exec -T scm /bin/bash -c "nc -z $host $port"
+     docker-compose exec -T scm /bin/bash -c "nc -z $host $port"
      status=$?
      set -e
      if [ $status -eq 0 ] ; then
@@ -202,9 +201,9 @@ wait_for_port(){
 
 ## @description  Stops a docker-compose based test environment (with saving the logs)
 stop_docker_env(){
-  docker-compose -f "$COMPOSE_FILE" --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+  docker-compose --no-ansi logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
   if [ "${KEEP_RUNNING:-false}" = false ]; then
-     docker-compose -f "$COMPOSE_FILE" --no-ansi down
+     docker-compose --no-ansi down
   fi
 }
 

hadoop-ozone/dist/src/main/compose/upgrade/.env

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_IMAGE=apache/ozone-runner:${docker.ozone-runner.version}
+OZONE_DIR=/opt/hadoop
+OZONE_VOLUME=.

hadoop-ozone/dist/src/main/compose/upgrade/README.md

@@ -0,0 +1,29 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Compose file for upgrade
+
+This directory contains a sample cluster definition and script for
+testing upgrade from previous version to the current one.
+
+Data for each container is persisted in mounted volume (by default it's
+`data` under the `compose/upgrade` directory, but can be overridden via
+`OZONE_VOLUME` environment variable).
+
+Prior version is run using an official `apache/ozone` image, while the
+current version is run with the `ozone-runner` image using locally built
+source code.
+
+Currently the test script only supports a single version upgrade (eg.
+from 0.5.0 to 0.6.0).

hadoop-ozone/dist/src/main/compose/upgrade/docker-compose.yaml

@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
+x-common-config:
+  &common-config
+  env_file:
+    - docker-config
+  image: ${OZONE_IMAGE}
+
+x-replication:
+  &replication
+  OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
+
+x-datanode:
+  &datanode
+  command: ["ozone","datanode"]
+  <<: *common-config
+  environment:
+    <<: *replication
+  ports:
+    - 9864
+    - 9882
+
+services:
+  dn1:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.11
+    volumes:
+      - ${OZONE_VOLUME}/dn1:/data
+      - ../..:${OZONE_DIR}
+  dn2:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.12
+    volumes:
+      - ${OZONE_VOLUME}/dn2:/data
+      - ../..:${OZONE_DIR}
+  dn3:
+    <<: *datanode
+    networks:
+      net:
+        ipv4_address: 10.9.0.13
+    volumes:
+      - ${OZONE_VOLUME}/dn3:/data
+      - ../..:${OZONE_DIR}
+  om:
+    command: ["ozone","om"]
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.14
+    ports:
+      - 9874:9874
+      - 9862:9862
+    volumes:
+      - ${OZONE_VOLUME}/om:/data
+      - ../..:${OZONE_DIR}
+  recon:
+    command: ["ozone","recon"]
+    <<: *common-config
+    environment:
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.15
+    ports:
+      - 9888:9888
+    volumes:
+      - ${OZONE_VOLUME}/recon:/data
+      - ../..:${OZONE_DIR}
+  s3g:
+    command: ["ozone","s3g"]
+    <<: *common-config
+    environment:
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.16
+    ports:
+      - 9878:9878
+    volumes:
+      - ${OZONE_VOLUME}/s3g:/data
+      - ../..:${OZONE_DIR}
+  scm:
+    command: ["ozone","scm"]
+    <<: *common-config
+    environment:
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+      <<: *replication
+    networks:
+      net:
+        ipv4_address: 10.9.0.17
+    ports:
+      - 9876:9876
+    volumes:
+      - ${OZONE_VOLUME}/scm:/data
+      - ../..:${OZONE_DIR}
+
+networks:
+  net:
+    driver: bridge
+    ipam:
+      config:
+      - subnet: 10.9.0.0/16

hadoop-ozone/dist/src/main/compose/upgrade/docker-config

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
+CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
+
+no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1