apache · dineshchitlangia · Oct 23, 2019 · Oct 11, 2019
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/.env b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   datanode:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true #required by the profiler
+      volumes:
+        - ../..:/opt/hadoop
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+      env_file:
+        - ./docker-config
+   om1:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true #required by the profiler
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9880:9874
+         - 9890:9872
+      environment:
+         ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","om"]
+   om2:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true #required by the profiler
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9882:9874
+         - 9892:9872
+      environment:
+         ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","om"]
+   om3:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true #required by the profiler
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9884:9874
+         - 9894:9872
+      environment:
+         ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","om"]
+   scm:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true #required by the profiler
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]
+   s3g:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9878:9878
+      env_file:
+         - ./docker-config
+      command: ["ozone","s3g"]
+
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
+OZONE-SITE.XML_ozone.om.service.ids=id1
+OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
+OZONE-SITE.XML_ozone.om.address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.ratis.enable=true
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=false
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test scm s3
+
+stop_docker_env
+
+generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
@@ -18,6 +18,8 @@
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
 
+export SECURITY_ENABLED=false
+
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 

diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -50,21 +50,22 @@ Setup v2 headers
 
 Setup v4 headers
     Run Keyword if      '${SECURITY_ENABLED}' == 'true'     Kinit test user    testuser    testuser.keytab
+    Run Keyword if      '${SECURITY_ENABLED}' == 'true'     Setup secure v4 headers
+    Run Keyword if      '${SECURITY_ENABLED}' == 'false'    Setup dummy credentials for S3
+
+Setup secure v4 headers
     ${result} =         Execute                    ozone s3 getsecret
     ${accessKey} =      Get Regexp Matches         ${result}     (?<=awsAccessKey=).*
     ${accessKey} =      Get Variable Value         ${accessKey}  sdsdasaasdasd
     ${secret} =         Get Regexp Matches         ${result}     (?<=awsSecret=).*
-
-    ${len}=             Get Length  ${accessKey}
-    ${accessKey}=       Set Variable If   ${len} > 0  ${accessKey[0]}    kljdfslff
-    ${len}=             Get Length  ${secret}
-    ${secret}=          Set Variable If    ${len} > 0  ${secret[0]}      dhafldhlf
+    ${accessKey} =      Set Variable               ${accessKey[0]}
+    ${secret} =         Set Variable               ${secret[0]}
                         Execute                    aws configure set default.s3.signature_version s3v4
                         Execute                    aws configure set aws_access_key_id ${accessKey}
                         Execute                    aws configure set aws_secret_access_key ${secret}
                         Execute                    aws configure set region us-west-1
 
-Setup incorrect credentials for S3
+Setup dummy credentials for S3
                         Execute                    aws configure set default.s3.signature_version s3v4
                         Execute                    aws configure set aws_access_key_id dlfknslnfslf
                         Execute                    aws configure set aws_secret_access_key dlfknslnfslf

diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot
@@ -38,7 +38,7 @@ Secure S3 test Success
                         Should contain   ${output}         bucket-test123
 
 Secure S3 test Failure
-    Run Keyword         Setup incorrect credentials for S3
+    Run Keyword         Setup dummy credentials for S3
     ${rc}  ${result} =  Run And Return Rc And Output  aws s3api --endpoint-url ${ENDPOINT_URL} create-bucket --bucket bucket-test123
     Should Be True	${rc} > 0