HDDS-1255. Refactor ozone acceptance test to allow run in secure mode. Contributed by Ajay Kumar.

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -108,6 +108,7 @@ LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
 LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
 
 OZONE_DATANODE_SECURE_USER=root
+SECURITY_ENABLED=true
 KEYTAB_DIR=/etc/security/keytabs
 KERBEROS_KEYTABS=dn om scm HTTP testuser s3g
 KERBEROS_KEYSTORES=hadoop

hadoop-ozone/dist/src/main/smoketest/__init__.robot

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*** Settings ***
+Documentation       Smoketest ozone secure cluster
+Resource            commonlib.robot
+Suite Setup         Run Keyword if    '${SECURITY_ENABLED}' == 'true'     Kinit test user

hadoop-ozone/dist/src/main/smoketest/auditparser/auditparser.robot

@@ -16,10 +16,20 @@
 *** Settings ***
 Documentation       Smoketest ozone cluster startup
 Library             OperatingSystem
+Library             BuiltIn
 Resource            ../commonlib.robot
 
-*** Test Cases ***
+*** Variables ***
+${user}        hadoop
+${count}       4
+
+*** Keywords ***
+Set username
+    ${hostname} =          Execute         hostname
+    Set Suite Variable     ${user}         testuser/${hostname}@EXAMPLE.COM
+    [return]               ${user}
 
+*** Test Cases ***
 Initiating freon to generate data
     ${result} =        Execute              ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 1
                        Wait Until Keyword Succeeds      3min       10sec     Should contain   ${result}   Number of Keys added: 125
@@ -31,10 +41,13 @@ Testing audit parser
     ${result} =        Execute              ozone auditparser /opt/hadoop/audit.db template top5cmds
                        Should Contain       ${result}  ALLOCATE_KEY
     ${result} =        Execute              ozone auditparser /opt/hadoop/audit.db template top5users
-                       Should Contain       ${result}  hadoop
+    Run Keyword If     '${SECURITY_ENABLED}' == 'true'      Set username
+                       Should Contain       ${result}  ${user}
     ${result} =        Execute              ozone auditparser /opt/hadoop/audit.db query "select count(*) from audit where op='CREATE_VOLUME' and RESULT='SUCCESS'"
-                       Should Contain       ${result}  5
+    ${result} =        Convert To Number     ${result}
+                       Should be true       ${result}>${count}
     ${result} =        Execute              ozone auditparser /opt/hadoop/audit.db query "select count(*) from audit where op='CREATE_BUCKET' and RESULT='SUCCESS'"
-                       Should Contain       ${result}  5
+    ${result} =        Convert To Number     ${result}
+                       Should be true       ${result}>${count}
     ${result} =        Execute              ozone auditparser /opt/hadoop/audit.db query "select count(*) from audit where RESULT='FAILURE'"
                        Should Contain       ${result}  0

hadoop-ozone/dist/src/main/smoketest/commonlib.robot

@@ -13,9 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-*** Keywords ***
+*** Settings ***
+Library             OperatingSystem
+Library             String
+Library             BuiltIn
 
+*** Variables ***
+${SECURITY_ENABLED}                 %{SECURITY_ENABLED}
 
+*** Keywords ***
 Execute
     [arguments]                     ${command}
     ${rc}                           ${output} =                 Run And Return Rc And Output           ${command}
@@ -35,3 +41,14 @@ Compare files
     ${checksumbefore} =         Execute                    md5sum ${file1} | awk '{print $1}'
     ${checksumafter} =          Execute                    md5sum ${file2} | awk '{print $1}'
                                 Should Be Equal            ${checksumbefore}            ${checksumafter}
+
+Install aws cli
+    ${rc}              ${output} =                 Run And Return Rc And Output           which apt-get
+    Run Keyword if     '${rc}' == '0'              Install aws cli s3 debian
+    ${rc}              ${output} =                 Run And Return Rc And Output           yum --help
+    Run Keyword if     '${rc}' == '0'              Install aws cli s3 centos
+
+Kinit test user
+    ${hostname} =       Execute                    hostname
+    Set Suite Variable  ${TEST_USER}               testuser/${hostname}@EXAMPLE.COM
+    Execute             kinit -k ${TEST_USER} -t /etc/security/keytabs/testuser.keytab

hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot

@@ -15,58 +15,66 @@
 
 *** Settings ***
 Resource            ../commonlib.robot
+Resource            ../commonlib.robot
 
 *** Variables ***
 ${OZONE_S3_HEADER_VERSION}     v4
 ${OZONE_S3_SET_CREDENTIALS}    true
+${BUCKET}                      bucket-999
 
 *** Keywords ***
 Execute AWSS3APICli
     [Arguments]       ${command}
-    ${output} =       Execute          aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
+    ${output} =       Execute                    aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
     [return]          ${output}
 
 Execute AWSS3APICli and checkrc
-    [Arguments]       ${command}             ${expected_error_code}
-    ${output} =       Execute and checkrc    aws s3api --endpoint-url ${ENDPOINT_URL} ${command}  ${expected_error_code}
+    [Arguments]       ${command}                 ${expected_error_code}
+    ${output} =       Execute and checkrc        aws s3api --endpoint-url ${ENDPOINT_URL} ${command}  ${expected_error_code}
     [return]          ${output}
 
 Execute AWSS3Cli
     [Arguments]       ${command}
-    ${output} =       Execute          aws s3 --endpoint-url ${ENDPOINT_URL} ${command}
+    ${output} =       Execute                     aws s3 --endpoint-url ${ENDPOINT_URL} ${command}
     [return]          ${output}
 
-Install aws cli
-    ${rc}              ${output} =                 Run And Return Rc And Output           which apt-get
-    Run Keyword if     '${rc}' == '0'              Install aws cli s3 debian
-    ${rc}              ${output} =                 Run And Return Rc And Output           yum --help
-    Run Keyword if     '${rc}' == '0'              Install aws cli s3 centos
-
-
 Install aws cli s3 centos
-    Execute                    sudo yum install -y awscli
+    Execute            sudo yum install -y awscli
+
 Install aws cli s3 debian
-    Execute                    sudo apt-get install -y awscli
+    Execute            sudo apt-get install -y awscli
 
 Setup v2 headers
                         Set Environment Variable   AWS_ACCESS_KEY_ID       ANYID
                         Set Environment Variable   AWS_SECRET_ACCESS_KEY   ANYKEY
 
 Setup v4 headers
+    ${result} =         Execute                    ozone s3 getsecret
+    ${accessKey} =      Get Regexp Matches         ${result}     (?<=awsAccessKey=).*
+    ${accessKey} =      Get Variable Value         ${accessKey}  sdsdasaasdasd
+    ${secret} =         Get Regexp Matches         ${result}     (?<=awsSecret=).*
+
+    ${len}=             Get Length  ${accessKey}
+    ${accessKey}=       Set Variable If   ${len} > 0  ${accessKey[0]}    kljdfslff
+    ${len}=             Get Length  ${secret}
+    ${secret}=          Set Variable If    ${len} > 0  ${secret[0]}      dhafldhlf
                         Execute                    aws configure set default.s3.signature_version s3v4
-                        Execute                    aws configure set aws_access_key_id default1
-                        Execute                    aws configure set aws_secret_access_key defaultsecret
+                        Execute                    aws configure set aws_access_key_id ${accessKey}
+                        Execute                    aws configure set aws_secret_access_key ${secret}
                         Execute                    aws configure set region us-west-1
+
+Setup incorrect credentials for S3
+                        Execute                    aws configure set default.s3.signature_version s3v4
+                        Execute                    aws configure set aws_access_key_id dlfknslnfslf
+                        Execute                    aws configure set aws_secret_access_key dlfknslnfslf
+                        Execute                    aws configure set region us-west-1
+
 Create bucket
     ${postfix} =         Generate Random String  5  [NUMBERS]
     Set Suite Variable   ${BUCKET}                  bucket-${postfix}
     Execute AWSS3APICli  create-bucket --bucket ${BUCKET}
 
-Setup credentials
-    Run Keyword if    '${OZONE_S3_HEADER_VERSION}' == 'v4'       Setup v4 headers
-    Run Keyword if    '${OZONE_S3_HEADER_VERSION}' != 'v4'       Setup v2 headers
-
 Setup s3 tests
     Run Keyword        Install aws cli
-    Run Keyword if    '${OZONE_S3_SET_CREDENTIALS}' == 'true'    Setup credentials
+    Run Keyword if    '${OZONE_S3_SET_CREDENTIALS}' == 'true'    Setup v4 headers
     Run Keyword if    '${BUCKET}' == 'generated'                 Create bucket

hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-fs.robot

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test to start cluster with docker-compose environments.
+Library             OperatingSystem
+Library             String
+Library             BuiltIn
+Resource            ../commonlib.robot
+
+*** Variables ***
+${ENDPOINT_URL}    http://s3g:9878
+
+*** Keywords ***
+Setup volume names
+    ${random}            Generate Random String  2   [NUMBERS]
+    Set Suite Variable   ${volume1}            fstest${random}
+    Set Suite Variable   ${volume2}            fstest2${random}
+
+*** Test Cases ***
+Create volume bucket with wrong credentials
+    Execute             kdestroy
+    ${rc}               ${output} =          Run And Return Rc And Output       ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+    Should contain      ${output}       Client cannot authenticate via
+
+Create volume bucket with credentials
+                        # Authenticate testuser
+    Run Keyword         Kinit test user
+    Run Keyword         Setup volume names
+    Execute             ozone sh volume create o3://om/${volume1} --user bilbo --quota 100TB --root
+    Execute             ozone sh volume create o3://om/${volume2} --user bilbo --quota 100TB --root
+    Execute             ozone sh bucket create o3://om/${volume1}/bucket1
+    Execute             ozone sh bucket create o3://om/${volume1}/bucket2
+    Execute             ozone sh bucket create o3://om/${volume2}/bucket3
+
+Check volume from ozonefs
+    ${result} =         Execute          ozone fs -ls o3fs://bucket1.${volume1}/

hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test to start cluster with docker-compose environments.
+Library             OperatingSystem
+Library             String
+Library             BuiltIn
+Resource            ../commonlib.robot
+Resource            ../s3/commonawslib.robot
+
+*** Variables ***
+${ENDPOINT_URL}     http://s3g:9878
+
+*** Keywords ***
+Setup volume names
+    ${random}            Generate Random String  2   [NUMBERS]
+    Set Suite Variable   ${volume1}            fstest${random}
+    Set Suite Variable   ${volume2}            fstest2${random}
+
+*** Test Cases ***
+Secure S3 test Success
+    Run Keyword         Setup s3 tests
+    ${output} =         Execute          aws s3api --endpoint-url ${ENDPOINT_URL} create-bucket --bucket bucket-test123
+    ${output} =         Execute          aws s3api --endpoint-url ${ENDPOINT_URL} list-buckets
+                        Should contain   ${output}         bucket-test123
+
+Secure S3 test Failure
+    Run Keyword         Setup incorrect credentials for S3
+    ${rc}  ${result} =  Run And Return Rc And Output  aws s3api --endpoint-url ${ENDPOINT_URL} create-bucket --bucket bucket-test123
+    Should Be True	${rc} > 0
+