HDDS-13405. ozone admin container create runs forever without kinit

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java

@@ -65,6 +65,7 @@
 import org.apache.hadoop.io.retry.RetryPolicies;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.ratis.util.ExitUtils;
 import org.apache.ratis.util.FileUtils;
@@ -347,21 +348,45 @@ public static List<String> getExistingSstFiles(File db) throws IOException {
 
   /**
    * Retry forever until CA list matches expected count.
+   * Fails fast on authentication exceptions.
    * @param task - task to get CA list.
    * @return CA list.
    */
   private static List<String> getCAListWithRetry(Callable<List<String>> task,
       long waitDuration) throws IOException {
-    RetryPolicy retryPolicy = RetryPolicies.retryForeverWithFixedSleep(
-        waitDuration, TimeUnit.SECONDS);
-    RetriableTask<List<String>> retriableTask =
-        new RetriableTask<>(retryPolicy, "getCAList", task);
+    RetryPolicy retryPolicy = new RetryPolicy() {
+      private final RetryPolicy defaultPolicy = RetryPolicies.retryForeverWithFixedSleep(
+          waitDuration, TimeUnit.SECONDS);
+
+      @Override
+      public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotent) throws Exception {
+        if (containsAccessControlException(e)) {
+          LOG.warn("AccessControlException encountered during getCAList; failing fast without retry.");
+          return new RetryAction(RetryAction.RetryDecision.FAIL);
+        }
+        return defaultPolicy.shouldRetry(e, retries, failovers, isIdempotent);
+      }
+    };
+
+    RetriableTask<List<String>> retriableTask = new RetriableTask<>(retryPolicy, "getCAList", task);
     try {
       return retriableTask.call();
     } catch (Exception ex) {
-      throw new SCMSecurityException("Unable to obtain complete CA " +
-          "list", ex);
+      if (containsAccessControlException(ex)) {
+        throw new AccessControlException();
+      }
+      throw new SCMSecurityException("Unable to obtain complete CA list", ex);
+    }
+  }
+
+  private static boolean containsAccessControlException(Throwable e) {
+    while (e != null) {
+      if (e instanceof AccessControlException) {
+        return true;
+      }
+      e = e.getCause();
     }
+    return false;
   }
 
   private static List<String> waitForCACerts(

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -35,6 +35,9 @@ start_docker_env
 
 execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
 
+#Run this test before kinit on a SCM HA secure cluster
+execute_robot_test s3g scmha/container-create.robot
+
 execute_robot_test s3g kinit.robot
 
 execute_robot_test s3g freon

hadoop-ozone/dist/src/main/smoketest/scmha/container-create.robot

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or moreD
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test ozone admin container create command without kinit on a SCM HA secure cluster
+Library             BuiltIn
+Resource            ../lib/os.robot
+
+*** Test Cases ***
+Create container without kinit
+    ${output} =         Execute And Ignore Error          ozone admin container create
+                        Should contain        ${output}   Permission denied