HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTransport support

.github/workflows/post-commit.yml

@@ -125,6 +125,12 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v2
+        if: matrix.check != 'bats'
+      - name: Checkout project with history
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+        if: matrix.check == 'bats'
       - name: Cache for maven dependencies
         uses: actions/cache@v2
         with:

.gitignore

@@ -68,6 +68,7 @@ hadoop-ozone/recon/node_modules
 .mvn
 
 .dev-tools
-
+dev-support/ci/bats-assert
+dev-support/ci/bats-support
 
 hadoop-ozone/dist/src/main/license/current.txt

CONTRIBUTING.md

@@ -20,7 +20,8 @@ We welcome contributions of:
       * Unit Tests (JUnit / Java)
       * Acceptance Tests (docker + robot framework)
       * Blockade tests (python + blockade) 
-      * Performance: We have multiple type of load generator / benchmark tools (`ozone freon`, `ozone genesis`), which can be used to test cluster and report problems.
+      * Performance: We have multiple type of load generator / benchmark tools (`ozone freon`),
+        which can be used to test cluster and report problems.
  * **Bug reports** pointing out broken functionality, docs, or suggestions for improvements are always welcome!
 
 ## Who To Contact

dev-support/ci/selective_ci_checks.bats

@@ -66,6 +66,17 @@ load bats-assert/load.bash
   assert_output -p needs-kubernetes-tests=true
 }
 
+@test "runner image update" {
+  run dev-support/ci/selective_ci_checks.sh b95eeba82a
+
+  assert_output -p 'basic-checks=["rat"]'
+  assert_output -p needs-build=true
+  assert_output -p needs-compose-tests=true
+  assert_output -p needs-dependency-check=true
+  assert_output -p needs-integration-tests=false
+  assert_output -p needs-kubernetes-tests=true
+}
+
 @test "check script" {
   run dev-support/ci/selective_ci_checks.sh 316899152
 
@@ -77,10 +88,21 @@ load bats-assert/load.bash
   assert_output -p needs-kubernetes-tests=true
 }
 
+@test "integration and unit" {
+  run dev-support/ci/selective_ci_checks.sh 9aebf6e25
+
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p needs-build=false
+  assert_output -p needs-compose-tests=false
+  assert_output -p needs-dependency-check=false
+  assert_output -p needs-integration-tests=true
+  assert_output -p needs-kubernetes-tests=false
+}
+
 @test "integration only" {
   run dev-support/ci/selective_ci_checks.sh 61396ba9f
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false

dev-support/ci/selective_ci_checks.sh

@@ -219,7 +219,10 @@ function get_count_compose_files() {
     start_end::group_start "Count compose files"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/acceptance.sh"
-        "^hadoop-ozone/dist/src/main/compose"
+        "^hadoop-ozone/dist"
+    )
+    local ignore_array=(
+        "^hadoop-ozone/dist/src/main/k8s"
     )
     filter_changed_files true
     COUNT_COMPOSE_CHANGED_FILES=${match_count}
@@ -258,7 +261,10 @@ function get_count_kubernetes_files() {
     start_end::group_start "Count kubernetes files"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/kubernetes.sh"
-        "^hadoop-ozone/dist/src/main/k8s"
+        "^hadoop-ozone/dist"
+    )
+    local ignore_array=(
+        "^hadoop-ozone/dist/src/main/compose"
     )
     filter_changed_files true
     COUNT_KUBERNETES_CHANGED_FILES=${match_count}
@@ -332,6 +338,9 @@ function check_needs_checkstyle() {
         "pom.xml"
         "src/..../java"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then
@@ -373,6 +382,9 @@ function check_needs_findbugs() {
         "pom.xml"
         "src/..../java"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then
@@ -391,6 +403,11 @@ function check_needs_unit_test() {
         "src/..../java"
         "src/..../resources"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+        "^hadoop-ozone/fault-injection-test/mini-chaos-tests"
+        "^hadoop-ozone/integration-test"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -159,7 +159,7 @@ public void connect(String encodedToken) throws Exception {
 
   private synchronized void connectToDatanode(DatanodeDetails dn)
       throws IOException {
-    if (isConnected(dn)){
+    if (isConnected(dn)) {
       return;
     }
     // read port from the data node, on failure use default configured
@@ -269,10 +269,10 @@ public ContainerCommandResponseProto sendCommand(
         Thread.currentThread().interrupt();
       }
     }
-    try{
+    try {
       for (Map.Entry<DatanodeDetails,
               CompletableFuture<ContainerCommandResponseProto> >
-              entry : futureHashMap.entrySet()){
+              entry : futureHashMap.entrySet()) {
         responseProtoHashMap.put(entry.getKey(), entry.getValue().get());
       }
     } catch (InterruptedException e) {
@@ -538,7 +538,7 @@ public void onCompleted() {
   }
 
   private synchronized void checkOpen(DatanodeDetails dn)
-      throws IOException{
+      throws IOException {
     if (closed) {
       throw new IOException("This channel is not connected.");
     }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java

@@ -206,8 +206,8 @@ public static void verifyKeyName(String keyName) {
     if (keyName == null) {
       throw new IllegalArgumentException("Key name is null");
     }
-    if(!OzoneConsts.KEYNAME_ILLEGAL_CHARACTER_CHECK_REGEX
-            .matcher(keyName).matches()){
+    if (!OzoneConsts.KEYNAME_ILLEGAL_CHARACTER_CHECK_REGEX
+            .matcher(keyName).matches()) {
       throw new IllegalArgumentException("Invalid key name: " + keyName);
     }
   }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java

@@ -152,7 +152,7 @@ public synchronized void initialize() throws IOException {
         // retry according to retry policy.
         chunks = getChunkInfos();
         break;
-      } catch(SCMSecurityException ex) {
+      } catch (SCMSecurityException ex) {
         throw ex;
       } catch (StorageContainerException ex) {
         refreshPipeline(ex);
@@ -340,9 +340,9 @@ synchronized int readWithStrategy(ByteReaderStrategy strategy) throws
         } else {
           throw e;
         }
-      } catch(SCMSecurityException ex) {
+      } catch (SCMSecurityException ex) {
         throw ex;
-      } catch(IOException ex) {
+      } catch (IOException ex) {
         // We got a IOException which might be due
         // to DN down or connectivity issue.
         if (shouldRetryRead(ex)) {
@@ -512,7 +512,7 @@ synchronized long getBlockPosition() {
   }
 
   @Override
-  public void unbuffer() {
+  public synchronized void unbuffer() {
     storePosition();
     releaseClient();
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java

@@ -487,7 +487,7 @@ public void flush() throws IOException {
       } catch (Throwable e) {
         String msg = "Failed to flush. error: " + e.getMessage();
         LOG.error(msg, e);
-        throw new RuntimeException(msg, e);
+        throw e;
       }
     }
   }
@@ -553,7 +553,7 @@ public void close() throws IOException {
       } catch (Throwable e) {
         String msg = "Failed to flush. error: " + e.getMessage();
         LOG.error(msg, e);
-        throw new RuntimeException(msg, e);
+        throw e;
       } finally {
         cleanup(false);
       }
@@ -708,7 +708,7 @@ private void handleInterruptedException(Exception ex,
       boolean processExecutionException)
       throws IOException {
     LOG.error("Command execution was interrupted.");
-    if(processExecutionException) {
+    if (processExecutionException) {
       handleExecutionException(ex);
     } else {
       throw new IOException(EXCEPTION_MSG + ex.toString(), ex);

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BufferPool.java

@@ -46,7 +46,7 @@ public BufferPool(int bufferSize, int capacity) {
   }
 
   public BufferPool(int bufferSize, int capacity,
-      Function<ByteBuffer, ByteString> byteStringConversion){
+      Function<ByteBuffer, ByteString> byteStringConversion) {
     this.capacity = capacity;
     this.bufferSize = bufferSize;
     bufferList = new ArrayList<>(capacity);

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/RatisBlockOutputStream.java

@@ -97,7 +97,7 @@ void releaseBuffersOnException() {
 
   @Override
   XceiverClientReply sendWatchForCommit(boolean bufferFull) throws IOException {
-    return bufferFull? commitWatcher.watchOnFirstIndex()
+    return bufferFull ? commitWatcher.watchOnFirstIndex()
         : commitWatcher.watchOnLastIndex();
   }
 

hadoop-hdds/common/pom.xml

@@ -166,6 +166,21 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-jupiter-api</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.jaegertracing</groupId>
       <artifactId>jaeger-client</artifactId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -151,7 +151,7 @@ public final class HddsConfigKeys {
    */
   public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
   // Limit Certificate duration to a max value of 5 years.
-  public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+  public static final String HDDS_X509_MAX_DURATION_DEFAULT = "P1865D";
   public static final String HDDS_X509_SIGNATURE_ALGO =
       "hdds.x509.signature.algorithm";
   public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -632,7 +632,7 @@ public static long getShutDownTimeOut(ConfigurationSource conf) {
    * Utility method to round up bytes into the nearest MB.
    */
   public static int roundupMb(long bytes) {
-    return (int)Math.ceil((double) bytes/(double) ONE_MB);
+    return (int)Math.ceil((double) bytes / (double) ONE_MB);
   }
 
   /**

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java

@@ -151,6 +151,6 @@ public static String createStartupShutdownMessage(VersionInfo versionInfo,
 
   public static String appendIfNotPresent(String str, char c) {
     Preconditions.checkNotNull(str, "Input string is null");
-    return str.isEmpty() || str.charAt(str.length() - 1) != c ? str + c: str;
+    return str.isEmpty() || str.charAt(str.length() - 1) != c ? str + c : str;
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceAudience.java

@@ -50,7 +50,7 @@ public final class InterfaceAudience {
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
-  public @interface Public {};
+  public @interface Public { };
 
   /**
    * Intended only for the project(s) specified in the annotation.
@@ -67,7 +67,7 @@ public final class InterfaceAudience {
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
-  public @interface Private {};
+  public @interface Private { };
 
-  private InterfaceAudience() {} // Audience can't exist on its own
+  private InterfaceAudience() { } // Audience can't exist on its own
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java

@@ -43,7 +43,7 @@ public final class OzoneQuota {
   public static final String OZONE_QUOTA_TB = "TB";
 
   /** Quota Units.*/
-  public enum Units {B, KB, MB, GB, TB}
+  public enum Units { B, KB, MB, GB, TB }
 
   // Quota to decide how many buckets can be created.
   private long quotaInNamespace;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java

@@ -28,13 +28,14 @@ public class QuotaList {
   private ArrayList<OzoneQuota.Units> unitQuota;
   private ArrayList<Long> sizeQuota;
 
-  public QuotaList(){
+  public QuotaList() {
     ozoneQuota = new ArrayList<String>();
     unitQuota = new ArrayList<OzoneQuota.Units>();
     sizeQuota = new ArrayList<Long>();
   }
 
-  public void addQuotaList(String oQuota, OzoneQuota.Units uQuota, Long sQuota){
+  public void addQuotaList(
+      String oQuota, OzoneQuota.Units uQuota, Long sQuota) {
     ozoneQuota.add(oQuota);
     unitQuota.add(uQuota);
     sizeQuota.add(sQuota);
@@ -52,15 +53,15 @@ public ArrayList<OzoneQuota.Units> getUnitQuotaArray() {
     return this.unitQuota;
   }
 
-  public OzoneQuota.Units getUnits(String oQuota){
+  public OzoneQuota.Units getUnits(String oQuota) {
     return unitQuota.get(ozoneQuota.indexOf(oQuota));
   }
 
-  public Long getQuotaSize(OzoneQuota.Units uQuota){
+  public Long getQuotaSize(OzoneQuota.Units uQuota) {
     return sizeQuota.get(unitQuota.indexOf(uQuota));
   }
 
-  public OzoneQuota.Units getQuotaUnit(Long sQuota){
+  public OzoneQuota.Units getQuotaUnit(Long sQuota) {
     return unitQuota.get(sizeQuota.indexOf(sQuota));
   }
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationFactor.java

@@ -46,7 +46,7 @@ public enum ReplicationFactor {
    * @return ReplicationFactor
    */
   public static ReplicationFactor valueOf(int value) {
-    if(value == 1) {
+    if (value == 1) {
       return ONE;
     }
     if (value == 3) {