Update branch

.github/workflows/ci.yml

@@ -434,12 +434,14 @@ jobs:
       matrix:
         profile:
           - client
-          - contract
+          - container
           - filesystem
           - hdds
           - om
           - ozone
+          - recon
           - shell
+          - snapshot
           - flaky
       fail-fast: false
     steps:

LICENSE.txt

@@ -213,8 +213,6 @@ Apache License 2.0
 hadoop-hdds/framework/src/main/resources/webapps/static/nvd3-1.8.5.min.js.map
 hadoop-hdds/framework/src/main/resources/webapps/static/nvd3-1.8.5.min.css.map
 hadoop-hdds/framework/src/main/resources/webapps/static/nvd3-1.8.5.min.js
-hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AbstractFuture.java
-hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/TimeoutFuture.java
 
 
 BSD 3-Clause

dev-support/ci/selective_ci_checks.bats

@@ -108,7 +108,7 @@ load bats-assert/load.bash
 @test "integration and unit: java change" {
   run dev-support/ci/selective_ci_checks.sh 9aebf6e25
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
@@ -120,7 +120,7 @@ load bats-assert/load.bash
 @test "integration and unit: script change" {
   run dev-support/ci/selective_ci_checks.sh c6850484f
 
-  assert_output -p 'basic-checks=["rat","bats","unit"]'
+  assert_output -p 'basic-checks=["rat","bats"]'
   assert_output -p needs-build=false
   assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
@@ -132,7 +132,7 @@ load bats-assert/load.bash
 @test "script change including junit.sh" {
   run dev-support/ci/selective_ci_checks.sh 66093e52c6
 
-  assert_output -p 'basic-checks=["rat","bats","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","bats","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
@@ -144,19 +144,19 @@ load bats-assert/load.bash
 @test "unit only" {
   run dev-support/ci/selective_ci_checks.sh 1dd1d0ba3
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
-  assert_output -p needs-integration-tests=false
+  assert_output -p needs-integration-tests=true
   assert_output -p needs-kubernetes-tests=false
 }
 
 @test "unit helper" {
   run dev-support/ci/selective_ci_checks.sh 88383d1d5
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
@@ -189,20 +189,17 @@ load bats-assert/load.bash
   assert_output -p needs-kubernetes-tests=false
 }
 
-# disabled, because this test fails if
-# hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java
-# is not present in the current tree (i.e. if file is renamed, moved or deleted)
-#@test "native test in other module" {
-#  run dev-support/ci/selective_ci_checks.sh 7d01cc14a6
-#
-#  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native","unit"]'
-#  assert_output -p needs-build=true
-#  assert_output -p needs-compile=true
-#  assert_output -p needs-compose-tests=false
-#  assert_output -p needs-dependency-check=false
-#  assert_output -p needs-integration-tests=false
-#  assert_output -p needs-kubernetes-tests=false
-#}
+@test "native test in other module" {
+  run dev-support/ci/selective_ci_checks.sh 822c0dee1a
+
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]'
+  assert_output -p needs-build=true
+  assert_output -p needs-compile=true
+  assert_output -p needs-compose-tests=false
+  assert_output -p needs-dependency-check=false
+  assert_output -p needs-integration-tests=false
+  assert_output -p needs-kubernetes-tests=false
+}
 
 @test "kubernetes only" {
   run dev-support/ci/selective_ci_checks.sh 5336bb9bd
@@ -231,7 +228,7 @@ load bats-assert/load.bash
 @test "main/java change" {
   run dev-support/ci/selective_ci_checks.sh 86a771dfe
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -243,7 +240,7 @@ load bats-assert/load.bash
 @test "..../java change" {
   run dev-support/ci/selective_ci_checks.sh 01c616536
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -255,7 +252,7 @@ load bats-assert/load.bash
 @test "java and compose change" {
   run dev-support/ci/selective_ci_checks.sh d0f0f806e
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -267,7 +264,7 @@ load bats-assert/load.bash
 @test "java and docs change" {
   run dev-support/ci/selective_ci_checks.sh 2c0adac26
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","docs","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","docs","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -279,7 +276,7 @@ load bats-assert/load.bash
 @test "pom change" {
   run dev-support/ci/selective_ci_checks.sh 9129424a9
 
-  assert_output -p 'basic-checks=["rat","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -291,7 +288,7 @@ load bats-assert/load.bash
 @test "CI lib change" {
   run dev-support/ci/selective_ci_checks.sh ceb79acaa
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat","unit"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -303,7 +300,7 @@ load bats-assert/load.bash
 @test "CI workflow change" {
   run dev-support/ci/selective_ci_checks.sh 90a8d7c01
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat","unit"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -328,7 +325,7 @@ load bats-assert/load.bash
 @test "CI workflow change (ci.yaml)" {
   run dev-support/ci/selective_ci_checks.sh 90fd5f2adc
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat","unit"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true

dev-support/ci/selective_ci_checks.sh

@@ -263,18 +263,10 @@ function get_count_integration_files() {
         "^hadoop-ozone/integration-test"
         "^hadoop-ozone/fault-injection-test/mini-chaos-tests"
         "src/test/java"
+        "src/test/resources"
     )
-    # Ozone's unit test naming convention: Test*.java
-    # The following makes this filter ignore all tests except those in
-    # integration-test and fault-injection-test.
-    # Directories starting with `i` under hadoop-ozone need to be listed
-    # explicitly, other subdirectories are captured by the second item.
     local ignore_array=(
-        "^hadoop-hdds/.*/src/test/java/.*/Test.*.java"
-        "^hadoop-ozone/[a-eghj-z].*/src/test/java/.*/Test.*.java"
-        "^hadoop-ozone/insight/src/test/java/.*/Test.*.java"
-        "^hadoop-ozone/interface-client/src/test/java/.*/Test.*.java"
-        "^hadoop-ozone/interface-storage/src/test/java/.*/Test.*.java"
+        $(grep -Flr 'org.apache.ozone.test.tag.Native' hadoop-ozone/integration-test)
     )
     filter_changed_files true
     COUNT_INTEGRATION_CHANGED_FILES=${match_count}
@@ -456,29 +448,6 @@ function check_needs_native() {
     start_end::group_end
 }
 
-function check_needs_unit_test() {
-    start_end::group_start "Check if unit test is needed"
-    local pattern_array=(
-        "^hadoop-ozone/dev-support/checks/_mvn_unit_report.sh"
-        "^hadoop-ozone/dev-support/checks/unit.sh"
-        "^hadoop-ozone/dev-support/checks/junit.sh"
-        "src/test/java"
-        "src/test/resources"
-    )
-    local ignore_array=(
-        "^hadoop-ozone/dist"
-        "^hadoop-ozone/fault-injection-test/mini-chaos-tests"
-        "^hadoop-ozone/integration-test"
-    )
-    filter_changed_files true
-
-    if [[ ${match_count} != "0" ]]; then
-        add_basic_check unit
-    fi
-
-    start_end::group_end
-}
-
 # Counts other files which do not need to trigger any functional test
 # (i.e. no compose/integration/kubernetes)
 function get_count_misc_files() {
@@ -502,6 +471,7 @@ function get_count_misc_files() {
         "^hadoop-ozone/dev-support/checks/_mvn_unit_report.sh"
         "^hadoop-ozone/dev-support/checks/acceptance.sh"
         "^hadoop-ozone/dev-support/checks/integration.sh"
+        "^hadoop-ozone/dev-support/checks/junit.sh"
         "^hadoop-ozone/dev-support/checks/kubernetes.sh"
     )
     filter_changed_files true
@@ -534,7 +504,6 @@ function calculate_test_types_to_run() {
         compose_tests_needed=true
         integration_tests_needed=true
         kubernetes_tests_needed=true
-        add_basic_check unit
     else
         echo "All ${COUNT_ALL_CHANGED_FILES} changed files are known to be handled by specific checks."
         echo
@@ -614,6 +583,5 @@ check_needs_dependency
 check_needs_docs
 check_needs_findbugs
 check_needs_native
-check_needs_unit_test
 calculate_test_types_to_run
 set_outputs

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java

@@ -27,6 +27,7 @@
 import org.apache.hadoop.metrics2.lib.Interns;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.ozone.OzoneConsts;
 
 import java.util.Map;
@@ -51,6 +52,11 @@ public final class ContainerClientMetrics {
   private MutableCounterLong totalWriteChunkCalls;
   @Metric
   private MutableCounterLong totalWriteChunkBytes;
+  private MutableQuantiles[] listBlockLatency;
+  private MutableQuantiles[] getBlockLatency;
+  private MutableQuantiles[] getCommittedBlockLengthLatency;
+  private MutableQuantiles[] readChunkLatency;
+  private MutableQuantiles[] getSmallFileLatency;
   private final Map<PipelineID, MutableCounterLong> writeChunkCallsByPipeline;
   private final Map<PipelineID, MutableCounterLong> writeChunkBytesByPipeline;
   private final Map<UUID, MutableCounterLong> writeChunksCallsByLeaders;
@@ -84,6 +90,36 @@ private ContainerClientMetrics() {
     writeChunkCallsByPipeline = new ConcurrentHashMap<>();
     writeChunkBytesByPipeline = new ConcurrentHashMap<>();
     writeChunksCallsByLeaders = new ConcurrentHashMap<>();
+
+    listBlockLatency = new MutableQuantiles[3];
+    getBlockLatency = new MutableQuantiles[3];
+    getCommittedBlockLengthLatency = new MutableQuantiles[3];
+    readChunkLatency = new MutableQuantiles[3];
+    getSmallFileLatency = new MutableQuantiles[3];
+    int[] intervals = {60, 300, 900};
+    for (int i = 0; i < intervals.length; i++) {
+      int interval = intervals[i];
+      listBlockLatency[i] = registry
+          .newQuantiles("listBlockLatency" + interval
+                  + "s", "ListBlock latency in microseconds", "ops",
+              "latency", interval);
+      getBlockLatency[i] = registry
+          .newQuantiles("getBlockLatency" + interval
+                  + "s", "GetBlock latency in microseconds", "ops",
+              "latency", interval);
+      getCommittedBlockLengthLatency[i] = registry
+          .newQuantiles("getCommittedBlockLengthLatency" + interval
+                  + "s", "GetCommittedBlockLength latency in microseconds",
+              "ops", "latency", interval);
+      readChunkLatency[i] = registry
+          .newQuantiles("readChunkLatency" + interval
+                  + "s", "ReadChunk latency in microseconds", "ops",
+              "latency", interval);
+      getSmallFileLatency[i] = registry
+          .newQuantiles("getSmallFileLatency" + interval
+                  + "s", "GetSmallFile latency in microseconds", "ops",
+              "latency", interval);
+    }
   }
 
   public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) {
@@ -111,7 +147,48 @@ public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) {
     totalWriteChunkBytes.incr(chunkSizeBytes);
   }
 
-  MutableCounterLong getTotalWriteChunkBytes() {
+  public void addListBlockLatency(long latency) {
+    for (MutableQuantiles q : listBlockLatency) {
+      if (q != null) {
+        q.add(latency);
+      }
+    }
+  }
+
+  public void addGetBlockLatency(long latency) {
+    for (MutableQuantiles q : getBlockLatency) {
+      if (q != null) {
+        q.add(latency);
+      }
+    }
+  }
+
+  public void addGetCommittedBlockLengthLatency(long latency) {
+    for (MutableQuantiles q : getCommittedBlockLengthLatency) {
+      if (q != null) {
+        q.add(latency);
+      }
+    }
+  }
+
+  public void addReadChunkLatency(long latency) {
+    for (MutableQuantiles q : readChunkLatency) {
+      if (q != null) {
+        q.add(latency);
+      }
+    }
+  }
+
+  public void addGetSmallFileLatency(long latency) {
+    for (MutableQuantiles q : getSmallFileLatency) {
+      if (q != null) {
+        q.add(latency);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  public MutableCounterLong getTotalWriteChunkBytes() {
     return totalWriteChunkBytes;
   }
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -256,6 +256,14 @@ public enum ChecksumCombineMode {
       tags = ConfigTag.CLIENT)
   private boolean incrementalChunkList = true;
 
+  @Config(key = "stream.putblock.piggybacking",
+          defaultValue = "false",
+          type = ConfigType.BOOLEAN,
+          description = "Allow PutBlock to be piggybacked in WriteChunk " +
+                  "requests if the chunk is small.",
+          tags = ConfigTag.CLIENT)
+  private boolean enablePutblockPiggybacking = false;
+
   @PostConstruct
   public void validate() {
     Preconditions.checkState(streamBufferSize > 0);
@@ -454,6 +462,14 @@ public String getFsDefaultBucketLayout() {
     return fsDefaultBucketLayout;
   }
 
+  public void setEnablePutblockPiggybacking(boolean enablePutblockPiggybacking) {
+    this.enablePutblockPiggybacking = enablePutblockPiggybacking;
+  }
+
+  public boolean getEnablePutblockPiggybacking() {
+    return enablePutblockPiggybacking;
+  }
+
   public boolean isDatastreamPipelineMode() {
     return datastreamPipelineMode;
   }