[DO NOT MERGE] Backport some fixes and compatibility commits from master to ozone-1.4

.github/workflows/ci.yml

@@ -143,7 +143,7 @@ jobs:
         include:
           - os: ubuntu-20.04
           - java: 8
-            os: macos-12
+            os: macos-13
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -170,13 +170,13 @@ public enum ChecksumCombineMode {
   private String checksumType = ChecksumType.CRC32.name();
 
   @Config(key = "bytes.per.checksum",
-      defaultValue = "1MB",
+      defaultValue = "16KB",
       type = ConfigType.SIZE,
       description = "Checksum will be computed for every bytes per checksum "
           + "number of bytes and stored sequentially. The minimum value for "
-          + "this config is 16KB.",
+          + "this config is 8KB.",
       tags = ConfigTag.CLIENT)
-  private int bytesPerChecksum = 1024 * 1024;
+  private int bytesPerChecksum = 16 * 1024;
 
   @Config(key = "verify.checksum",
       defaultValue = "true",

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -149,7 +149,8 @@ public long updateCommitInfosMap(
     } else {
       stream = commitInfoProtos.stream().map(proto -> commitInfoMap
           .computeIfPresent(RatisHelper.toDatanodeId(proto.getServer()),
-              (address, index) -> proto.getCommitIndex()));
+              (address, index) -> proto.getCommitIndex()))
+          .filter(Objects::nonNull);
     }
     return stream.mapToLong(Long::longValue).min().orElse(0);
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java

@@ -458,7 +458,7 @@ public final class OzoneConfigKeys {
       "hdds.datanode.replication.work.dir";
 
 
-  public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 16 * 1024;
+  public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 8 * 1024;
 
   public static final String OZONE_CLIENT_READ_TIMEOUT
           = "ozone.client.read.timeout";

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChunkBufferImplWithByteBufferList.java

@@ -61,6 +61,7 @@ public class ChunkBufferImplWithByteBufferList implements ChunkBuffer {
 
   private void findCurrent() {
     boolean found = false;
+    limitPrecedingCurrent = 0;
     for (int i = 0; i < buffers.size(); i++) {
       final ByteBuffer buf = buffers.get(i);
       final int pos = buf.position();
@@ -185,6 +186,8 @@ public ChunkBuffer duplicate(int newPosition, int newLimit) {
    */
   @Override
   public Iterable<ByteBuffer> iterate(int bufferSize) {
+    Preconditions.checkArgument(bufferSize > 0);
+
     return () -> new Iterator<ByteBuffer>() {
       @Override
       public boolean hasNext() {
@@ -198,10 +201,40 @@ public ByteBuffer next() {
         }
         findCurrent();
         ByteBuffer current = buffers.get(currentIndex);
-        final ByteBuffer duplicated = current.duplicate();
-        duplicated.limit(current.limit());
-        current.position(current.limit());
-        return duplicated;
+
+        // If current buffer has enough space or it's the last one, return it.
+        if (current.remaining() >= bufferSize || currentIndex == buffers.size() - 1) {
+          final ByteBuffer duplicated = current.duplicate();
+          int duplicatedLimit = Math.min(current.position() + bufferSize, current.limit());
+          duplicated.limit(duplicatedLimit);
+          duplicated.position(current.position());
+
+          current.position(duplicatedLimit);
+          return duplicated;
+        }
+
+        // Otherwise, create a new buffer.
+        int newBufferSize = Math.min(bufferSize, remaining());
+        ByteBuffer allocated = ByteBuffer.allocate(newBufferSize);
+        int remainingToFill = allocated.remaining();
+
+        while (remainingToFill > 0) {
+          final ByteBuffer b = current();
+          int bytes = Math.min(b.remaining(), remainingToFill);
+          b.limit(b.position() + bytes);
+          allocated.put(b);
+          remainingToFill -= bytes;
+          advanceCurrent();
+        }
+
+        allocated.flip();
+
+        // Up-to-date current.
+        current = buffers.get(currentIndex);
+        // Reset
+        current.limit(current.capacity());
+
+        return allocated;
       }
     };
   }

hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChunkBufferImplWithByteBufferList.java

@@ -67,6 +67,76 @@ public void rejectsMultipleCurrentBuffers() {
     assertThrows(IllegalArgumentException.class, () -> ChunkBuffer.wrap(list));
   }
 
+  @Test
+  public void testIterateSmallerOverSingleChunk() {
+    ChunkBuffer subject = ChunkBuffer.wrap(ImmutableList.of(ByteBuffer.allocate(100)));
+
+    assertEquals(0, subject.position());
+    assertEquals(100, subject.remaining());
+    assertEquals(100, subject.limit());
+
+    subject.iterate(25).forEach(buffer -> assertEquals(25, buffer.remaining()));
+
+    assertEquals(100, subject.position());
+    assertEquals(0, subject.remaining());
+    assertEquals(100, subject.limit());
+  }
+
+  @Test
+  public void testIterateOverMultipleChunksFitChunkSize() {
+    ByteBuffer b1 = ByteBuffer.allocate(100);
+    ByteBuffer b2 = ByteBuffer.allocate(100);
+    ByteBuffer b3 = ByteBuffer.allocate(100);
+    ChunkBuffer subject = ChunkBuffer.wrap(ImmutableList.of(b1, b2, b3));
+
+    assertEquals(0, subject.position());
+    assertEquals(300, subject.remaining());
+    assertEquals(300, subject.limit());
+
+    subject.iterate(100).forEach(buffer -> assertEquals(100, buffer.remaining()));
+
+    assertEquals(300, subject.position());
+    assertEquals(0, subject.remaining());
+    assertEquals(300, subject.limit());
+  }
+
+  @Test
+  public void testIterateOverMultipleChunksSmallerChunks() {
+    ByteBuffer b1 = ByteBuffer.allocate(100);
+    ByteBuffer b2 = ByteBuffer.allocate(100);
+    ByteBuffer b3 = ByteBuffer.allocate(100);
+    ChunkBuffer subject = ChunkBuffer.wrap(ImmutableList.of(b1, b2, b3));
+
+    assertEquals(0, subject.position());
+    assertEquals(300, subject.remaining());
+    assertEquals(300, subject.limit());
+
+    subject.iterate(50).forEach(buffer -> assertEquals(50, buffer.remaining()));
+
+    assertEquals(300, subject.position());
+    assertEquals(0, subject.remaining());
+    assertEquals(300, subject.limit());
+  }
+
+  @Test
+  public void testIterateOverMultipleChunksBiggerChunks() {
+    ByteBuffer b1 = ByteBuffer.allocate(100);
+    ByteBuffer b2 = ByteBuffer.allocate(100);
+    ByteBuffer b3 = ByteBuffer.allocate(100);
+    ByteBuffer b4 = ByteBuffer.allocate(100);
+    ChunkBuffer subject = ChunkBuffer.wrap(ImmutableList.of(b1, b2, b3, b4));
+
+    assertEquals(0, subject.position());
+    assertEquals(400, subject.remaining());
+    assertEquals(400, subject.limit());
+
+    subject.iterate(200).forEach(buffer -> assertEquals(200, buffer.remaining()));
+
+    assertEquals(400, subject.position());
+    assertEquals(0, subject.remaining());
+    assertEquals(400, subject.limit());
+  }
+
   private static void assertEmpty(ChunkBuffer subject) {
     assertEquals(0, subject.position());
     assertEquals(0, subject.remaining());

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java

@@ -354,10 +354,11 @@ private void processCmd(DeleteCmdInfo cmd) {
       DeletedContainerBlocksSummary summary =
           DeletedContainerBlocksSummary.getFrom(containerBlocks);
       LOG.info("Summary of deleting container blocks, numOfTransactions={}, "
-              + "numOfContainers={}, numOfBlocks={}",
+              + "numOfContainers={}, numOfBlocks={}, commandId={}.",
           summary.getNumOfTxs(),
           summary.getNumOfContainers(),
-          summary.getNumOfBlocks());
+          summary.getNumOfBlocks(),
+          cmd.getCmd().getId());
       if (LOG.isDebugEnabled()) {
         LOG.debug("Start to delete container blocks, TXIDs={}",
             summary.getTxIDSummary());
@@ -384,7 +385,8 @@ private void processCmd(DeleteCmdInfo cmd) {
           LOG.debug("Sending following block deletion ACK to SCM");
           for (DeleteBlockTransactionResult result : blockDeletionACK
               .getResultsList()) {
-            LOG.debug("{} : {}", result.getTxID(), result.getSuccess());
+            LOG.debug("TxId = {} : ContainerId = {} : {}",
+                result.getTxID(), result.getContainerID(), result.getSuccess());
           }
         }
       }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.container.keyvalue;
 
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdfs.util.Canceler;
@@ -43,7 +44,6 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.util.Arrays;
 
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.slf4j.Logger;
@@ -404,8 +404,8 @@ private static ScanResult verifyChecksum(BlockData block,
                   " for block %s",
                   ChunkInfo.getFromProtoBuf(chunk),
                   i,
-                  Arrays.toString(expected.toByteArray()),
-                  Arrays.toString(actual.toByteArray()),
+                  StringUtils.bytes2Hex(expected.asReadOnlyByteBuffer()),
+                  StringUtils.bytes2Hex(actual.asReadOnlyByteBuffer()),
                   block.getBlockID());
           return ScanResult.unhealthy(
               ScanResult.FailureType.CORRUPT_CHUNK, chunkFile,

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java

@@ -126,7 +126,7 @@ static int getBufferCapacityForChunkRead(ChunkInfo chunkInfo,
     } else {
       // Set buffer capacity to checksum boundary size so that each buffer
       // corresponds to one checksum. If checksum is NONE, then set buffer
-      // capacity to default (OZONE_CHUNK_READ_BUFFER_DEFAULT_SIZE_KEY = 64KB).
+      // capacity to default (OZONE_CHUNK_READ_BUFFER_DEFAULT_SIZE_KEY = 1MB).
       ChecksumData checksumData = chunkInfo.getChecksumData();
 
       if (checksumData != null) {

hadoop-hdds/docs/content/feature/Quota.md

@@ -1,6 +1,6 @@
 ---
 title: "Quota in Ozone"
-date: "2020-October-22"
+date: "2020-10-22"
 weight: 4
 summary: Quota in Ozone
 icon: user

hadoop-hdds/docs/content/feature/Quota.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "Ozone 中的配额"
-date: "2020-October-22"
+date: "2020-10-22"
 weight: 4
 summary: Ozone中的配额
 icon: user

hadoop-hdds/docs/content/security/GDPR.md

@@ -1,6 +1,6 @@
 ---
 title: "GDPR in Ozone"
-date: "2019-September-17"
+date: "2019-09-17"
 weight: 3
 icon: user
 menu:

hadoop-hdds/docs/content/security/GDPR.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "Ozone 中的 GDPR"
-date: "2019-September-17"
+date: "2019-09-17"
 weight: 3
 summary: Ozone 中的 GDPR
 menu:

hadoop-hdds/docs/content/security/SecureOzone.md

@@ -1,6 +1,6 @@
 ---
 title: "Securing Ozone"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: Overview of Ozone security concepts and steps to secure Ozone Manager and SCM.
 weight: 1
 menu:

hadoop-hdds/docs/content/security/SecureOzone.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "安全化 Ozone"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: 简要介绍 Ozone 中的安全概念以及安全化 OM 和 SCM 的步骤。
 weight: 1
 menu:

hadoop-hdds/docs/content/security/SecuringDatanodes.md

@@ -1,6 +1,6 @@
 ---
 title: "Securing Datanodes"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 3
 menu:
    main:

hadoop-hdds/docs/content/security/SecuringDatanodes.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "安全化 Datanode"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 3
 menu:
   main:

hadoop-hdds/docs/content/security/SecuringOzoneHTTP.md

@@ -1,6 +1,6 @@
 ---
 title: "Securing HTTP"
-date: "2020-June-17"
+date: "2020-06-17"
 summary: Secure HTTP web-consoles for Ozone services 
 weight: 4
 menu:

hadoop-hdds/docs/content/security/SecuringS3.md

@@ -1,6 +1,6 @@
 ---
 title: "Securing S3"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: Ozone supports S3 protocol, and uses AWS Signature Version 4 protocol which allows a seamless S3 experience.
 weight: 5
 menu:

hadoop-hdds/docs/content/security/SecuringS3.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "安全化 S3"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: Ozone 支持 S3 协议，并使用 AWS Signature Version 4 protocol which allows a seamless S3
  experience.
 weight: 5

hadoop-hdds/docs/content/security/SecuringTDE.md

@@ -1,6 +1,6 @@
 ---
 title: "Transparent Data Encryption"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: TDE allows data on the disks to be encrypted-at-rest and automatically decrypted during access. 
 weight: 2
 menu:

hadoop-hdds/docs/content/security/SecuringTDE.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "透明数据加密"
-date: "2019-April-03"
+date: "2019-04-03"
 summary: 透明数据加密（Transparent Data Encryption，TDE）以密文形式在磁盘上保存数据，但可以在用户访问的时候自动进行解密。
 weight: 2
 menu:

hadoop-hdds/docs/content/security/SecurityAcls.md

@@ -1,6 +1,6 @@
 ---
 title: "Ozone ACLs"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 6
 menu:
    main:

hadoop-hdds/docs/content/security/SecurityAcls.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "Ozone 访问控制列表"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 6
 menu:
    main:

hadoop-hdds/docs/content/security/SecurityWithRanger.md

@@ -1,6 +1,6 @@
 ---
 title: "Apache Ranger"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 7
 menu:
    main:

hadoop-hdds/docs/content/security/SecurityWithRanger.zh.md

@@ -1,6 +1,6 @@
 ---
 title: "Apache Ranger"
-date: "2019-April-03"
+date: "2019-04-03"
 weight: 7
 menu:
    main:

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java

@@ -47,7 +47,8 @@ void addTransactionToDN(UUID dnID, DeletedBlocksTransaction tx) {
     blocksDeleted += tx.getLocalIDCount();
     if (SCMBlockDeletingService.LOG.isDebugEnabled()) {
       SCMBlockDeletingService.LOG
-          .debug("Transaction added: {} <- TX({})", dnID, tx.getTxID());
+          .debug("Transaction added: {} <- TX({}), DN {} <- blocksDeleted Add {}.",
+          dnID, tx.getTxID(), dnID, tx.getLocalIDCount());
     }
   }