From 807f6d7f8f0f252683e0ebfd3b4ca6a6356b513b Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 26 Feb 2024 11:19:54 -0800 Subject: [PATCH 1/5] WIP --- .../java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +- .../main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index d1dcc654b100..0b7c8652d7e7 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@ -230,7 +230,7 @@ public enum ChecksumCombineMode { "list rather than full chunk list to optimize performance. " + "Critical to HBase.", tags = ConfigTag.CLIENT) - private boolean incrementalChunkList = false; + private boolean incrementalChunkList = true; @PostConstruct private void validate() { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 42a74dd12c2e..af0892e1152f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -147,7 +147,7 @@ public final class ScmConfigKeys { public static final String OZONE_CHUNK_LIST_INCREMENTAL = "ozone.chunk.list.incremental"; - public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = false; + public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = true; public static final String OZONE_SCM_CONTAINER_LAYOUT_KEY = "ozone.scm.container.layout"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index a0d4b59db168..555b180625c6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -126,7 +126,7 @@ public final class OzoneConfigKeys { public static final String OZONE_FS_HSYNC_ENABLED = "ozone.fs.hsync.enabled"; public static final boolean OZONE_FS_HSYNC_ENABLED_DEFAULT - = false; + = true; /** * hsync lease soft limit. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 47067de5fede..25553dc583cc 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -857,7 +857,7 @@ ozone.chunk.list.incremental - false + true OZONE, CLIENT, DATANODE, PERFORMANCE By default, a writer client sends full chunk list of a block when it @@ -4116,7 +4116,7 @@ ozone.fs.hsync.enabled - false + true OZONE, CLIENT Enable hsync/hflush. By default they are disabled. From f0f486e98db1e60da168193641f2646d8aadaf82 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 27 Feb 2024 10:23:38 -0800 Subject: [PATCH 2/5] Instead of throwing an exception and abort, log a warning and fallback to full chunk list, if DataNode has not finished upgrade. --- .../keyvalue/impl/BlockManagerImpl.java | 21 ++++++++++--------- .../containergenerator/GeneratorDatanode.java | 4 +++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index e40434f508e6..2c076bc48025 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -66,6 +66,7 @@ public class BlockManagerImpl implements BlockManager { // Default Read Buffer capacity when Checksum is not present private final int defaultReadBufferCapacity; private final int readMappedBufferThreshold; + private boolean incrementalEnabled; /** * Constructs a Block Manager. @@ -81,6 +82,15 @@ public BlockManagerImpl(ConfigurationSource conf) { this.readMappedBufferThreshold = config.getBufferSize( ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_KEY, ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_DEFAULT); + incrementalEnabled = + config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL, + OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT); + if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized( + HDDSLayoutFeature.HBASE_SUPPORT)) { + LOG.warn("DataNode has not finalized upgrading to a version that " + + "supports incremental chunk list. Fallback to full chunk list"); + incrementalEnabled = false; + } } @Override @@ -98,18 +108,9 @@ public long putBlock(Container container, BlockData data, endOfBlock); } - public static long persistPutBlock(KeyValueContainer container, + public long persistPutBlock(KeyValueContainer container, BlockData data, ConfigurationSource config, boolean endOfBlock) throws IOException { - boolean incrementalEnabled = - config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL, - OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT); - if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized( - HDDSLayoutFeature.HBASE_SUPPORT)) { - throw new StorageContainerException("DataNode has not finalized " + - "upgrading to a version that supports incremental chunk list.", - UNSUPPORTED_REQUEST); - } Preconditions.checkNotNull(data, "BlockData cannot be null for put " + "operation."); Preconditions.checkState(data.getContainerID() >= 0, "Container Id " + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index 3a43ddd8ab09..59d9a44fb188 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -111,6 +111,7 @@ public class GeneratorDatanode extends BaseGenerator { private int overlap; private ChunkManager chunkManager; + private BlockManagerImpl blockManager; private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; @@ -136,6 +137,7 @@ public Void call() throws Exception { BlockManager blockManager = new BlockManagerImpl(config); chunkManager = ChunkManagerFactory .createChunkManager(config, blockManager, null); + blockManager = new BlockManagerImpl(config); final Collection storageDirs = HddsServerUtil.getDatanodeStorageDirs(config); @@ -286,7 +288,7 @@ public void generateData(long index) throws Exception { writtenBytes += currentChunkSize; } - BlockManagerImpl.persistPutBlock(container, blockData, config, true); + blockManager.persistPutBlock(container, blockData, config, true); } From 06565a8d21bc5d4045b8e2581108fa3393e51c26 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 27 Feb 2024 11:38:19 -0800 Subject: [PATCH 3/5] Fix checkstyle and findbugs. --- .../ozone/container/keyvalue/impl/BlockManagerImpl.java | 7 ++----- .../ozone/freon/containergenerator/GeneratorDatanode.java | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index 2c076bc48025..4690565b0bb1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -42,7 +42,6 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT; @@ -103,13 +102,11 @@ public long putBlock(Container container, BlockData data, boolean endOfBlock) throws IOException { return persistPutBlock( (KeyValueContainer) container, - data, - config, - endOfBlock); + data, endOfBlock); } public long persistPutBlock(KeyValueContainer container, - BlockData data, ConfigurationSource config, boolean endOfBlock) + BlockData data, boolean endOfBlock) throws IOException { Preconditions.checkNotNull(data, "BlockData cannot be null for put " + "operation."); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index 59d9a44fb188..dbca12c8b26d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -60,7 +60,6 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory; -import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; import com.codahale.metrics.Timer; @@ -134,10 +133,9 @@ public Void call() throws Exception { config = createOzoneConfiguration(); - BlockManager blockManager = new BlockManagerImpl(config); + blockManager = new BlockManagerImpl(config); chunkManager = ChunkManagerFactory .createChunkManager(config, blockManager, null); - blockManager = new BlockManagerImpl(config); final Collection storageDirs = HddsServerUtil.getDatanodeStorageDirs(config); @@ -288,7 +286,7 @@ public void generateData(long index) throws Exception { writtenBytes += currentChunkSize; } - blockManager.persistPutBlock(container, blockData, config, true); + blockManager.persistPutBlock(container, blockData, true); } From 8ad6323e8b6c8127addfc1aece622adf88d3ec3b Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 29 Feb 2024 10:13:01 -0800 Subject: [PATCH 4/5] Change configuration key name. --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 25553dc583cc..d9a39d377269 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -856,7 +856,7 @@ - ozone.chunk.list.incremental + ozone.incremental.chunk.list true OZONE, CLIENT, DATANODE, PERFORMANCE From a65aa4defb52f4ba0898ea740f3590f74fe075c5 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 29 Feb 2024 11:45:04 -0800 Subject: [PATCH 5/5] Update ScmConfigKeys too. --- .../src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index af0892e1152f..c0a48efead04 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -146,7 +146,7 @@ public final class ScmConfigKeys { "32KB"; public static final String OZONE_CHUNK_LIST_INCREMENTAL = - "ozone.chunk.list.incremental"; + "ozone.incremental.chunk.list"; public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = true; public static final String OZONE_SCM_CONTAINER_LAYOUT_KEY =