diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index a065c4b6d6e0..d4bfa79142e2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -1099,11 +1099,17 @@ public void writeChunkForClosedContainer(ChunkInfo chunkInfo, BlockID blockID, /** * Handle Put Block operation for closed container. Calls BlockManager to process the request. - * + * This is primarily used by container reconciliation process to persist the block data for closed container. + * @param kvContainer - Container for which block data need to be persisted. + * @param blockData - Block Data to be persisted (BlockData should have the chunks). + * @param blockCommitSequenceId - Block Commit Sequence ID for the block. + * @param overwriteBscId - To overwrite bcsId in the block data and container. In case of chunk failure + * during reconciliation, we do not want to overwrite the bcsId as this block/container + * is incomplete in its current state. */ - public void putBlockForClosedContainer(List chunkInfos, KeyValueContainer kvContainer, - BlockData blockData, long blockCommitSequenceId) - throws IOException { + public void putBlockForClosedContainer(KeyValueContainer kvContainer, BlockData blockData, + long blockCommitSequenceId, boolean overwriteBscId) + throws IOException { Preconditions.checkNotNull(kvContainer); Preconditions.checkNotNull(blockData); long startTime = Time.monotonicNowNanos(); @@ -1112,11 +1118,12 @@ public void putBlockForClosedContainer(List chunkInfo throw new IOException("Container #" + kvContainer.getContainerData().getContainerID() + " is not in closed state, Container state is " + kvContainer.getContainerState()); } - blockData.setChunks(chunkInfos); // To be set from the Replica's BCSId - blockData.setBlockCommitSequenceId(blockCommitSequenceId); + if (overwriteBscId) { + blockData.setBlockCommitSequenceId(blockCommitSequenceId); + } - blockManager.putBlock(kvContainer, blockData, false); + blockManager.putBlockForClosedContainer(kvContainer, blockData, overwriteBscId); ContainerProtos.BlockData blockDataProto = blockData.getProtoBufMessage(); final long numBytes = blockDataProto.getSerializedSize(); // Increment write stats for PutBlock after write. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index 4e0e1899739a..ba3d40489777 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -98,6 +98,80 @@ public long putBlock(Container container, BlockData data, data, endOfBlock); } + /** + * {@inheritDoc} + */ + @Override + public long putBlockForClosedContainer(Container container, BlockData data, boolean overwriteBcsId) + throws IOException { + Preconditions.checkNotNull(data, "BlockData cannot be null for put operation."); + Preconditions.checkState(data.getContainerID() >= 0, "Container Id cannot be negative"); + + KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData(); + + // We are not locking the key manager since RocksDB serializes all actions + // against a single DB. We rely on DB level locking to avoid conflicts. + try (DBHandle db = BlockUtils.getDB(containerData, config)) { + // This is a post condition that acts as a hint to the user. + // Should never fail. + Preconditions.checkNotNull(db, DB_NULL_ERR_MSG); + + long blockBcsID = data.getBlockCommitSequenceId(); + long containerBcsID = containerData.getBlockCommitSequenceId(); + + // Check if the block is already present in the DB of the container to determine whether + // the blockCount is already incremented for this block in the DB or not. + long localID = data.getLocalID(); + boolean incrBlockCount = false; + + // update the blockData as well as BlockCommitSequenceId here + try (BatchOperation batch = db.getStore().getBatchHandler() + .initBatchOperation()) { + // If block already exists in the DB, blockCount should not be incremented. + if (db.getStore().getBlockDataTable().get(containerData.getBlockKey(localID)) == null) { + incrBlockCount = true; + } + + db.getStore().getBlockDataTable().putWithBatch(batch, containerData.getBlockKey(localID), data); + if (overwriteBcsId && blockBcsID > containerBcsID) { + db.getStore().getMetadataTable().putWithBatch(batch, containerData.getBcsIdKey(), blockBcsID); + } + + // Set Bytes used, this bytes used will be updated for every write and + // only get committed for every put block. In this way, when datanode + // is up, for computation of disk space by container only committed + // block length is used, And also on restart the blocks committed to DB + // is only used to compute the bytes used. This is done to keep the + // current behavior and avoid DB write during write chunk operation. + db.getStore().getMetadataTable().putWithBatch(batch, containerData.getBytesUsedKey(), + containerData.getBytesUsed()); + + // Set Block Count for a container. + if (incrBlockCount) { + db.getStore().getMetadataTable().putWithBatch(batch, containerData.getBlockCountKey(), + containerData.getBlockCount() + 1); + } + + db.getStore().getBatchHandler().commitBatchOperation(batch); + } + + if (overwriteBcsId && blockBcsID > containerBcsID) { + container.updateBlockCommitSequenceId(blockBcsID); + } + + // Increment block count in-memory after the DB update. + if (incrBlockCount) { + containerData.incrBlockCount(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Block {} successfully persisted for closed container {} with bcsId {} chunk size {}", + data.getBlockID(), containerData.getContainerID(), blockBcsID, data.getChunks().size()); + } + return data.getSize(); + } + } + public long persistPutBlock(KeyValueContainer container, BlockData data, boolean endOfBlock) throws IOException { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java index 0621be4aae27..db9e36bbd71e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.container.keyvalue.impl; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CHUNK_FILE_INCONSISTENCY; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST; import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_BLOCK; import static org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage.COMMIT_DATA; @@ -149,7 +150,7 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info, .getContainerData(); final File chunkFile = getChunkFile(container, blockID); - long len = info.getLen(); + long chunkLength = info.getLen(); long offset = info.getOffset(); HddsVolume volume = containerData.getVolume(); @@ -174,10 +175,27 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info, ChunkUtils.validateChunkSize(channel, info, chunkFile.getName()); } - ChunkUtils - .writeData(channel, chunkFile.getName(), data, offset, len, volume); + long fileLengthBeforeWrite; + try { + fileLengthBeforeWrite = channel.size(); + } catch (IOException e) { + throw new StorageContainerException("Encountered an error while getting the file size for " + + chunkFile.getName(), CHUNK_FILE_INCONSISTENCY); + } + + ChunkUtils.writeData(channel, chunkFile.getName(), data, offset, chunkLength, volume); + + // When overwriting, update the bytes used if the new length is greater than the old length + // This is to ensure that the bytes used is updated correctly when overwriting a smaller chunk + // with a larger chunk at the end of the block. + if (overwrite) { + long fileLengthAfterWrite = offset + chunkLength; + if (fileLengthAfterWrite > fileLengthBeforeWrite) { + containerData.incrBytesUsed(fileLengthAfterWrite - fileLengthBeforeWrite); + } + } - containerData.updateWriteStats(len, overwrite); + containerData.updateWriteStats(chunkLength, overwrite); } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java index 5e20e8413561..10562f450b19 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java @@ -18,10 +18,13 @@ package org.apache.hadoop.ozone.container.keyvalue.interfaces; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.container.common.helpers.BlockData; +import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; /** * BlockManager is for performing key related operations on the container. @@ -49,6 +52,18 @@ public interface BlockManager { long putBlock(Container container, BlockData data, boolean endOfBlock) throws IOException; + /** + * Persists the block data for a closed container. The block data should have all the chunks and bcsId. + * Overwrites the block if it already exists, The container's used bytes should be updated by the caller with + * {@link ChunkManager#writeChunk(Container, BlockID, ChunkInfo, ByteBuffer, DispatcherContext)}. + * + * @param container - Container for which block data need to be persisted. + * @param data - Block Data to be persisted (BlockData should have the chunks). + * @param overwriteBcsId - To overwrite bcsId of the container. + */ + long putBlockForClosedContainer(Container container, BlockData data, boolean overwriteBcsId) + throws IOException; + /** * Gets an existing block. * diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java index 07414d701aad..7bbfa1f7e906 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestBlockManagerImpl.java @@ -17,11 +17,13 @@ package org.apache.hadoop.ozone.container.keyvalue.impl; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED; import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST; import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion; import static org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl.FULL_CHUNK; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -36,10 +38,12 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; +import org.apache.hadoop.ozone.container.common.interfaces.DBHandle; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; @@ -81,10 +85,10 @@ private void initTest(ContainerTestVersionInfo versionInfo) this.schemaVersion = versionInfo.getSchemaVersion(); this.config = new OzoneConfiguration(); ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, config); - initilaze(); + initialize(); } - private void initilaze() throws Exception { + private void initialize() throws Exception { UUID datanodeId = UUID.randomUUID(); HddsVolume hddsVolume = new HddsVolume.Builder(folder.toString()) .conf(config) @@ -196,6 +200,73 @@ public void testPutAndGetBlock(ContainerTestVersionInfo versionInfo) } + @ContainerTestVersionInfo.ContainerTest + public void testPutBlockForClosed(ContainerTestVersionInfo versionInfo) + throws Exception { + initTest(versionInfo); + KeyValueContainerData containerData = keyValueContainer.getContainerData(); + assertEquals(0, containerData.getBlockCount()); + keyValueContainer.close(); + assertEquals(CLOSED, keyValueContainer.getContainerState()); + + try (DBHandle db = BlockUtils.getDB(containerData, config)) { + // 1. Put Block with bcsId = 1, Overwrite BCS ID = true + blockData1 = createBlockData(1L, 2L, 1, 0, 2048, 1); + blockManager.putBlockForClosedContainer(keyValueContainer, blockData1, true); + + BlockData fromGetBlockData; + //Check Container's bcsId + fromGetBlockData = blockManager.getBlock(keyValueContainer, blockData.getBlockID()); + assertEquals(1, containerData.getBlockCount()); + assertEquals(1, containerData.getBlockCommitSequenceId()); + assertEquals(1, fromGetBlockData.getBlockCommitSequenceId()); + assertEquals(1, db.getStore().getMetadataTable().get(containerData.getBcsIdKey())); + assertEquals(1, db.getStore().getMetadataTable().get(containerData.getBlockCountKey())); + + // 2. Put Block with bcsId = 2, Overwrite BCS ID = false + BlockData blockData2 = createBlockData(1L, 3L, 1, 0, 2048, 2); + blockManager.putBlockForClosedContainer(keyValueContainer, blockData2, false); + + // The block should be written, but we won't be able to read it, As expected BcsId < container's BcsId + // fails during block read. + assertThrows(StorageContainerException.class, () -> blockManager + .getBlock(keyValueContainer, blockData2.getBlockID())); + fromGetBlockData = db.getStore().getBlockDataTable().get(containerData.getBlockKey(blockData2.getLocalID())); + assertEquals(2, containerData.getBlockCount()); + // BcsId should still be 1, as the BcsId is not overwritten + assertEquals(1, containerData.getBlockCommitSequenceId()); + assertEquals(2, fromGetBlockData.getBlockCommitSequenceId()); + assertEquals(2, db.getStore().getMetadataTable().get(containerData.getBlockCountKey())); + assertEquals(1, db.getStore().getMetadataTable().get(containerData.getBcsIdKey())); + + // 3. Put Block with bcsId = 2, Overwrite BCS ID = true + // This should succeed as we are overwriting the BcsId, The container BcsId should be updated to 2 + // The block count should not change. + blockManager.putBlockForClosedContainer(keyValueContainer, blockData2, true); + fromGetBlockData = blockManager.getBlock(keyValueContainer, blockData2.getBlockID()); + assertEquals(2, containerData.getBlockCount()); + assertEquals(2, containerData.getBlockCommitSequenceId()); + assertEquals(2, fromGetBlockData.getBlockCommitSequenceId()); + assertEquals(2, db.getStore().getMetadataTable().get(containerData.getBlockCountKey())); + assertEquals(2, db.getStore().getMetadataTable().get(containerData.getBcsIdKey())); + + // 4. Put Block with bcsId = 1 < container bcsId, Overwrite BCS ID = true + // We are overwriting an existing block with lower bcsId than container bcsId. Container bcsId should not change + BlockData blockData3 = createBlockData(1L, 1L, 1, 0, 2048, 1); + blockManager.putBlockForClosedContainer(keyValueContainer, blockData3, true); + fromGetBlockData = blockManager.getBlock(keyValueContainer, blockData3.getBlockID()); + assertEquals(3, containerData.getBlockCount()); + assertEquals(2, containerData.getBlockCommitSequenceId()); + assertEquals(1, fromGetBlockData.getBlockCommitSequenceId()); + assertEquals(3, db.getStore().getMetadataTable().get(containerData.getBlockCountKey())); + assertEquals(2, db.getStore().getMetadataTable().get(containerData.getBcsIdKey())); + + // writeChunk updates the in-memory state of the used bytes, putBlock persists the in-memory state to the DB. + // We are only doing putBlock without writeChunk, the used bytes should be 0. + assertEquals(0, db.getStore().getMetadataTable().get(containerData.getBytesUsedKey())); + } + } + @ContainerTestVersionInfo.ContainerTest public void testListBlock(ContainerTestVersionInfo versionInfo) throws Exception { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java index 7bfce86fb605..16e9968b381e 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java @@ -164,8 +164,8 @@ public void testWriteChunkAndPutBlockFailureForNonClosedContainer( ChunkBuffer.wrap(getData()); Assertions.assertThrows(IOException.class, () -> keyValueHandler.writeChunkForClosedContainer( getChunkInfo(), getBlockID(), ChunkBuffer.wrap(getData()), keyValueContainer)); - Assertions.assertThrows(IOException.class, () -> keyValueHandler.putBlockForClosedContainer( - null, keyValueContainer, new BlockData(getBlockID()), 0L)); + Assertions.assertThrows(IOException.class, () -> keyValueHandler.putBlockForClosedContainer(keyValueContainer, + new BlockData(getBlockID()), 0L, true)); } @Test @@ -225,37 +225,72 @@ public void testPutBlockForClosedContainer() throws IOException { containerSet.addContainer(kvContainer); KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet); List chunkInfoList = new ArrayList<>(); - chunkInfoList.add(getChunkInfo().getProtoBufMessage()); + ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 0), 0L, 20L); + + chunkInfoList.add(info.getProtoBufMessage()); BlockData putBlockData = new BlockData(getBlockID()); - keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 1L); - Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 1L); - Assertions.assertEquals(containerData.getBlockCount(), 1L); + putBlockData.setChunks(chunkInfoList); + + ChunkBuffer chunkData = ContainerTestHelper.getData(20); + keyValueHandler.writeChunkForClosedContainer(info, getBlockID(), chunkData, kvContainer); + keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 1L, true); + assertEquals(1L, containerData.getBlockCommitSequenceId()); + assertEquals(1L, containerData.getBlockCount()); + assertEquals(20L, containerData.getBytesUsed()); try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) { long localID = putBlockData.getLocalID(); BlockData getBlockData = dbHandle.getStore().getBlockDataTable() .get(containerData.getBlockKey(localID)); Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData)); + assertEquals(20L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey())); } // Add another chunk and check the put block data - ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID() - .getLocalID(), 1L), 0, 20L); + ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 1L), 20L, 20L); + chunkInfoList.add(newChunkInfo.getProtoBufMessage()); + putBlockData.setChunks(chunkInfoList); + + chunkData = ContainerTestHelper.getData(20); + keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), chunkData, kvContainer); + keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true); + assertEquals(2L, containerData.getBlockCommitSequenceId()); + assertEquals(1L, containerData.getBlockCount()); + assertEquals(40L, containerData.getBytesUsed()); + + try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) { + long localID = putBlockData.getLocalID(); + BlockData getBlockData = dbHandle.getStore().getBlockDataTable() + .get(containerData.getBlockKey(localID)); + Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData)); + assertEquals(40L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey())); + } + + // Replace the last chunk with a chunk of greater size, This should only update the bytesUsed with + // difference in length between the old last chunk and new last chunk + newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID().getLocalID(), 1L), 20L, 30L); + chunkInfoList.remove(chunkInfoList.size() - 1); chunkInfoList.add(newChunkInfo.getProtoBufMessage()); - keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 2L); - Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 2L); - Assertions.assertEquals(containerData.getBlockCount(), 1L); + putBlockData.setChunks(chunkInfoList); + + chunkData = ContainerTestHelper.getData(30); + keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), chunkData, kvContainer); + keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true); + assertEquals(2L, containerData.getBlockCommitSequenceId()); + assertEquals(1L, containerData.getBlockCount()); + // Old chunk size 20, new chunk size 30, difference 10. So bytesUsed should be 40 + 10 = 50 + assertEquals(50L, containerData.getBytesUsed()); try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) { long localID = putBlockData.getLocalID(); BlockData getBlockData = dbHandle.getStore().getBlockDataTable() .get(containerData.getBlockKey(localID)); Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData)); + assertEquals(50L, dbHandle.getStore().getMetadataTable().get(containerData.getBytesUsedKey())); } - // Put block on bcsId <= containerBcsId should be a no-op - keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 2L); - Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 2L); + keyValueHandler.putBlockForClosedContainer(kvContainer, putBlockData, 2L, true); + assertEquals(2L, containerData.getBlockCommitSequenceId()); } private boolean blockDataEquals(BlockData putBlockData, BlockData getBlockData) {