From fa9f4ccc0645703b7b1857f1a6b31014c414f926 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 4 Aug 2020 10:03:45 -0400 Subject: [PATCH 1/3] Create filter to include #deleting# blocks in key count when initializing container data --- .../keyvalue/helpers/KeyValueContainerUtil.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index d5fea23a882f..2958e9e5d4f5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -230,10 +230,22 @@ public static void parseKVContainerData(KeyValueContainerData kvContainerData, private static void initializeUsedBytesAndBlockCount( KeyValueContainerData kvContainerData) throws IOException { + MetadataKeyFilters.KeyPrefixFilter filter = + new MetadataKeyFilters.KeyPrefixFilter(); + + // Ignore all blocks except those with no prefix, or those with + // #deleting# prefix. + filter.addFilter(OzoneConsts.DELETED_KEY_PREFIX, true) + .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true) + .addFilter(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX, true) + .addFilter(OzoneConsts.BLOCK_COUNT, true) + .addFilter(OzoneConsts.CONTAINER_BYTES_USED, true) + .addFilter(OzoneConsts.PENDING_DELETE_BLOCK_COUNT, true); + long blockCount = 0; try (KeyValueBlockIterator blockIter = new KeyValueBlockIterator( kvContainerData.getContainerID(), - new File(kvContainerData.getContainerPath()))) { + new File(kvContainerData.getContainerPath()), filter)) { long usedBytes = 0; From 8d687813c7683cba0d91845ef8d297103a63c40b Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 4 Aug 2020 10:12:15 -0400 Subject: [PATCH 2/3] Update unit tests to include pending delete blocks in the block count and bytes used metadata values --- .../container/ozoneimpl/TestContainerReader.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java index 15c023641321..692d288912dd 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java @@ -139,17 +139,10 @@ private void markBlocksForDelete(KeyValueContainer keyValueContainer, } if (setMetaData) { + // Pending delete blocks are still counted towards the block count + // and bytes used metadata values, so those do not change. metadataStore.getStore().put(DB_PENDING_DELETE_BLOCK_COUNT_KEY, Longs.toByteArray(count)); - long blkCount = Longs.fromByteArray( - metadataStore.getStore().get(DB_BLOCK_COUNT_KEY)); - metadataStore.getStore().put(DB_BLOCK_COUNT_KEY, - Longs.toByteArray(blkCount - count)); - long bytesUsed = Longs.fromByteArray( - metadataStore.getStore().get(DB_CONTAINER_BYTES_USED_KEY)); - metadataStore.getStore().put(DB_CONTAINER_BYTES_USED_KEY, - Longs.toByteArray(bytesUsed - (count * blockLen))); - } } @@ -210,10 +203,10 @@ public void testContainerReader() throws Exception { keyValueContainer.getContainerData(); // Verify block related metadata. - Assert.assertEquals(blockCount - i, + Assert.assertEquals(blockCount, keyValueContainerData.getKeyCount()); - Assert.assertEquals((blockCount - i) * blockLen, + Assert.assertEquals(blockCount * blockLen, keyValueContainerData.getBytesUsed()); Assert.assertEquals(i, From cdc6a4d67552f0ac8d133133eb46612a6b747250 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 4 Aug 2020 10:47:27 -0400 Subject: [PATCH 3/3] Fix checkstyle violation --- .../hadoop/ozone/container/ozoneimpl/TestContainerReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java index 692d288912dd..0014a754899b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java @@ -52,7 +52,6 @@ import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConsts.DB_BLOCK_COUNT_KEY; -import static org.apache.hadoop.ozone.OzoneConsts.DB_CONTAINER_BYTES_USED_KEY; import static org.apache.hadoop.ozone.OzoneConsts.DB_PENDING_DELETE_BLOCK_COUNT_KEY; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong;