diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java index 0c5ae6aeebc7..2f4eaf5ec754 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java @@ -48,7 +48,7 @@ /** * Class that manages Containers created on the datanode. */ -public class ContainerSet { +public class ContainerSet implements Iterable> { private static final Logger LOG = LoggerFactory.getLogger(ContainerSet.class); @@ -201,6 +201,11 @@ public void handleVolumeFailures() { * @return {@literal Iterator>} */ public Iterator> getContainerIterator() { + return iterator(); + } + + @Override + public Iterator> iterator() { return containerMap.values().iterator(); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java index 595aa925a4fc..6a41416bbaa0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java @@ -102,6 +102,10 @@ public String toString() { private Mode mode; + public KeyValueContainerMetadataInspector(Mode mode) { + this.mode = mode; + } + public KeyValueContainerMetadataInspector() { mode = Mode.OFF; } @@ -155,10 +159,15 @@ public boolean isReadOnly() { @Override public void process(ContainerData containerData, DatanodeStore store) { + process(containerData, store, REPORT_LOG); + } + + public String process(ContainerData containerData, DatanodeStore store, + Logger log) { // If the system property to process container metadata was not // specified, or the inspector is unloaded, this method is a no-op. if (mode == Mode.OFF) { - return; + return null; } KeyValueContainerData kvData = null; @@ -167,7 +176,7 @@ public void process(ContainerData containerData, DatanodeStore store) { } else { LOG.error("This inspector only works on KeyValueContainers. Inspection " + "will not be run for container {}", containerData.getContainerID()); - return; + return null; } JsonObject containerJson = inspectContainer(kvData, store); @@ -178,14 +187,17 @@ public void process(ContainerData containerData, DatanodeStore store) { .serializeNulls() .create(); String jsonReport = gson.toJson(containerJson); - if (correct) { - REPORT_LOG.trace(jsonReport); - } else { - REPORT_LOG.error(jsonReport); + if (log != null) { + if (correct) { + log.trace(jsonReport); + } else { + log.error(jsonReport); + } } + return jsonReport; } - private JsonObject inspectContainer(KeyValueContainerData containerData, + static JsonObject inspectContainer(KeyValueContainerData containerData, DatanodeStore store) { JsonObject containerJson = new JsonObject(); @@ -224,7 +236,7 @@ private JsonObject inspectContainer(KeyValueContainerData containerData, return containerJson; } - private JsonObject getDBMetadataJson(Table metadataTable, + static JsonObject getDBMetadataJson(Table metadataTable, KeyValueContainerData containerData) throws IOException { JsonObject dBMetadata = new JsonObject(); @@ -242,14 +254,13 @@ private JsonObject getDBMetadataJson(Table metadataTable, return dBMetadata; } - private JsonObject getAggregateValues(DatanodeStore store, + static JsonObject getAggregateValues(DatanodeStore store, KeyValueContainerData containerData, String schemaVersion) throws IOException { JsonObject aggregates = new JsonObject(); long usedBytesTotal = 0; long blockCountTotal = 0; - long pendingDeleteBlockCountTotal = 0; // Count normal blocks. try (BlockIterator blockIter = store.getBlockIterator(containerData.getContainerID(), @@ -262,7 +273,10 @@ private JsonObject getAggregateValues(DatanodeStore store, } // Count pending delete blocks. + final PendingDelete pendingDelete; if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) { + long pendingDeleteBlockCountTotal = 0; + long pendingDeleteBytes = 0; try (BlockIterator blockIter = store.getBlockIterator(containerData.getContainerID(), containerData.getDeletingBlockKeyFilter())) { @@ -270,18 +284,22 @@ private JsonObject getAggregateValues(DatanodeStore store, while (blockIter.hasNext()) { blockCountTotal++; pendingDeleteBlockCountTotal++; - usedBytesTotal += getBlockLength(blockIter.nextBlock()); + final long bytes = getBlockLength(blockIter.nextBlock()); + usedBytesTotal += bytes; + pendingDeleteBytes += bytes; } } + pendingDelete = new PendingDelete( + pendingDeleteBlockCountTotal, pendingDeleteBytes); } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) { DatanodeStoreSchemaTwoImpl schemaTwoStore = (DatanodeStoreSchemaTwoImpl) store; - pendingDeleteBlockCountTotal = - countPendingDeletesSchemaV2(schemaTwoStore); + pendingDelete = + countPendingDeletesSchemaV2(schemaTwoStore, containerData); } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V3)) { DatanodeStoreSchemaThreeImpl schemaThreeStore = (DatanodeStoreSchemaThreeImpl) store; - pendingDeleteBlockCountTotal = + pendingDelete = countPendingDeletesSchemaV3(schemaThreeStore, containerData); } else { throw new IOException("Failed to process deleted blocks for unknown " + @@ -290,13 +308,12 @@ private JsonObject getAggregateValues(DatanodeStore store, aggregates.addProperty("blockCount", blockCountTotal); aggregates.addProperty("usedBytes", usedBytesTotal); - aggregates.addProperty("pendingDeleteBlocks", - pendingDeleteBlockCountTotal); + pendingDelete.addToJson(aggregates); return aggregates; } - private JsonObject getChunksDirectoryJson(File chunksDir) throws IOException { + static JsonObject getChunksDirectoryJson(File chunksDir) throws IOException { JsonObject chunksDirectory = new JsonObject(); chunksDirectory.addProperty("path", chunksDir.getAbsolutePath()); @@ -321,6 +338,9 @@ private boolean checkAndRepair(JsonObject parent, Table metadataTable = store.getMetadataTable(); + final JsonObject dBMetadata = parent.getAsJsonObject("dBMetadata"); + final JsonObject aggregates = parent.getAsJsonObject("aggregates"); + // Check and repair block count. JsonElement blockCountDB = parent.getAsJsonObject("dBMetadata") .get(OzoneConsts.BLOCK_COUNT); @@ -392,6 +412,36 @@ private boolean checkAndRepair(JsonObject parent, errors.add(usedBytesError); } + // check and repair if db delete count mismatches delete transaction count. + final JsonElement pendingDeleteCountDB = dBMetadata.get( + OzoneConsts.PENDING_DELETE_BLOCK_COUNT); + final long dbDeleteCount = jsonToLong(pendingDeleteCountDB); + final JsonElement pendingDeleteCountAggregate + = aggregates.get(PendingDelete.COUNT); + final long deleteTransactionCount = jsonToLong(pendingDeleteCountAggregate); + if (dbDeleteCount != deleteTransactionCount) { + passed = false; + + final BooleanSupplier deleteCountRepairAction = () -> { + final String key = containerData.getPendingDeleteBlockCountKey(); + try { + // set delete block count metadata table to delete transaction count + metadataTable.put(key, deleteTransactionCount); + return true; + } catch (IOException ex) { + LOG.error("Failed to reset {} for container {}.", + key, containerData.getContainerID(), ex); + } + return false; + }; + + final JsonObject deleteCountError = buildErrorAndRepair( + "dBMetadata." + OzoneConsts.PENDING_DELETE_BLOCK_COUNT, + pendingDeleteCountAggregate, pendingDeleteCountDB, + deleteCountRepairAction); + errors.add(deleteCountError); + } + // check and repair chunks dir. JsonElement chunksDirPresent = parent.getAsJsonObject("chunksDirectory") .get("present"); @@ -421,6 +471,10 @@ private boolean checkAndRepair(JsonObject parent, return passed; } + static long jsonToLong(JsonElement e) { + return e == null || e.isJsonNull() ? 0 : e.getAsLong(); + } + private JsonObject buildErrorAndRepair(String property, JsonElement expected, JsonElement actual, BooleanSupplier repairAction) { JsonObject error = new JsonObject(); @@ -437,30 +491,81 @@ private JsonObject buildErrorAndRepair(String property, JsonElement expected, return error; } - private long countPendingDeletesSchemaV2(DatanodeStoreSchemaTwoImpl - schemaTwoStore) throws IOException { + static class PendingDelete { + static final String COUNT = "pendingDeleteBlocks"; + static final String BYTES = "pendingDeleteBytes"; + + private final long count; + private final long bytes; + + PendingDelete(long count, long bytes) { + this.count = count; + this.bytes = bytes; + } + + void addToJson(JsonObject json) { + json.addProperty(COUNT, count); + json.addProperty(BYTES, bytes); + } + } + + static PendingDelete countPendingDeletesSchemaV2( + DatanodeStoreSchemaTwoImpl schemaTwoStore, + KeyValueContainerData containerData) throws IOException { long pendingDeleteBlockCountTotal = 0; + long pendingDeleteBytes = 0; + Table delTxTable = schemaTwoStore.getDeleteTransactionTable(); + final Table blockDataTable + = schemaTwoStore.getBlockDataTable(); + try (TableIterator> iterator = delTxTable.iterator()) { while (iterator.hasNext()) { DeletedBlocksTransaction txn = iterator.next().getValue(); + final List localIDs = txn.getLocalIDList(); // In schema 2, pending delete blocks are stored in the // transaction object. Since the actual blocks still exist in the // block data table with no prefix, they have already been // counted towards bytes used and total block count above. - pendingDeleteBlockCountTotal += txn.getLocalIDList().size(); + pendingDeleteBlockCountTotal += localIDs.size(); + pendingDeleteBytes += computePendingDeleteBytes( + localIDs, containerData, blockDataTable); } } - return pendingDeleteBlockCountTotal; + return new PendingDelete(pendingDeleteBlockCountTotal, + pendingDeleteBytes); + } + + static long computePendingDeleteBytes(List localIDs, + KeyValueContainerData containerData, + Table blockDataTable) { + long pendingDeleteBytes = 0; + for (long id : localIDs) { + try { + final String blockKey = containerData.getBlockKey(id); + final BlockData blockData = blockDataTable.get(blockKey); + if (blockData != null) { + pendingDeleteBytes += blockData.getSize(); + } + } catch (IOException e) { + LOG.error("Failed to get block " + id + + " in container " + containerData.getContainerID() + + " from blockDataTable", e); + } + } + return pendingDeleteBytes; } - private long countPendingDeletesSchemaV3( + static PendingDelete countPendingDeletesSchemaV3( DatanodeStoreSchemaThreeImpl schemaThreeStore, KeyValueContainerData containerData) throws IOException { long pendingDeleteBlockCountTotal = 0; + long pendingDeleteBytes = 0; + final Table blockDataTable + = schemaThreeStore.getBlockDataTable(); try ( TableIterator> @@ -468,10 +573,14 @@ private long countPendingDeletesSchemaV3( .iterator(containerData.containerPrefix())) { while (iter.hasNext()) { DeletedBlocksTransaction delTx = iter.next().getValue(); - pendingDeleteBlockCountTotal += delTx.getLocalIDList().size(); + final List localIDs = delTx.getLocalIDList(); + pendingDeleteBlockCountTotal += localIDs.size(); + pendingDeleteBytes += computePendingDeleteBytes( + localIDs, containerData, blockDataTable); } - return pendingDeleteBlockCountTotal; } + return new PendingDelete(pendingDeleteBlockCountTotal, + pendingDeleteBytes); } private static long getBlockLength(BlockData block) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java index 4087483d723f..683aa0b0531d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java @@ -196,6 +196,10 @@ public Iterator> getContainers() { return containerSet.getContainerIterator(); } + public Iterable> getContainerSet() { + return containerSet; + } + /** * Return an iterator of containers which are associated with the specified * volume. diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java index c1292ea46c3e..61487f18b91c 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java @@ -65,6 +65,11 @@ public static Iterable versionParameters() { .collect(toList()); } + @Override + public String toString() { + return "schema=" + schemaVersion + ", layout=" + layout; + } + public static List getLayoutList() { return layoutList; } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java index cf18fa8948db..49dc2e9554d2 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java @@ -58,7 +58,7 @@ */ public class TestKeyValueContainerIntegrityChecks { - private static final Logger LOG = + static final Logger LOG = LoggerFactory.getLogger(TestKeyValueContainerIntegrityChecks.class); private final ContainerLayoutTestInfo containerLayoutTestInfo; @@ -75,6 +75,7 @@ public class TestKeyValueContainerIntegrityChecks { public TestKeyValueContainerIntegrityChecks( ContainerTestVersionInfo versionInfo) { + LOG.info("new {} for {}", getClass().getSimpleName(), versionInfo); this.conf = new OzoneConfiguration(); ContainerTestVersionInfo.setTestSchemaVersion( versionInfo.getSchemaVersion(), conf); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java index aea451bc3a0e..f201bf5b4827 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java @@ -22,6 +22,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.interfaces.ContainerInspector; @@ -29,6 +31,9 @@ import org.apache.hadoop.ozone.container.common.utils.ContainerInspectorUtil; import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil; +import org.apache.hadoop.ozone.container.metadata.DatanodeStore; +import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl; +import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl; import org.apache.log4j.PatternLayout; import org.apache.ozone.test.GenericTestUtils; import org.junit.Assert; @@ -36,6 +41,11 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * Tests for {@link KeyValueContainerMetadataInspector}. */ @@ -113,7 +123,7 @@ public void testIncorrectTotalsNoData() throws Exception { KeyValueContainer container = createClosedContainer(createBlocks); setDBBlockAndByteCounts(container.getContainerData(), setBlocks, setBytes); inspectThenRepairOnIncorrectContainer(container.getContainerData(), - createBlocks, setBlocks, setBytes); + createBlocks, setBlocks, setBytes, 0, 0); } @Test @@ -126,7 +136,7 @@ public void testIncorrectTotalsWithData() throws Exception { KeyValueContainer container = createOpenContainer(createBlocks); setDBBlockAndByteCounts(container.getContainerData(), setBlocks, setBytes); inspectThenRepairOnIncorrectContainer(container.getContainerData(), - createBlocks, setBlocks, setBytes); + createBlocks, setBlocks, setBytes, 0, 0); } @Test @@ -151,6 +161,99 @@ public void testCorrectTotalsWithData() throws Exception { inspectThenRepairOnCorrectContainer(container.getContainerData()); } + static class DeletedBlocksTransactionGeneratorForTesting { + private long txId = 100; + private long localId = 2000; + + DeletedBlocksTransaction next(long containerId, int numBlocks) { + final DeletedBlocksTransaction.Builder b + = DeletedBlocksTransaction.newBuilder() + .setContainerID(containerId) + .setTxID(txId++) + .setCount(0); + for (int i = 0; i < numBlocks; i++) { + b.addLocalID(localId++); + } + return b.build(); + } + + List generate( + long containerId, List numBlocks) { + final List transactions = new ArrayList<>(); + for (int n : numBlocks) { + transactions.add(next(containerId, n)); + } + return transactions; + } + } + + static final DeletedBlocksTransactionGeneratorForTesting GENERATOR + = new DeletedBlocksTransactionGeneratorForTesting(); + + @Test + public void testCorrectDeleteWithTransaction() throws Exception { + final int createBlocks = 4; + final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks; + final int deleteCount = 10; + + final KeyValueContainer container = createClosedContainer(createBlocks); + final List deleteTransactions + = GENERATOR.generate(container.getContainerData().getContainerID(), + Arrays.asList(1, 6, 3)); + final long numDeletedLocalIds = deleteTransactions.stream() + .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum(); + LOG.info("deleteTransactions = {}", deleteTransactions); + LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds); + Assert.assertEquals(deleteCount, numDeletedLocalIds); + + setDB(container.getContainerData(), createBlocks, + setBytes, deleteCount, deleteTransactions); + inspectThenRepairOnCorrectContainer(container.getContainerData()); + } + + @Test + public void testIncorrectDeleteWithTransaction() throws Exception { + final int createBlocks = 4; + final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks; + final int deleteCount = 10; + + final KeyValueContainer container = createClosedContainer(createBlocks); + final List deleteTransactions + = GENERATOR.generate(container.getContainerData().getContainerID(), + Arrays.asList(1, 3)); + final long numDeletedLocalIds = deleteTransactions.stream() + .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum(); + LOG.info("deleteTransactions = {}", deleteTransactions); + LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds); + + setDB(container.getContainerData(), createBlocks, + setBytes, deleteCount, deleteTransactions); + inspectThenRepairOnIncorrectContainer(container.getContainerData(), + createBlocks, createBlocks, setBytes, + deleteCount, numDeletedLocalIds); + } + + @Test + public void testIncorrectDeleteWithoutTransaction() throws Exception { + final int createBlocks = 4; + final int setBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createBlocks; + final int deleteCount = 10; + + final KeyValueContainer container = createClosedContainer(createBlocks); + final List deleteTransactions + = Collections.emptyList(); + final long numDeletedLocalIds = deleteTransactions.stream() + .mapToLong(DeletedBlocksTransaction::getLocalIDCount).sum(); + LOG.info("deleteTransactions = {}", deleteTransactions); + LOG.info("numDeletedLocalIds = {}", numDeletedLocalIds); + + setDB(container.getContainerData(), createBlocks, + setBytes, deleteCount, deleteTransactions); + inspectThenRepairOnIncorrectContainer(container.getContainerData(), + createBlocks, createBlocks, setBytes, + deleteCount, numDeletedLocalIds); + } + public void inspectThenRepairOnCorrectContainer( KeyValueContainerData containerData) throws Exception { // No output for correct containers. @@ -169,10 +272,14 @@ public void inspectThenRepairOnCorrectContainer( * @param createdBlocks Number of blocks to create in the container. * @param setBlocks total block count value set in the database. * @param setBytes total used bytes value set in the database. + * @param deleteCount total deleted block count value set in the database. + * @param numDeletedLocalIds total number of deleted block local id count + * in the transactions */ public void inspectThenRepairOnIncorrectContainer( KeyValueContainerData containerData, int createdBlocks, int setBlocks, - int setBytes) throws Exception { + int setBytes, int deleteCount, long numDeletedLocalIds) + throws Exception { int createdBytes = CHUNK_LEN * CHUNKS_PER_BLOCK * createdBlocks; int createdFiles = 0; switch (getChunkLayout()) { @@ -194,24 +301,26 @@ public void inspectThenRepairOnIncorrectContainer( checkJsonReportForIncorrectContainer(inspectJson, containerState, createdBlocks, setBlocks, createdBytes, setBytes, - createdFiles, false); + createdFiles, deleteCount, numDeletedLocalIds, false); // Container should not have been modified in inspect mode. - checkDBBlockAndByteCounts(containerData, setBlocks, setBytes); + checkDbCounts(containerData, setBlocks, setBytes, deleteCount); // Now repair the container. JsonObject repairJson = runInspectorAndGetReport(containerData, KeyValueContainerMetadataInspector.Mode.REPAIR); checkJsonReportForIncorrectContainer(repairJson, containerState, createdBlocks, setBlocks, createdBytes, setBytes, - createdFiles, true); + createdFiles, deleteCount, numDeletedLocalIds, true); // Metadata keys should have been fixed. - checkDBBlockAndByteCounts(containerData, createdBlocks, createdBytes); + checkDbCounts(containerData, createdBlocks, createdBytes, + numDeletedLocalIds); } @SuppressWarnings("checkstyle:ParameterNumber") private void checkJsonReportForIncorrectContainer(JsonObject inspectJson, String expectedContainerState, long createdBlocks, long setBlocks, long createdBytes, long setBytes, long createdFiles, + long setPendingDeleteCount, long createdPendingDeleteCount, boolean shouldRepair) { // Check main container properties. Assert.assertEquals(inspectJson.get("containerID").getAsLong(), @@ -232,7 +341,7 @@ private void checkJsonReportForIncorrectContainer(JsonObject inspectJson, jsonAggregates.get("blockCount").getAsLong()); Assert.assertEquals(createdBytes, jsonAggregates.get("usedBytes").getAsLong()); - Assert.assertEquals(0, + Assert.assertEquals(createdPendingDeleteCount, jsonAggregates.get("pendingDeleteBlocks").getAsLong()); // Check chunks directory. @@ -243,11 +352,24 @@ private void checkJsonReportForIncorrectContainer(JsonObject inspectJson, // Check errors. checkJsonErrorsReport(inspectJson, "dBMetadata.#BLOCKCOUNT", - new JsonPrimitive(createdBlocks), new JsonPrimitive(setBlocks), - shouldRepair); + createdBlocks, setBlocks, shouldRepair); checkJsonErrorsReport(inspectJson, "dBMetadata.#BYTESUSED", - new JsonPrimitive(createdBytes), new JsonPrimitive(setBytes), - shouldRepair); + createdBytes, setBytes, shouldRepair); + checkJsonErrorsReport(inspectJson, "dBMetadata.#PENDINGDELETEBLOCKCOUNT", + createdPendingDeleteCount, setPendingDeleteCount, shouldRepair); + } + + private void checkJsonErrorsReport( + JsonObject jsonReport, String propertyValue, + long correctExpected, long correctActual, + boolean correctRepair) { + if (correctExpected == correctActual) { + return; + } + checkJsonErrorsReport(jsonReport, propertyValue, + new JsonPrimitive(correctExpected), + new JsonPrimitive(correctActual), + correctRepair); } /** @@ -290,16 +412,59 @@ private void checkJsonErrorsReport(JsonObject jsonReport, public void setDBBlockAndByteCounts(KeyValueContainerData containerData, long blockCount, long byteCount) throws Exception { + setDB(containerData, blockCount, byteCount, + 0, Collections.emptyList()); + } + + public void setDB(KeyValueContainerData containerData, + long blockCount, long byteCount, + long dbDeleteCount, List deleteTransactions) + throws Exception { try (DBHandle db = BlockUtils.getDB(containerData, getConf())) { Table metadataTable = db.getStore().getMetadataTable(); // Don't care about in memory state. Just change the DB values. metadataTable.put(containerData.getBlockCountKey(), blockCount); metadataTable.put(containerData.getBytesUsedKey(), byteCount); + metadataTable.put(containerData.getPendingDeleteBlockCountKey(), + dbDeleteCount); + + final DatanodeStore store = db.getStore(); + LOG.info("store {}", store.getClass().getSimpleName()); + if (store instanceof DatanodeStoreSchemaTwoImpl) { + final DatanodeStoreSchemaTwoImpl s2store + = (DatanodeStoreSchemaTwoImpl)store; + final Table delTxTable + = s2store.getDeleteTransactionTable(); + try (BatchOperation batch = store.getBatchHandler() + .initBatchOperation()) { + for (DeletedBlocksTransaction t : deleteTransactions) { + delTxTable.putWithBatch(batch, t.getTxID(), t); + } + store.getBatchHandler().commitBatchOperation(batch); + } + } else if (store instanceof DatanodeStoreSchemaThreeImpl) { + final DatanodeStoreSchemaThreeImpl s3store + = (DatanodeStoreSchemaThreeImpl)store; + final Table delTxTable + = s3store.getDeleteTransactionTable(); + try (BatchOperation batch = store.getBatchHandler() + .initBatchOperation()) { + for (DeletedBlocksTransaction t : deleteTransactions) { + final String key = containerData.getDeleteTxnKey(t.getTxID()); + delTxTable.putWithBatch(batch, key, t); + } + store.getBatchHandler().commitBatchOperation(batch); + } + } else { + throw new UnsupportedOperationException( + "Unsupported store class " + store.getClass().getSimpleName()); + } } } - public void checkDBBlockAndByteCounts(KeyValueContainerData containerData, - long expectedBlockCount, long expectedBytesUsed) throws Exception { + void checkDbCounts(KeyValueContainerData containerData, + long expectedBlockCount, long expectedBytesUsed, + long expectedDeletedCount) throws Exception { try (DBHandle db = BlockUtils.getDB(containerData, getConf())) { Table metadataTable = db.getStore().getMetadataTable(); @@ -308,6 +473,10 @@ public void checkDBBlockAndByteCounts(KeyValueContainerData containerData, long blockCount = metadataTable.get(containerData.getBlockCountKey()); Assert.assertEquals(expectedBlockCount, blockCount); + + final long deleteCount = metadataTable.get( + containerData.getPendingDeleteBlockCountKey()); + Assert.assertEquals(expectedDeletedCount, deleteCount); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java index ea7b10216c05..5592926bf883 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java @@ -79,6 +79,7 @@ ListSubcommand.class, InfoSubcommand.class, ExportSubcommand.class, + InspectSubcommand.class }) @MetaInfServices(SubcommandWithParent.class) public class ContainerCommands implements Callable, SubcommandWithParent { @@ -107,6 +108,10 @@ public Class getParentType() { return OzoneDebug.class; } + OzoneConfiguration getOzoneConf() { + return parent.getOzoneConf(); + } + public void loadContainersFromVolumes() throws IOException { OzoneConfiguration conf = parent.getOzoneConf(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java new file mode 100644 index 000000000000..86f5d54e202c --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/InspectSubcommand.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.container; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerMetadataInspector; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; +import org.apache.hadoop.ozone.container.metadata.DatanodeStore; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +import java.io.IOException; +import java.util.concurrent.Callable; + +/** + * {@code ozone debug container inspect}, + * a command to run {@link KeyValueContainerMetadataInspector}. + */ +@Command( + name = "inspect", + description + = "Check the metadata of all container replicas on this datanode.") +public class InspectSubcommand implements Callable { + + @CommandLine.ParentCommand + private ContainerCommands parent; + + @Override + public Void call() throws IOException { + final OzoneConfiguration conf = parent.getOzoneConf(); + parent.loadContainersFromVolumes(); + + final KeyValueContainerMetadataInspector inspector + = new KeyValueContainerMetadataInspector( + KeyValueContainerMetadataInspector.Mode.INSPECT); + for (Container container : parent.getController().getContainerSet()) { + final ContainerData data = container.getContainerData(); + if (!(data instanceof KeyValueContainerData)) { + continue; + } + final KeyValueContainerData kvData = (KeyValueContainerData) data; + try (DatanodeStore store = BlockUtils.getUncachedDatanodeStore( + kvData, conf, true)) { + final String json = inspector.process(kvData, store, null); + System.out.println(json); + } catch (IOException e) { + System.err.print("Failed to inspect container " + + kvData.getContainerID() + ": "); + e.printStackTrace(); + } + } + + return null; + } +}