Skip to content

HDDS-7441. Rename function names of retrieving metadata keys #3918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChenSammi merged 4 commits into from
Jan 13, 2023
Merged

HDDS-7441. Rename function names of retrieving metadata keys #3918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
48b90b2
HDDS-7441. Rename function names of retrieving metadata keys
symious Oct 31, 2022
c8cdff7
trigger new CI check
symious Oct 31, 2022
886f16f
trigger new CI check
symious Nov 2, 2022
bc0fd5a
HDDS-7441. Remove meta in function name
symious Jan 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions ...hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,8 @@ private void markBlocksForDeletionSchemaV3(
DeletionMarker schemaV3Marker = (table, batch, tid, txn) -> {
Table<String, DeletedBlocksTransaction> delTxTable =
(Table<String, DeletedBlocksTransaction>) table;
delTxTable.putWithBatch(batch, containerData.deleteTxnKey(tid), txn);
delTxTable.putWithBatch(batch, containerData.getDeleteTxnKey(tid),
txn);
};

markBlocksForDeletionTransaction(containerData, delTX, newDeletionBlocks,
Expand Down Expand Up @@ -402,10 +403,10 @@ private void markBlocksForDeletionSchemaV1(
try (BatchOperation batch = containerDB.getStore().getBatchHandler()
.initBatchOperation()) {
for (Long blkLong : delTX.getLocalIDList()) {
String blk = containerData.blockKey(blkLong);
String blk = containerData.getBlockKey(blkLong);
BlockData blkInfo = blockDataTable.get(blk);
if (blkInfo != null) {
String deletingKey = containerData.deletingBlockKey(blkLong);
String deletingKey = containerData.getDeletingBlockKey(blkLong);
if (blockDataTable.get(deletingKey) != null
|| deletedBlocksTable.get(blk) != null) {
if (LOG.isDebugEnabled()) {
Expand Down Expand Up @@ -457,15 +458,15 @@ private void updateMetaData(KeyValueContainerData containerData,
if (delTX.getTxID() > containerData.getDeleteTransactionId()) {
// Update in DB pending delete key count and delete transaction ID.
metadataTable
.putWithBatch(batchOperation, containerData.latestDeleteTxnKey(),
delTX.getTxID());
.putWithBatch(batchOperation,
containerData.getLatestDeleteTxnKey(), delTX.getTxID());
}

long pendingDeleteBlocks =
containerData.getNumPendingDeletionBlocks() + newDeletionBlocks;
metadataTable
.putWithBatch(batchOperation,
containerData.pendingDeleteBlockCountKey(),
containerData.getPendingDeleteBlockCountKey(),
pendingDeleteBlocks);

// update pending deletion blocks count and delete transaction ID in
Expand Down
2 changes: 1 addition & 1 deletion ...vice/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ private void scanData(DataTransferThrottler throttler, Canceler canceler)
private BlockData getBlockDataFromDB(DBHandle db, BlockData block)
throws IOException {
String blockKey =
onDiskContainerData.blockKey(block.getBlockID().getLocalID());
onDiskContainerData.getBlockKey(block.getBlockID().getLocalID());
return db.getStore().getBlockDataTable().get(blockKey);
}

Expand Down
27 changes: 14 additions & 13 deletions ...rvice/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,11 +306,12 @@ public void updateAndCommitDBCounters(DBHandle db,
Table<String, Long> metadataTable = db.getStore().getMetadataTable();

// Set Bytes used and block count key.
metadataTable.putWithBatch(batchOperation, bytesUsedKey(),
metadataTable.putWithBatch(batchOperation, getBytesUsedKey(),
getBytesUsed() - releasedBytes);
metadataTable.putWithBatch(batchOperation, blockCountKey(),
metadataTable.putWithBatch(batchOperation, getBlockCountKey(),
getBlockCount() - deletedBlockCount);
metadataTable.putWithBatch(batchOperation, pendingDeleteBlockCountKey(),
metadataTable.putWithBatch(batchOperation,
getPendingDeleteBlockCountKey(),
getNumPendingDeletionBlocks() - deletedBlockCount);

db.getStore().getBatchHandler().commitBatchOperation(batchOperation);
Expand All @@ -328,39 +329,39 @@ public void setReplicaIndex(int replicaIndex) {
// to container schemas, we should use them instead of using
// raw const variables defined.

public String blockKey(long localID) {
public String getBlockKey(long localID) {
return formatKey(Long.toString(localID));
}

public String deletingBlockKey(long localID) {
public String getDeletingBlockKey(long localID) {
return formatKey(DELETING_KEY_PREFIX + localID);
}

public String deleteTxnKey(long txnID) {
public String getDeleteTxnKey(long txnID) {
return formatKey(Long.toString(txnID));
}

public String latestDeleteTxnKey() {
public String getLatestDeleteTxnKey() {
return formatKey(DELETE_TRANSACTION_KEY);
}

public String bcsIdKey() {
public String getBcsIdKey() {
return formatKey(BLOCK_COMMIT_SEQUENCE_ID);
}

public String blockCountKey() {
public String getBlockCountKey() {
return formatKey(BLOCK_COUNT);
}

public String bytesUsedKey() {
public String getBytesUsedKey() {
return formatKey(CONTAINER_BYTES_USED);
}

public String pendingDeleteBlockCountKey() {
public String getPendingDeleteBlockCountKey() {
return formatKey(PENDING_DELETE_BLOCK_COUNT);
}

public String deletingBlockKeyPrefix() {
public String getDeletingBlockKeyPrefix() {
return formatKey(DELETING_KEY_PREFIX);
}

Expand All @@ -370,7 +371,7 @@ public KeyPrefixFilter getUnprefixedKeyFilter() {
}

public KeyPrefixFilter getDeletingBlockKeyFilter() {
return new KeyPrefixFilter().addFilter(deletingBlockKeyPrefix());
return new KeyPrefixFilter().addFilter(getDeletingBlockKeyPrefix());
}

/**
Expand Down
14 changes: 7 additions & 7 deletions ...n/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,15 +229,15 @@ private JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
JsonObject dBMetadata = new JsonObject();

dBMetadata.addProperty(OzoneConsts.BLOCK_COUNT,
metadataTable.get(containerData.blockCountKey()));
metadataTable.get(containerData.getBlockCountKey()));
dBMetadata.addProperty(OzoneConsts.CONTAINER_BYTES_USED,
metadataTable.get(containerData.bytesUsedKey()));
metadataTable.get(containerData.getBytesUsedKey()));
dBMetadata.addProperty(OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
metadataTable.get(containerData.pendingDeleteBlockCountKey()));
metadataTable.get(containerData.getPendingDeleteBlockCountKey()));
dBMetadata.addProperty(OzoneConsts.DELETE_TRANSACTION_KEY,
metadataTable.get(containerData.latestDeleteTxnKey()));
metadataTable.get(containerData.getLatestDeleteTxnKey()));
dBMetadata.addProperty(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID,
metadataTable.get(containerData.bcsIdKey()));
metadataTable.get(containerData.getBcsIdKey()));

return dBMetadata;
}
Expand Down Expand Up @@ -341,7 +341,7 @@ private boolean checkAndRepair(JsonObject parent,
BooleanSupplier keyRepairAction = () -> {
boolean repaired = false;
try {
metadataTable.put(containerData.blockCountKey(),
metadataTable.put(containerData.getBlockCountKey(),
blockCountAggregate.getAsLong());
repaired = true;
} catch (IOException ex) {
Expand Down Expand Up @@ -376,7 +376,7 @@ private boolean checkAndRepair(JsonObject parent,
BooleanSupplier keyRepairAction = () -> {
boolean repaired = false;
try {
metadataTable.put(containerData.bytesUsedKey(),
metadataTable.put(containerData.getBytesUsedKey(),
usedBytesAggregate.getAsLong());
repaired = true;
} catch (IOException ex) {
Expand Down
10 changes: 5 additions & 5 deletions ...c/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ private static void populateContainerMetadata(
// Set pending deleted block count.
Long pendingDeleteBlockCount =
metadataTable.get(kvContainerData
.pendingDeleteBlockCountKey());
.getPendingDeleteBlockCountKey());
if (pendingDeleteBlockCount != null) {
kvContainerData.incrPendingDeletionBlocks(
pendingDeleteBlockCount);
Expand All @@ -263,15 +263,15 @@ private static void populateContainerMetadata(

// Set delete transaction id.
Long delTxnId =
metadataTable.get(kvContainerData.latestDeleteTxnKey());
metadataTable.get(kvContainerData.getLatestDeleteTxnKey());
if (delTxnId != null) {
kvContainerData
.updateDeleteTransactionId(delTxnId);
}

// Set BlockCommitSequenceId.
Long bcsId = metadataTable.get(
kvContainerData.bcsIdKey());
kvContainerData.getBcsIdKey());
if (bcsId != null) {
kvContainerData
.updateBlockCommitSequenceId(bcsId);
Expand All @@ -280,15 +280,15 @@ private static void populateContainerMetadata(
// Set bytes used.
// commitSpace for Open Containers relies on usedBytes
Long bytesUsed =
metadataTable.get(kvContainerData.bytesUsedKey());
metadataTable.get(kvContainerData.getBytesUsedKey());
if (bytesUsed != null) {
isBlockMetadataSet = true;
kvContainerData.setBytesUsed(bytesUsed);
}

// Set block count.
Long blockCount = metadataTable.get(
kvContainerData.blockCountKey());
kvContainerData.getBlockCountKey());
if (blockCount != null) {
isBlockMetadataSet = true;
kvContainerData.setBlockCount(blockCount);
Expand Down
14 changes: 7 additions & 7 deletions ...rvice/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,18 +158,18 @@ public static long persistPutBlock(KeyValueContainer container,
// If block exists in cache, blockCount should not be incremented.
if (!isBlockInCache) {
if (db.getStore().getBlockDataTable().get(
containerData.blockKey(localID)) == null) {
containerData.getBlockKey(localID)) == null) {
// Block does not exist in DB => blockCount needs to be
// incremented when the block is added into DB.
incrBlockCount = true;
}
}

db.getStore().getBlockDataTable().putWithBatch(
batch, containerData.blockKey(localID), data);
batch, containerData.getBlockKey(localID), data);
if (bcsId != 0) {
db.getStore().getMetadataTable().putWithBatch(
batch, containerData.bcsIdKey(), bcsId);
batch, containerData.getBcsIdKey(), bcsId);
}

// Set Bytes used, this bytes used will be updated for every write and
Expand All @@ -179,13 +179,13 @@ public static long persistPutBlock(KeyValueContainer container,
// is only used to compute the bytes used. This is done to keep the
// current behavior and avoid DB write during write chunk operation.
db.getStore().getMetadataTable().putWithBatch(
batch, containerData.bytesUsedKey(),
batch, containerData.getBytesUsedKey(),
containerData.getBytesUsed());

// Set Block Count for a container.
if (incrBlockCount) {
db.getStore().getMetadataTable().putWithBatch(
batch, containerData.blockCountKey(),
batch, containerData.getBlockCountKey(),
containerData.getBlockCount() + 1);
}

Expand Down Expand Up @@ -327,7 +327,7 @@ public List<BlockData> listBlock(Container container, long startLocalID, int
try (DBHandle db = BlockUtils.getDB(cData, config)) {
result = new ArrayList<>();
String startKey = (startLocalID == -1) ? cData.startKeyEmpty()
: cData.blockKey(startLocalID);
: cData.getBlockKey(startLocalID);
List<? extends Table.KeyValue<String, BlockData>> range =
db.getStore().getBlockDataTable()
.getSequentialRangeKVs(startKey, count,
Expand All @@ -352,7 +352,7 @@ public void shutdown() {

private BlockData getBlockByID(DBHandle db, BlockID blockID,
KeyValueContainerData containerData) throws IOException {
String blockKey = containerData.blockKey(blockID.getLocalID());
String blockKey = containerData.getBlockKey(blockID.getLocalID());

BlockData blockData = db.getStore().getBlockDataTable().get(blockKey);
if (blockData == null) {
Expand Down
7 changes: 4 additions & 3 deletions .../apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,8 @@ public ContainerBackgroundTaskResult deleteViaSchema3(
Deleter schema3Deleter = (table, batch, tid) -> {
Table<String, DeletedBlocksTransaction> delTxTable =
(Table<String, DeletedBlocksTransaction>) table;
delTxTable.deleteWithBatch(batch, containerData.deleteTxnKey(tid));
delTxTable.deleteWithBatch(batch,
containerData.getDeleteTxnKey(tid));
};
Table<String, DeletedBlocksTransaction> deleteTxns =
((DeleteTransactionStore<String>) meta.getStore())
Expand Down Expand Up @@ -502,7 +503,7 @@ private ContainerBackgroundTaskResult deleteViaTransactionStore(
deleter.apply(deleteTxns, batch, delTx.getTxID());
for (Long blk : delTx.getLocalIDList()) {
blockDataTable.deleteWithBatch(batch,
containerData.blockKey(blk));
containerData.getBlockKey(blk));
}
}

Expand Down Expand Up @@ -550,7 +551,7 @@ private DeleteTransactionStats deleteTransactions(
long bytesReleased = 0;
for (DeletedBlocksTransaction entry : delBlocks) {
for (Long blkLong : entry.getLocalIDList()) {
String blk = containerData.blockKey(blkLong);
String blk = containerData.getBlockKey(blkLong);
BlockData blkInfo = blockDataTable.get(blk);
LOG.debug("Deleting block {}", blkLong);
if (blkInfo == null) {
Expand Down
19 changes: 10 additions & 9 deletions ...vice/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,8 @@ private void createPendingDeleteBlocksSchema1(int numOfBlocksPerContainer,
try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
for (int j = 0; j < numOfBlocksPerContainer; j++) {
blockID = ContainerTestHelper.getTestBlockID(containerID);
String deleteStateName = data.deletingBlockKey(blockID.getLocalID());
String deleteStateName = data.getDeletingBlockKey(
blockID.getLocalID());
BlockData kd = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
putChunksInBlock(numOfChunksPerBlock, j, chunks, buffer, chunkManager,
Expand Down Expand Up @@ -250,7 +251,7 @@ private void createPendingDeleteBlocksViaTxn(int numOfBlocksPerContainer,
container, blockID);
kd.setChunks(chunks);
try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
String blockKey = data.blockKey(blockID.getLocalID());
String blockKey = data.getBlockKey(blockID.getLocalID());
metadata.getStore().getBlockDataTable().put(blockKey, kd);
} catch (IOException exception) {
LOG.info("Exception = " + exception);
Expand Down Expand Up @@ -285,7 +286,7 @@ private void createTxn(KeyValueContainerData data, List<Long> containerBlocks,
DatanodeStoreSchemaThreeImpl dnStoreThreeImpl =
(DatanodeStoreSchemaThreeImpl) ds;
dnStoreThreeImpl.getDeleteTransactionTable()
.putWithBatch(batch, data.deleteTxnKey(txnID), dtx);
.putWithBatch(batch, data.getDeleteTxnKey(txnID), dtx);
} else {
DatanodeStoreSchemaTwoImpl dnStoreTwoImpl =
(DatanodeStoreSchemaTwoImpl) ds;
Expand Down Expand Up @@ -334,12 +335,12 @@ private void updateMetaData(KeyValueContainerData data,
container.getContainerData().setBlockCount(numOfBlocksPerContainer);
// Set block count, bytes used and pending delete block count.
metadata.getStore().getMetadataTable()
.put(data.blockCountKey(), (long) numOfBlocksPerContainer);
.put(data.getBlockCountKey(), (long) numOfBlocksPerContainer);
metadata.getStore().getMetadataTable()
.put(data.bytesUsedKey(),
.put(data.getBytesUsedKey(),
chunkLength * numOfChunksPerBlock * numOfBlocksPerContainer);
metadata.getStore().getMetadataTable()
.put(data.pendingDeleteBlockCountKey(),
.put(data.getPendingDeleteBlockCountKey(),
(long) numOfBlocksPerContainer);
} catch (IOException exception) {
LOG.warn("Meta Data update was not successful for container: "
Expand Down Expand Up @@ -455,7 +456,7 @@ public void testBlockDeletion() throws Exception {
// Ensure there are 3 blocks under deletion and 0 deleted blocks
Assert.assertEquals(3, getUnderDeletionBlocksCount(meta, data));
Assert.assertEquals(3, meta.getStore().getMetadataTable()
.get(data.pendingDeleteBlockCountKey()).longValue());
.get(data.getPendingDeleteBlockCountKey()).longValue());

// Container contains 3 blocks. So, space used by the container
// should be greater than zero.
Expand Down Expand Up @@ -485,9 +486,9 @@ public void testBlockDeletion() throws Exception {
// Check finally DB counters.
// Not checking bytes used, as handler is a mock call.
Assert.assertEquals(0, meta.getStore().getMetadataTable()
.get(data.pendingDeleteBlockCountKey()).longValue());
.get(data.getPendingDeleteBlockCountKey()).longValue());
Assert.assertEquals(0,
meta.getStore().getMetadataTable().get(data.blockCountKey())
meta.getStore().getMetadataTable().get(data.getBlockCountKey())
.longValue());
Assert.assertEquals(3,
deletingServiceMetrics.getSuccessCount()
Expand Down
Loading