Skip to content

HDDS-8586 Recon. - API for Count of deletePending keys and amount of data mapped to such keys. #4923

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sumitagrawl merged 3 commits into from
Jun 25, 2023
Merged

HDDS-8586 Recon. - API for Count of deletePending keys and amount of data mapped to such keys. #4923

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d4a240d
HDDS-8586. Recon - API for Count of deletePending keys and amount of …
ArafatKhan2198 Jun 17, 2023
caa51c0
Fixed failing UT
ArafatKhan2198 Jun 17, 2023
57d467c
Refactored the process method of OmTableInsightTask
ArafatKhan2198 Jun 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.CopyObject;
Expand Down Expand Up @@ -72,6 +73,24 @@ public List<OmKeyInfo> getOmKeyInfoList() {
return omKeyInfoList;
}

/**
* Returns a pair of long values representing the replicated size and
* unreplicated size of all the keys in the list.
*/
public ImmutablePair<Long, Long> getTotalSize() {
long replicatedSize = 0;
long unreplicatedSize = 0;

for (OmKeyInfo omKeyInfo : omKeyInfoList) {
if (omKeyInfo.getReplicatedSize() != 0) {
replicatedSize += omKeyInfo.getReplicatedSize();
}
unreplicatedSize += omKeyInfo.getDataSize();
}
return new ImmutablePair<Long, Long>(unreplicatedSize, replicatedSize);
}


// HDDS-7041. Return a new ArrayList to avoid ConcurrentModifyException
public List<OmKeyInfo> cloneOmKeyInfoList() {
return new ArrayList<>(omKeyInfoList);
Expand Down
31 changes: 31 additions & 0 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@

import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_KEY_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT;
import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
Expand Down Expand Up @@ -309,6 +310,8 @@ private void getPendingForDeletionKeyInfo(
deletedKeyAndDirInsightInfo.getRepeatedOmKeyInfoList();
Table<String, RepeatedOmKeyInfo> deletedTable =
omMetadataManager.getDeletedTable();
// Create a HashMap for the keysSummary
Map<String, Long> keysSummary = new HashMap<>();
try (
TableIterator<String, ? extends Table.KeyValue<String,
RepeatedOmKeyInfo>>
Expand Down Expand Up @@ -345,6 +348,10 @@ private void getPendingForDeletionKeyInfo(
break;
}
}
// Create the keysSummary for deleted keys
createKeysSummaryForDeletedKey(keysSummary);
// Set the keysSummary and lastKey in the response
deletedKeyAndDirInsightInfo.setKeysSummary(keysSummary);
deletedKeyAndDirInsightInfo.setLastKey(lastKey);
} catch (IOException ex) {
throw new WebApplicationException(ex,
Expand Down Expand Up @@ -422,6 +429,30 @@ public Response getDeletedKeyInfo(
return Response.ok(deletedKeyInsightInfo).build();
}

/**
* Creates a keys summary for deleted keys and updates the provided
* keysSummary map. Calculates the total number of deleted keys, replicated
* data size, and unreplicated data size.
*
* @param keysSummary A map to store the keys summary information.
*/
private void createKeysSummaryForDeletedKey(Map<String, Long> keysSummary) {
// Fetch the necessary metrics for deleted keys
Long replicatedSizeDeleted = getValueFromId(globalStatsDao.findById(
OmTableInsightTask.getReplicatedSizeKeyFromTable(DELETED_TABLE)));
Long unreplicatedSizeDeleted = getValueFromId(globalStatsDao.findById(
OmTableInsightTask.getUnReplicatedSizeKeyFromTable(DELETED_TABLE)));
Long deletedKeyCount = getValueFromId(globalStatsDao.findById(
OmTableInsightTask.getTableCountKeyFromTable(DELETED_TABLE)));

// Calculate the total number of deleted keys
keysSummary.put("totalDeletedKeys", deletedKeyCount);
// Calculate the total replicated and unreplicated sizes
keysSummary.put("totalReplicatedDataSize", replicatedSizeDeleted);
keysSummary.put("totalUnreplicatedDataSize", unreplicatedSizeDeleted);
}


private void getPendingForDeletionDirInfo(
int limit, String prevKey,
KeyInsightInfoResponse pendingForDeletionKeyInfo) {
Expand Down
227 changes: 172 additions & 55 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableInsightTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao;
import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats;
Expand All @@ -43,10 +44,12 @@
import java.util.List;
import java.util.Map;


import java.util.Map.Entry;

import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_KEY_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
import static org.jooq.impl.DSL.currentTimestamp;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.using;
Expand Down Expand Up @@ -158,8 +161,17 @@ private Triple<Long, Long, Long> getTableSizeAndCount(
OmKeyInfo omKeyInfo = (OmKeyInfo) kv.getValue();
unReplicatedSize += omKeyInfo.getDataSize();
replicatedSize += omKeyInfo.getReplicatedSize();
count++;
}
if (kv.getValue() instanceof RepeatedOmKeyInfo) {
RepeatedOmKeyInfo repeatedOmKeyInfo = (RepeatedOmKeyInfo) kv
.getValue();
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
unReplicatedSize += result.getRight();
replicatedSize += result.getLeft();
// Since we can have multiple deleted keys of same name
count += repeatedOmKeyInfo.getOmKeyInfoList().size();
}
count++; // Increment count for each row
}
}
}
Expand All @@ -174,6 +186,7 @@ public Collection<String> getTablesToCalculateSize() {
List<String> taskTables = new ArrayList<>();
taskTables.add(OPEN_KEY_TABLE);
taskTables.add(OPEN_FILE_TABLE);
taskTables.add(DELETED_TABLE);
return taskTables;
}

Expand All @@ -196,80 +209,38 @@ public Collection<String> getTaskTables() {
@Override
public Pair<String, Boolean> process(OMUpdateEventBatch events) {
Iterator<OMDBUpdateEvent> eventIterator = events.getIterator();
// Initialize maps to store count and size information
HashMap<String, Long> objectCountMap = initializeCountMap();
HashMap<String, Long> unreplicatedSizeCountMap = initializeSizeMap(false);
HashMap<String, Long> replicatedSizeCountMap = initializeSizeMap(true);
final Collection<String> taskTables = getTaskTables();
final Collection<String> sizeRelatedTables = getTablesToCalculateSize();

// Process each update event
while (eventIterator.hasNext()) {
OMDBUpdateEvent<String, Object> omdbUpdateEvent = eventIterator.next();
String tableName = omdbUpdateEvent.getTable();

if (!taskTables.contains(tableName)) {
continue;
}

String countKey = getTableCountKeyFromTable(tableName);
String unReplicatedSizeKey =
getUnReplicatedSizeKeyFromTable(tableName);
String replicatedSizeKey =
getReplicatedSizeKeyFromTable(tableName);

try {
switch (omdbUpdateEvent.getAction()) {
case PUT:
objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);

// Compute unreplicated and replicated sizes for size-related tables
if (sizeRelatedTables.contains(tableName) &&
omdbUpdateEvent.getValue() instanceof OmKeyInfo) {
OmKeyInfo omKeyInfo = (OmKeyInfo) omdbUpdateEvent.getValue();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size + omKeyInfo.getDataSize());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size + omKeyInfo.getReplicatedSize());
}
handlePutEvent(omdbUpdateEvent, tableName, sizeRelatedTables,
objectCountMap, unreplicatedSizeCountMap, replicatedSizeCountMap);
break;

case DELETE:
if (omdbUpdateEvent.getValue() != null) {
objectCountMap.computeIfPresent(countKey,
(k, count) -> count > 0 ? count - 1L : 0L);

// Compute unreplicated and replicated sizes for size-related tables
if (sizeRelatedTables.contains(tableName) &&
omdbUpdateEvent.getValue() instanceof OmKeyInfo) {
OmKeyInfo omKeyInfo = (OmKeyInfo) omdbUpdateEvent.getValue();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) ->
size > omKeyInfo.getDataSize() ?
size - omKeyInfo.getDataSize() : 0L);
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) ->
size > omKeyInfo.getReplicatedSize() ?
size - omKeyInfo.getReplicatedSize() : 0L);
}
}
handleDeleteEvent(omdbUpdateEvent, tableName, sizeRelatedTables,
objectCountMap, unreplicatedSizeCountMap, replicatedSizeCountMap);
break;

case UPDATE:
if (omdbUpdateEvent.getValue() instanceof OmKeyInfo &&
sizeRelatedTables.contains(tableName) &&
omdbUpdateEvent.getOldValue() != null) {
OmKeyInfo oldKeyInfo = (OmKeyInfo) omdbUpdateEvent.getOldValue();
OmKeyInfo newKeyInfo = (OmKeyInfo) omdbUpdateEvent.getValue();
// Update key size by subtracting the oldSize and adding newSize
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size - oldKeyInfo.getDataSize() +
newKeyInfo.getDataSize());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size - oldKeyInfo.getReplicatedSize() +
newKeyInfo.getReplicatedSize());
} else if (omdbUpdateEvent.getValue() != null) {
LOG.warn("Update event does not have the old Key Info for {}.",
omdbUpdateEvent.getKey());
}
handleUpdateEvent(omdbUpdateEvent, tableName, sizeRelatedTables,
objectCountMap, unreplicatedSizeCountMap, replicatedSizeCountMap);
break;

default:
LOG.trace("Skipping DB update event : Table: {}, Action: {}",
tableName, omdbUpdateEvent.getAction());
Expand All @@ -281,7 +252,7 @@ public Pair<String, Boolean> process(OMUpdateEventBatch events) {
return new ImmutablePair<>(getTaskName(), false);
}
}

// Write the updated count and size information to the database
if (!objectCountMap.isEmpty()) {
writeDataToDB(objectCountMap);
}
Expand All @@ -291,11 +262,157 @@ public Pair<String, Boolean> process(OMUpdateEventBatch events) {
if (!replicatedSizeCountMap.isEmpty()) {
writeDataToDB(replicatedSizeCountMap);
}

LOG.info("Completed a 'process' run of OmTableInsightTask.");
return new ImmutablePair<>(getTaskName(), true);
}

private void handlePutEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
Collection<String> sizeRelatedTables,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
String countKey = getTableCountKeyFromTable(tableName);
String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);

if (sizeRelatedTables.contains(tableName)) {
handleSizeRelatedTablePutEvent(event, countKey, unReplicatedSizeKey,
replicatedSizeKey, objectCountMap,
unreplicatedSizeCountMap, replicatedSizeCountMap);
} else {
objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
}
}

private void handleSizeRelatedTablePutEvent(
OMDBUpdateEvent<String, Object> event, String countKey,
String unReplicatedSizeKey, String replicatedSizeKey,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
// Handle PUT for OpenKeyTable & OpenFileTable
if (event.getValue() instanceof OmKeyInfo) {
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey, (k, count) -> count + 1L);
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size + omKeyInfo.getDataSize());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size + omKeyInfo.getReplicatedSize());
} else if (event.getValue() instanceof RepeatedOmKeyInfo) {
// Handle PUT for DeletedTable
RepeatedOmKeyInfo repeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey,
(k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size());
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size + result.getLeft());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size + result.getRight());
}
}

private void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
Collection<String> sizeRelatedTables,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
String countKey = getTableCountKeyFromTable(tableName);
String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);

if (event.getValue() != null) {
if (sizeRelatedTables.contains(tableName)) {
handleSizeRelatedTableDeleteEvent(event, countKey, unReplicatedSizeKey,
replicatedSizeKey, objectCountMap,
unreplicatedSizeCountMap, replicatedSizeCountMap);
} else {
objectCountMap.computeIfPresent(countKey,
(k, count) -> count > 0 ? count - 1L : 0L);
}
}
}

private void handleSizeRelatedTableDeleteEvent(
OMDBUpdateEvent<String, Object> event, String countKey,
String unReplicatedSizeKey, String replicatedSizeKey,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
if (event.getValue() instanceof OmKeyInfo) {
// Handle DELETE for OpenKeyTable & OpenFileTable
OmKeyInfo omKeyInfo = (OmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey,
(k, count) -> count > 0 ? count - 1L : 0L);
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size > omKeyInfo.getDataSize() ?
size - omKeyInfo.getDataSize() : 0L);
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size > omKeyInfo.getReplicatedSize() ?
size - omKeyInfo.getReplicatedSize() : 0L);
} else if (event.getValue() instanceof RepeatedOmKeyInfo) {
// Handle DELETE for DeletedTable
RepeatedOmKeyInfo repeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey, (k, count) ->
count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() : 0L);
Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L);
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size > result.getRight() ? size - result.getRight() :
0L);
}
}

private void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
String tableName,
Collection<String> sizeRelatedTables,
HashMap<String, Long> objectCountMap,
HashMap<String, Long> unreplicatedSizeCountMap,
HashMap<String, Long> replicatedSizeCountMap) {
String countKey = getTableCountKeyFromTable(tableName);
String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
// In Update event the count for the table will not change. So we don't
// need to update the count. Except for RepeatedOmKeyInfo, for which the
// size of omKeyInfoList can change
if (event.getValue() instanceof OmKeyInfo && event.getOldValue() != null &&
sizeRelatedTables.contains(tableName)) {
// Handle UPDATE for OpenKeyTable & OpenFileTable
OmKeyInfo oldKeyInfo = (OmKeyInfo) event.getOldValue();
OmKeyInfo newKeyInfo = (OmKeyInfo) event.getValue();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size - oldKeyInfo.getDataSize() +
newKeyInfo.getDataSize());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size - oldKeyInfo.getReplicatedSize() +
newKeyInfo.getReplicatedSize());
} else if (event.getValue() instanceof RepeatedOmKeyInfo &&
sizeRelatedTables.contains(tableName) && event.getOldValue() != null) {
// Handle UPDATE for DeletedTable
RepeatedOmKeyInfo oldRepeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getOldValue();
RepeatedOmKeyInfo newRepeatedOmKeyInfo =
(RepeatedOmKeyInfo) event.getValue();
objectCountMap.computeIfPresent(countKey,
(k, count) -> count > 0 ?
count - oldRepeatedOmKeyInfo.getOmKeyInfoList().size() +
newRepeatedOmKeyInfo.getOmKeyInfoList().size() : 0L);
Pair<Long, Long> oldSize = oldRepeatedOmKeyInfo.getTotalSize();
Pair<Long, Long> newSize = newRepeatedOmKeyInfo.getTotalSize();
unreplicatedSizeCountMap.computeIfPresent(unReplicatedSizeKey,
(k, size) -> size - oldSize.getLeft() + newSize.getLeft());
replicatedSizeCountMap.computeIfPresent(replicatedSizeKey,
(k, size) -> size - oldSize.getRight() + newSize.getRight());
} else if (event.getValue() != null) {
LOG.warn("Update event does not have the old Key Info for {}.",
event.getKey());
}
}


private void writeDataToDB(Map<String, Long> dataMap) {
List<GlobalStats> insertGlobalStats = new ArrayList<>();
Expand Down
Loading