diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index dec55627f5df..c0a92936512a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneVolume; @@ -118,8 +119,10 @@ public void testNamespaceSummaryAPI() throws Exception { (ReconOMMetadataManager) cluster.getReconServer().getOzoneManagerServiceProvider() .getOMMetadataManagerInstance(); + OzoneStorageContainerManager reconSCM = + cluster.getReconServer().getReconStorageContainerManager(); NSSummaryEndpoint endpoint = new NSSummaryEndpoint(namespaceSummaryManager, - omMetadataManagerInstance); + omMetadataManagerInstance, reconSCM); Response basicInfo = endpoint.getBasicInfo("/vol1/bucket1/dir1"); NamespaceSummaryResponse entity = (NamespaceSummaryResponse) basicInfo.getEntity(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index 4660c6731fdb..5dec34e0d57a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -19,12 +19,19 @@ package org.apache.hadoop.ozone.recon.api; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManagerV2; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.ReconConstants; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; @@ -36,8 +43,11 @@ import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.inject.Inject; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -48,7 +58,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -62,17 +71,24 @@ @Path("/namespace") @Produces(MediaType.APPLICATION_JSON) public class NSSummaryEndpoint { + + private static final Logger LOG = LoggerFactory.getLogger( + NSSummaryEndpoint.class); @Inject private ReconNamespaceSummaryManager reconNamespaceSummaryManager; @Inject private ReconOMMetadataManager omMetadataManager; + private ContainerManagerV2 containerManager; + @Inject public NSSummaryEndpoint(ReconNamespaceSummaryManager namespaceSummaryManager, - ReconOMMetadataManager omMetadataManager) { + ReconOMMetadataManager omMetadataManager, + OzoneStorageContainerManager reconSCM) { this.reconNamespaceSummaryManager = namespaceSummaryManager; this.omMetadataManager = omMetadataManager; + this.containerManager = reconSCM.getContainerManager(); } /** @@ -104,7 +120,7 @@ public Response getBasicInfo( List allBuckets = listBucketsUnderVolume(null); namespaceSummaryResponse.setNumBucket(allBuckets.size()); int totalNumDir = 0; - int totalNumKey = 0; + long totalNumKey = 0L; for (OmBucketInfo bucket : allBuckets) { long bucketObjectId = bucket.getObjectID(); totalNumDir += getTotalDirCount(bucketObjectId); @@ -119,7 +135,7 @@ public Response getBasicInfo( List buckets = listBucketsUnderVolume(names[0]); namespaceSummaryResponse.setNumBucket(buckets.size()); int totalDir = 0; - int totalKey = 0; + long totalKey = 0L; // iterate all buckets to collect the total object count. for (OmBucketInfo bucket : buckets) { @@ -163,12 +179,19 @@ public Response getBasicInfo( /** * DU endpoint to return datasize for subdirectory (bucket for volume). * @param path request path + * @param listFile show subpath/disk usage for each key + * @param withReplica count actual DU with replication * @return DU response * @throws IOException */ @GET @Path("/du") - public Response getDiskUsage(@QueryParam("path") String path) + @SuppressWarnings("methodlength") + public Response getDiskUsage(@QueryParam("path") String path, + @DefaultValue("false") + @QueryParam("files") boolean listFile, + @DefaultValue("false") + @QueryParam("replica") boolean withReplica) throws IOException { if (path == null || path.length() == 0) { return Response.status(Response.Status.BAD_REQUEST).build(); @@ -179,12 +202,15 @@ public Response getDiskUsage(@QueryParam("path") String path) EntityType type = getEntityType(normalizedPath, names); DUResponse duResponse = new DUResponse(); + duResponse.setPath(normalizedPath); switch (type) { case ROOT: List volumes = listVolumes(); duResponse.setCount(volumes.size()); List volumeDuData = new ArrayList<>(); + long totalDataSize = 0L; + long totalDataSizeWithReplica = 0L; for (OmVolumeArgs volume: volumes) { String volumeName = volume.getVolume(); String subpath = omMetadataManager.getVolumeKey(volumeName); @@ -196,9 +222,22 @@ public Response getDiskUsage(@QueryParam("path") String path) long bucketObjectID = bucket.getObjectID(); dataSize += getTotalSize(bucketObjectID); } + totalDataSize += dataSize; + + // count replicas + // TODO: to be dropped or optimized in the future + if (withReplica) { + long volumeDU = calculateDUForVolume(volumeName); + totalDataSizeWithReplica += volumeDU; + diskUsage.setSizeWithReplica(volumeDU); + } diskUsage.setSize(dataSize); volumeDuData.add(diskUsage); } + if (withReplica) { + duResponse.setSizeWithReplica(totalDataSizeWithReplica); + } + duResponse.setSize(totalDataSize); duResponse.setDuData(volumeDuData); break; case VOLUME: @@ -208,16 +247,28 @@ public Response getDiskUsage(@QueryParam("path") String path) // List of DiskUsage data for all buckets List bucketDuData = new ArrayList<>(); - for (OmBucketInfo bucket : buckets) { + long volDataSize = 0L; + long volDataSizeWithReplica = 0L; + for (OmBucketInfo bucket: buckets) { String bucketName = bucket.getBucketName(); long bucketObjectID = bucket.getObjectID(); String subpath = omMetadataManager.getBucketKey(volName, bucketName); DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); diskUsage.setSubpath(subpath); long dataSize = getTotalSize(bucketObjectID); + volDataSize += dataSize; + if (withReplica) { + long bucketDU = calculateDUUnderObject(bucketObjectID); + diskUsage.setSizeWithReplica(bucketDU); + volDataSizeWithReplica += bucketDU; + } diskUsage.setSize(dataSize); bucketDuData.add(diskUsage); } + if (withReplica) { + duResponse.setSizeWithReplica(volDataSizeWithReplica); + } + duResponse.setSize(volDataSize); duResponse.setDuData(bucketDuData); break; case BUCKET: @@ -227,9 +278,10 @@ public Response getDiskUsage(@QueryParam("path") String path) // get object IDs for all its subdirectories Set bucketSubdirs = bucketNSSummary.getChildDir(); - duResponse.setCount(bucketSubdirs.size()); duResponse.setKeySize(bucketNSSummary.getSizeOfFiles()); List dirDUData = new ArrayList<>(); + long bucketDataSize = duResponse.getKeySize(); + long bucketDataSizeWithReplica = 0L; for (long subdirObjectId: bucketSubdirs) { NSSummary subdirNSSummary = reconNamespaceSummaryManager .getNSSummary(subdirObjectId); @@ -242,9 +294,26 @@ public Response getDiskUsage(@QueryParam("path") String path) DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); diskUsage.setSubpath(subpath); long dataSize = getTotalSize(subdirObjectId); + bucketDataSize += dataSize; + + if (withReplica) { + long dirDU = calculateDUUnderObject(subdirObjectId); + diskUsage.setSizeWithReplica(dirDU); + bucketDataSizeWithReplica += dirDU; + } diskUsage.setSize(dataSize); dirDUData.add(diskUsage); } + // Either listFile or withReplica is enabled, we need the directKeys info + if (listFile || withReplica) { + bucketDataSizeWithReplica += handleDirectKeys(bucketObjectId, + withReplica, listFile, dirDUData, normalizedPath); + } + if (withReplica) { + duResponse.setSizeWithReplica(bucketDataSizeWithReplica); + } + duResponse.setCount(dirDUData.size()); + duResponse.setSize(bucketDataSize); duResponse.setDuData(dirDUData); break; case DIRECTORY: @@ -253,8 +322,9 @@ public Response getDiskUsage(@QueryParam("path") String path) reconNamespaceSummaryManager.getNSSummary(dirObjectId); Set subdirs = dirNSSummary.getChildDir(); - duResponse.setCount(subdirs.size()); duResponse.setKeySize(dirNSSummary.getSizeOfFiles()); + long dirDataSize = duResponse.getKeySize(); + long dirDataSizeWithReplica = 0L; List subdirDUData = new ArrayList<>(); // iterate all subdirectories to get disk usage data for (long subdirObjectId: subdirs) { @@ -267,15 +337,34 @@ public Response getDiskUsage(@QueryParam("path") String path) // reformat the response diskUsage.setSubpath(subpath); long dataSize = getTotalSize(subdirObjectId); + dirDataSize += dataSize; + + if (withReplica) { + long subdirDU = calculateDUUnderObject(subdirObjectId); + diskUsage.setSizeWithReplica(subdirDU); + dirDataSizeWithReplica += subdirDU; + } + diskUsage.setSize(dataSize); subdirDUData.add(diskUsage); } + + // handle direct keys under directory + if (listFile || withReplica) { + dirDataSizeWithReplica += handleDirectKeys(dirObjectId, withReplica, + listFile, subdirDUData, normalizedPath); + } + + if (withReplica) { + duResponse.setSizeWithReplica(dirDataSizeWithReplica); + } + duResponse.setCount(subdirDUData.size()); + duResponse.setSize(dirDataSize); duResponse.setDuData(subdirDUData); break; case KEY: - // DU for key is the data size - duResponse.setCount(1); - DUResponse.DiskUsage keyDU = new DUResponse.DiskUsage(); + // DU for key doesn't have subpaths + duResponse.setCount(0); // The object ID for the directory that the key is directly in long parentObjectId = getDirObjectId(names, names.length - 1); String fileName = names[names.length - 1]; @@ -283,10 +372,11 @@ public Response getDiskUsage(@QueryParam("path") String path) omMetadataManager.getOzonePathKey(parentObjectId, fileName); OmKeyInfo keyInfo = omMetadataManager.getFileTable().getSkipCache(ozoneKey); - String subpath = buildSubpath(normalizedPath, null); - keyDU.setSubpath(subpath); - keyDU.setSize(keyInfo.getDataSize()); - duResponse.setDuData(Collections.singletonList(keyDU)); + duResponse.setSize(keyInfo.getDataSize()); + if (withReplica) { + long keySizeWithReplica = getKeySizeWithReplication(keyInfo); + duResponse.setSizeWithReplica(keySizeWithReplica); + } break; case UNKNOWN: duResponse.setStatus(ResponseStatus.PATH_NOT_FOUND); @@ -590,12 +680,12 @@ private boolean bucketExists(String volName, String bucketName) * @return count of keys * @throws IOException ioEx */ - private int getTotalKeyCount(long objectId) throws IOException { + private long getTotalKeyCount(long objectId) throws IOException { NSSummary nsSummary = reconNamespaceSummaryManager.getNSSummary(objectId); if (nsSummary == null) { - return 0; + return 0L; } - int totalCnt = nsSummary.getNumOfFiles(); + long totalCnt = nsSummary.getNumOfFiles(); for (long childId: nsSummary.getChildDir()) { totalCnt += getTotalKeyCount(childId); } @@ -724,6 +814,138 @@ private List listBucketsUnderVolume(final String volumeName) return result; } + private long calculateDUForVolume(String volumeName) + throws IOException { + long result = 0L; + + Table keyTable = omMetadataManager.getFileTable(); + + TableIterator> + iterator = keyTable.iterator(); + + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + OmKeyInfo keyInfo = kv.getValue(); + + if (keyInfo != null) { + if (volumeName.equals(keyInfo.getVolumeName())) { + result += getKeySizeWithReplication(keyInfo); + } + } + } + return result; + } + + // FileTable's key is in the format of "parentId/fileName" + // Make use of RocksDB's order to seek to the prefix and avoid full iteration + private long calculateDUUnderObject(long parentId) throws IOException { + Table keyTable = omMetadataManager.getFileTable(); + + TableIterator> + iterator = keyTable.iterator(); + + String seekPrefix = parentId + OM_KEY_PREFIX; + iterator.seek(seekPrefix); + long totalDU = 0L; + // handle direct keys + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + String dbKey = kv.getKey(); + // since the RocksDB is ordered, seek until the prefix isn't matched + if (!dbKey.startsWith(seekPrefix)) { + break; + } + OmKeyInfo keyInfo = kv.getValue(); + if (keyInfo != null) { + totalDU += getKeySizeWithReplication(keyInfo); + } + } + + // handle nested keys (DFS) + NSSummary nsSummary = reconNamespaceSummaryManager.getNSSummary(parentId); + Set subDirIds = nsSummary.getChildDir(); + for (long subDirId: subDirIds) { + totalDU += calculateDUUnderObject(subDirId); + } + return totalDU; + } + + /** + * This method handles disk usage of direct keys. + * @param parentId parent directory/bucket + * @param withReplica if withReplica is enabled, set sizeWithReplica + * for each direct key's DU + * @param listFile if listFile is enabled, append key DU as a subpath + * @param duData the current DU data + * @param normalizedPath the normalized path request + * @return the total DU of all direct keys + * @throws IOException IOE + */ + private long handleDirectKeys(long parentId, boolean withReplica, + boolean listFile, + List duData, + String normalizedPath) throws IOException { + + Table keyTable = omMetadataManager.getFileTable(); + TableIterator> + iterator = keyTable.iterator(); + + String seekPrefix = parentId + OM_KEY_PREFIX; + iterator.seek(seekPrefix); + + long keyDataSizeWithReplica = 0L; + + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + String dbKey = kv.getKey(); + + if (!dbKey.startsWith(seekPrefix)) { + break; + } + OmKeyInfo keyInfo = kv.getValue(); + if (keyInfo != null) { + DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); + String subpath = buildSubpath(normalizedPath, + keyInfo.getFileName()); + diskUsage.setSubpath(subpath); + diskUsage.setSize(keyInfo.getDataSize()); + + if (withReplica) { + long keyDU = getKeySizeWithReplication(keyInfo); + keyDataSizeWithReplica += keyDU; + diskUsage.setSizeWithReplica(keyDU); + } + // list the key as a subpath + if (listFile) { + duData.add(diskUsage); + } + } + } + + return keyDataSizeWithReplica; + } + + private long getKeySizeWithReplication(OmKeyInfo keyInfo) { + OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations(); + List keyLocations = + locationGroup.getBlocksLatestVersionOnly(); + long du = 0L; + // a key could be too large to fit in one single container + for (OmKeyLocationInfo location: keyLocations) { + BlockID block = location.getBlockID(); + ContainerID containerId = new ContainerID(block.getContainerID()); + try { + int replicationFactor = + containerManager.getContainerReplicas(containerId).size(); + long blockSize = location.getLength() * replicationFactor; + du += blockSize; + } catch (ContainerNotFoundException cnfe) { + LOG.warn("Cannot find container {}", block.getContainerID(), cnfe); + } + } + return du; + } + /** * Helper function to check if a path is a directory, key, or invalid. * @param keyName key name diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DUResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DUResponse.java index 347ca5d0185a..68a1fa67521b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DUResponse.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DUResponse.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; import java.util.List; /** @@ -30,6 +31,18 @@ public class DUResponse { @JsonProperty("status") private ResponseStatus status; + /** The current path request. */ + @JsonProperty("path") + private String path; + + /** Total size under current path.*/ + @JsonProperty("size") + private long size; + + /** Total size with replicas counted.*/ + @JsonProperty("sizeWithReplica") + private long sizeWithReplica; + /** The number of subpaths under the request path. */ @JsonProperty("subPathCount") private int count; @@ -43,6 +56,9 @@ public class DUResponse { public DUResponse() { this.status = ResponseStatus.OK; + this.duData = new ArrayList<>(); + // by default, the replication feature is disabled + this.sizeWithReplica = -1L; } public ResponseStatus getStatus() { @@ -53,6 +69,30 @@ public void setStatus(ResponseStatus status) { this.status = status; } + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public long getSizeWithReplica() { + return sizeWithReplica; + } + + public void setSizeWithReplica(long sizeWithReplica) { + this.sizeWithReplica = sizeWithReplica; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + public int getCount() { return count; } @@ -89,6 +129,14 @@ public static class DiskUsage { @JsonProperty("size") private long size; + /** Disk usage with replication under the subpath. */ + @JsonProperty("sizeWithReplica") + private long sizeWithReplica; + + public DiskUsage() { + this.sizeWithReplica = -1L; + } + public long getSize() { return size; } @@ -97,6 +145,10 @@ public String getSubpath() { return subpath; } + public long getSizeWithReplica() { + return sizeWithReplica; + } + public void setSize(long size) { this.size = size; } @@ -104,5 +156,9 @@ public void setSize(long size) { public void setSubpath(String subpath) { this.subpath = subpath; } + + public void setSizeWithReplica(long sizeWithReplica) { + this.sizeWithReplica = sizeWithReplica; + } } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java index 3b31207901da..2a2d9f6de746 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java @@ -27,20 +27,21 @@ public class NamespaceSummaryResponse { @JsonProperty("type") private EntityType entityType; + /** Total number of volumes under root, -1 for other types. */ @JsonProperty("numVolume") private int numVolume; - /** Total number of buckets for volume, 0 for other types. */ + /** Total number of buckets for root/volume, -1 for other types. */ @JsonProperty("numBucket") private int numBucket; - /** Total number of directories for a bucket or directory, 0 for others. */ + /** Total number of directories for all types except key, -1 for key. */ @JsonProperty("numDir") private int numTotalDir; - /** Total number of keys for a bucket or directory, 0 for others. */ + /** Total number of keys. */ @JsonProperty("numKey") - private int numTotalKey; + private long numTotalKey; /** Path Status. */ @JsonProperty("status") @@ -48,9 +49,9 @@ public class NamespaceSummaryResponse { public NamespaceSummaryResponse(EntityType entityType) { this.entityType = entityType; - this.numVolume = 0; - this.numBucket = 0; - this.numTotalDir = 0; + this.numVolume = -1; + this.numBucket = -1; + this.numTotalDir = -1; this.numTotalKey = 0; this.status = ResponseStatus.OK; } @@ -71,7 +72,7 @@ public int getNumTotalDir() { return this.numTotalDir; } - public int getNumTotalKey() { + public long getNumTotalKey() { return this.numTotalKey; } @@ -95,7 +96,7 @@ public void setNumTotalDir(int numTotalDir) { this.numTotalDir = numTotalDir; } - public void setNumTotalKey(int numTotalKey) { + public void setNumTotalKey(long numTotalKey) { this.numTotalKey = numTotalKey; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index 07040a2a32ef..11f0373ca0db 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -210,6 +210,29 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager, .build()); } + @SuppressWarnings("checkstyle:parameternumber") + public static void writeKeyToOm(OMMetadataManager omMetadataManager, + long parentObjectId, + long objectId, + String volName, + String bucketName, + String keyName, + String fileName, + List locationVersions) + throws IOException { + String omKey = omMetadataManager.getOzonePathKey(parentObjectId, fileName); + omMetadataManager.getKeyTable().put(omKey, + new OmKeyInfo.Builder() + .setBucketName(bucketName) + .setVolumeName(volName) + .setKeyName(keyName) + .setOmKeyLocationInfos(locationVersions) + .setReplicationConfig(new StandaloneReplicationConfig(ONE)) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build()); + } + public static void writeDirToOm(OMMetadataManager omMetadataManager, long objectId, long parentObjectId, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java index b663f470c845..5b6887116e37 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java @@ -18,11 +18,21 @@ package org.apache.hadoop.ozone.recon.api; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManagerV2; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.recon.ReconConstants; @@ -34,8 +44,11 @@ import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask; import org.junit.Assert; import org.junit.Before; @@ -50,12 +63,18 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.ArrayList; +import java.util.Set; +import java.util.HashSet; +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Test for NSSummary REST APIs. @@ -99,6 +118,8 @@ public class TestNSSummaryEndpoint { private static final String KEY_FOUR = "file4"; private static final String KEY_FIVE = "file5"; private static final String KEY_SIX = "dir1/dir4/file6"; + private static final String MULTI_BLOCK_KEY = "dir1/file7"; + private static final String MULTI_BLOCK_FILE = "file7"; private static final String FILE_ONE = "file1"; private static final String FILE_TWO = "file2"; private static final String FILE_THREE = "file3"; @@ -124,6 +145,22 @@ public class TestNSSummaryEndpoint { private static final long KEY_SIX_OBJECT_ID = 10L; private static final long DIR_THREE_OBJECT_ID = 11L; private static final long DIR_FOUR_OBJECT_ID = 12L; + private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; + + // container IDs + private static final long CONTAINER_ONE_ID = 1L; + private static final long CONTAINER_TWO_ID = 2L; + private static final long CONTAINER_THREE_ID = 3L; + + // replication factors + private static final int THREE = 3; + private static final int TWO = 2; + private static final int FOUR = 4; + + // block lengths + private static final long BLOCK_ONE_LENGTH = 1000L; + private static final long BLOCK_TWO_LENGTH = 2000L; + private static final long BLOCK_THREE_LENGTH = 3000L; // data size in bytes private static final long KEY_ONE_SIZE = 500L; // bin 0 @@ -132,6 +169,10 @@ public class TestNSSummaryEndpoint { private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 private static final long KEY_FIVE_SIZE = 100L; // bin 0 private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA + = THREE * BLOCK_ONE_LENGTH + + TWO * BLOCK_TWO_LENGTH + + FOUR * BLOCK_THREE_LENGTH; // quota in bytes private static final long VOL_QUOTA = 2 * OzoneConsts.MB; @@ -148,6 +189,7 @@ public class TestNSSummaryEndpoint { private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; private static final String KEY_PATH = "/vol/bucket2/file4"; + private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; private static final String INVALID_PATH = "/vol/path/not/found"; // some expected answers @@ -178,6 +220,10 @@ public void setUp() throws Exception { .withOmServiceProvider(ozoneManagerServiceProvider) .withReconSqlDb() .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) .addBinding(NSSummaryEndpoint.class) .build(); reconNamespaceSummaryManager = @@ -257,7 +303,8 @@ public void testBasic() throws Exception { @Test public void testDiskUsage() throws Exception { // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH); + Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, + false, false); DUResponse duVolRes = (DUResponse) volResponse.getEntity(); Assert.assertEquals(2, duVolRes.getCount()); List duData = duVolRes.getDuData(); @@ -272,7 +319,8 @@ public void testDiskUsage() throws Exception { Assert.assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH); + Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, + false, false); DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); Assert.assertEquals(1, duBucketResponse.getCount()); DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); @@ -280,7 +328,8 @@ public void testDiskUsage() throws Exception { Assert.assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); // dir level DU - Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH); + Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, + false, false); DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); Assert.assertEquals(3, duDirReponse.getCount()); List duSubDir = duDirReponse.getDuData(); @@ -299,18 +348,31 @@ public void testDiskUsage() throws Exception { Assert.assertEquals(KEY_SIX_SIZE, duDir4.getSize()); // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH); + Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, + false, false); DUResponse keyObj = (DUResponse) keyResponse.getEntity(); - Assert.assertEquals(1, keyObj.getCount()); - Assert.assertEquals(KEY_FOUR_SIZE, keyObj.getDuData().get(0).getSize()); + Assert.assertEquals(0, keyObj.getCount()); + Assert.assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH); + Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, + false, false); DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); } + @Test + public void testDiskUsageWithReplication() throws Exception { + setUpMultiBlockKey(); + Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, + replicaDUResponse.getSizeWithReplica()); + } + @Test public void testQuotaUsage() throws Exception { // volume level quota usage @@ -495,4 +557,92 @@ private static OMMetadataManager initializeNewOmMetadataManager( return omMetadataManager; } + + private void setUpMultiBlockKey() throws IOException { + List locationInfoList = new ArrayList<>(); + BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); + BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); + BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); + + OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() + .setBlockID(block1) + .setLength(BLOCK_ONE_LENGTH) + .build(); + OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() + .setBlockID(block2) + .setLength(BLOCK_TWO_LENGTH) + .build(); + OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() + .setBlockID(block3) + .setLength(BLOCK_THREE_LENGTH) + .build(); + locationInfoList.add(location1); + locationInfoList.add(location2); + locationInfoList.add(location3); + + OmKeyLocationInfoGroup locationInfoGroup = + new OmKeyLocationInfoGroup(0L, locationInfoList); + + // add the multi-block key to Recon's OM + writeKeyToOm(reconOMMetadataManager, + DIR_ONE_OBJECT_ID, + MULTI_BLOCK_KEY_OBJECT_ID, + VOL, BUCKET_ONE, + MULTI_BLOCK_KEY, + MULTI_BLOCK_FILE, + Collections.singletonList(locationInfoGroup)); + } + + /** + * Generate a set of mock container replica with a size of + * replication factor for container. + * @param replicationFactor number of replica + * @param containerID the container replicated based upon + * @return a set of container replica for testing + */ + private static Set generateMockContainerReplicas( + int replicationFactor, ContainerID containerID) { + Set result = new HashSet<>(); + for (int i = 0; i < replicationFactor; ++i) { + DatanodeDetails randomDatanode = randomDatanodeDetails(); + ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(containerID) + .setContainerState(State.OPEN) + .setDatanodeDetails(randomDatanode) + .build(); + result.add(replica); + } + return result; + } + + private static ReconStorageContainerManagerFacade getMockReconSCM() + throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManagerV2 containerManagerV2 = mock(ContainerManagerV2.class); + + // Container 1 is 3-way replicated + ContainerID containerID1 = new ContainerID(CONTAINER_ONE_ID); + Set containerReplicas1 = generateMockContainerReplicas( + THREE, containerID1); + when(containerManagerV2.getContainerReplicas(containerID1)) + .thenReturn(containerReplicas1); + + // Container 2 is under replicated with 2 replica + ContainerID containerID2 = new ContainerID(CONTAINER_TWO_ID); + Set containerReplicas2 = generateMockContainerReplicas( + TWO, containerID2); + when(containerManagerV2.getContainerReplicas(containerID2)) + .thenReturn(containerReplicas2); + + // Container 3 is over replicated with 4 replica + ContainerID containerID3 = new ContainerID(CONTAINER_THREE_ID); + Set containerReplicas3 = generateMockContainerReplicas( + FOUR, containerID3); + when(containerManagerV2.getContainerReplicas(containerID3)) + .thenReturn(containerReplicas3); + + when(reconSCM.getContainerManager()).thenReturn(containerManagerV2); + return reconSCM; + } }