HDDS-7472. EC: Fix NSSummaryEndpoint#getDiskUsage for EC keys

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java

@@ -17,14 +17,9 @@
  */
 package org.apache.hadoop.ozone.recon.api.handlers;
 
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
@@ -115,27 +110,6 @@ public static String buildSubpath(String path, String nextLevel) {
     return subpath;
   }
 
-  public long getKeySizeWithReplication(OmKeyInfo keyInfo) {
-    OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations();
-    List<OmKeyLocationInfo> keyLocations =
-        locationGroup.getBlocksLatestVersionOnly();
-    long du = 0L;
-    // a key could be too large to fit in one single container
-    for (OmKeyLocationInfo location: keyLocations) {
-      BlockID block = location.getBlockID();
-      ContainerID containerId = new ContainerID(block.getContainerID());
-      try {
-        int replicationFactor =
-            containerManager.getContainerReplicas(containerId).size();
-        long blockSize = location.getLength() * replicationFactor;
-        du += blockSize;
-      } catch (ContainerNotFoundException cnfe) {
-        LOG.warn("Cannot find container {}", block.getContainerID(), cnfe);
-      }
-    }
-    return du;
-  }
-
   /**
    * Example: /vol1/buck1/a/b/c/d/e/file1.txt -> a/b/c/d/e/file1.txt.
    * @param names parsed request

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java

@@ -143,7 +143,7 @@ public long calculateDUUnderObject(long parentId)
         }
         OmKeyInfo keyInfo = kv.getValue();
         if (keyInfo != null) {
-          totalDU += getKeySizeWithReplication(keyInfo);
+          totalDU += keyInfo.getReplicatedSize();
         }
       }
     }
@@ -212,7 +212,7 @@ public long handleDirectKeys(long parentId, boolean withReplica,
           diskUsage.setSize(keyInfo.getDataSize());
 
           if (withReplica) {
-            long keyDU = getKeySizeWithReplication(keyInfo);
+            long keyDU = keyInfo.getReplicatedSize();
             keyDataSizeWithReplica += keyDU;
             diskUsage.setSizeWithReplica(keyDU);
           }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java

@@ -64,8 +64,7 @@ public DUResponse getDuResponse(
 
     duResponse.setSize(keyInfo.getDataSize());
     if (withReplica) {
-      long keySizeWithReplica = getBucketHandler()
-              .getKeySizeWithReplication(keyInfo);
+      long keySizeWithReplica = keyInfo.getReplicatedSize();
       duResponse.setSizeWithReplica(keySizeWithReplica);
     }
     return duResponse;

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java

@@ -163,7 +163,7 @@ public long calculateDUUnderObject(long parentId)
         if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) {
           continue;
         }
-        totalDU += getKeySizeWithReplication(keyInfo);
+        totalDU += keyInfo.getReplicatedSize();
       }
     }
 
@@ -248,7 +248,7 @@ public long handleDirectKeys(long parentId, boolean withReplica,
         diskUsage.setSize(keyInfo.getDataSize());
 
         if (withReplica) {
-          long keyDU = getKeySizeWithReplication(keyInfo);
+          long keyDU = keyInfo.getReplicatedSize();
           keyDataSizeWithReplica += keyDU;
           diskUsage.setSizeWithReplica(keyDU);
         }

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java

@@ -233,7 +233,8 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager,
                                   long bucketObjectId,
                                   long volumeObjectId,
                                   List<OmKeyLocationInfoGroup> locationVersions,
-                                  BucketLayout bucketLayout)
+                                  BucketLayout bucketLayout,
+                                  long dataSize)
           throws IOException {
 
     String omKey;
@@ -248,6 +249,7 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager,
                     .setBucketName(bucketName)
                     .setVolumeName(volName)
                     .setKeyName(keyName)
+                    .setDataSize(dataSize)
                     .setOmKeyLocationInfos(locationVersions)
                     .setReplicationConfig(
                         StandaloneReplicationConfig.getInstance(ONE))

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java

@@ -18,7 +18,11 @@
 
 package org.apache.hadoop.ozone.recon.api;
 
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize;
+
 import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
@@ -209,46 +213,47 @@ public class TestNSSummaryEndpointWithFSO {
   private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
   private static final long KEY_FIVE_SIZE = 100L; // bin 0
   private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
+  private static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1;
   private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1
   private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
   private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
   private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1
-  private static final long LOCATION_INFO_GROUP_ONE_SIZE
-          = CONTAINER_ONE_REPLICA_COUNT * BLOCK_ONE_LENGTH
-          + CONTAINER_TWO_REPLICA_COUNT * BLOCK_TWO_LENGTH
-          + CONTAINER_THREE_REPLICA_COUNT * BLOCK_THREE_LENGTH;
-
-  private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA
-          = LOCATION_INFO_GROUP_ONE_SIZE;
-
-  private static final long LOCATION_INFO_GROUP_TWO_SIZE
-      = CONTAINER_FOUR_REPLICA_COUNT * BLOCK_FOUR_LENGTH
-      + CONTAINER_FIVE_REPLICA_COUNT * BLOCK_FIVE_LENGTH
-      + CONTAINER_SIX_REPLICA_COUNT * BLOCK_SIX_LENGTH;
 
   private static final long FILE1_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
+      getReplicatedSize(KEY_ONE_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE2_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_TWO_SIZE;
+      getReplicatedSize(KEY_TWO_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE3_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
+      getReplicatedSize(KEY_THREE_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE4_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_TWO_SIZE;
+      getReplicatedSize(KEY_FOUR_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE5_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
+      getReplicatedSize(KEY_FIVE_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE6_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_TWO_SIZE;
+      getReplicatedSize(KEY_SIX_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE7_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
+      getReplicatedSize(KEY_SEVEN_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE8_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_TWO_SIZE;
+      getReplicatedSize(KEY_EIGHT_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE9_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
+      getReplicatedSize(KEY_NINE_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE10_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_TWO_SIZE;
+      getReplicatedSize(KEY_TEN_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
   private static final long FILE11_SIZE_WITH_REPLICA =
-      LOCATION_INFO_GROUP_ONE_SIZE;
-
+      getReplicatedSize(KEY_ELEVEN_SIZE,
+              StandaloneReplicationConfig.getInstance(ONE));
+  private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA
+          = FILE7_SIZE_WITH_REPLICA;
   private static final long
       MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT
       = FILE1_SIZE_WITH_REPLICA
@@ -943,7 +948,8 @@ private void setUpMultiBlockKey() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_SEVEN_SIZE);
   }
 
   private OmKeyLocationInfoGroup getLocationInfoGroup1() {
@@ -1036,7 +1042,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_ONE_SIZE);
 
     //vol/bucket1/dir1/dir2/file2
     writeKeyToOm(reconOMMetadataManager,
@@ -1049,7 +1056,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup2),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_TWO_SIZE);
 
     //vol/bucket1/dir1/dir3/file3
     writeKeyToOm(reconOMMetadataManager,
@@ -1062,7 +1070,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_THREE_SIZE);
 
     //vol/bucket2/file4
     writeKeyToOm(reconOMMetadataManager,
@@ -1075,7 +1084,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_TWO_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup2),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_FOUR_SIZE);
 
     //vol/bucket2/file5
     writeKeyToOm(reconOMMetadataManager,
@@ -1088,7 +1098,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_TWO_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_FIVE_SIZE);
 
     //vol/bucket1/dir1/dir4/file6
     writeKeyToOm(reconOMMetadataManager,
@@ -1101,7 +1112,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup2),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_SIX_SIZE);
 
     //vol/bucket1/dir1/file7
     writeKeyToOm(reconOMMetadataManager,
@@ -1114,7 +1126,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_ONE_OBJECT_ID,
         VOL_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_SEVEN_SIZE);
 
     //vol2/bucket3/file8
     writeKeyToOm(reconOMMetadataManager,
@@ -1127,7 +1140,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_THREE_OBJECT_ID,
         VOL_TWO_OBJECT_ID,
         Collections.singletonList(locationInfoGroup2),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_EIGHT_SIZE);
 
     //vol2/bucket3/dir5/file9
     writeKeyToOm(reconOMMetadataManager,
@@ -1140,7 +1154,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_THREE_OBJECT_ID,
         VOL_TWO_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_NINE_SIZE);
 
     //vol2/bucket3/dir5/file10
     writeKeyToOm(reconOMMetadataManager,
@@ -1153,7 +1168,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_THREE_OBJECT_ID,
         VOL_TWO_OBJECT_ID,
         Collections.singletonList(locationInfoGroup2),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_TEN_SIZE);
 
     //vol2/bucket4/file11
     writeKeyToOm(reconOMMetadataManager,
@@ -1166,7 +1182,8 @@ private void setUpMultiBlockReplicatedKeys() throws IOException {
         BUCKET_FOUR_OBJECT_ID,
         VOL_TWO_OBJECT_ID,
         Collections.singletonList(locationInfoGroup1),
-        getBucketLayout());
+        getBucketLayout(),
+        KEY_ELEVEN_SIZE);
   }
 
   /**