HDDS-4254. Bucket space: add usedBytes and update it when create and delete key.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -92,6 +92,11 @@ public class OzoneBucket extends WithMetadata {
    */
   private int listCacheSize;
 
+  /**
+   * Used bytes of the bucket.
+   */
+  private long usedBytes;
+
   /**
    * Creation time of the bucket.
    */
@@ -174,6 +179,18 @@ public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
     this.modificationTime = Instant.ofEpochMilli(modificationTime);
   }
 
+  @SuppressWarnings("parameternumber")
+  public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
+      String volumeName, String bucketName, StorageType storageType,
+      Boolean versioning, long creationTime, long modificationTime,
+      Map<String, String> metadata, String encryptionKeyName,
+      String sourceVolume, String sourceBucket, long usedBytes) {
+    this(conf, proxy, volumeName, bucketName, storageType, versioning,
+        creationTime, metadata, encryptionKeyName, sourceVolume, sourceBucket);
+    this.usedBytes = usedBytes;
+    this.modificationTime = Instant.ofEpochMilli(modificationTime);
+  }
+
   /**
    * Constructs OzoneBucket instance.
    * @param conf Configuration object.
@@ -416,6 +433,10 @@ public OzoneKeyDetails getKey(String key) throws IOException {
     return proxy.getKeyDetails(volumeName, name, key);
   }
 
+  public long getUsedBytes() {
+    return usedBytes;
+  }
+
   /**
    * Returns Iterator to iterate over all keys in the bucket.
    * The result can be restricted using key prefix, will return all

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -634,7 +634,8 @@ public OzoneBucket getBucketDetails(
         bucketInfo.getEncryptionKeyInfo() != null ? bucketInfo
             .getEncryptionKeyInfo().getKeyName() : null,
         bucketInfo.getSourceVolume(),
-        bucketInfo.getSourceBucket()
+        bucketInfo.getSourceBucket(),
+        bucketInfo.getUsedBytes().sum()
     );
   }
 
@@ -658,7 +659,8 @@ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
         bucket.getEncryptionKeyInfo() != null ? bucket
             .getEncryptionKeyInfo().getKeyName() : null,
         bucket.getSourceVolume(),
-        bucket.getSourceBucket()))
+        bucket.getSourceBucket(),
+        bucket.getUsedBytes().sum()))
         .collect(Collectors.toList());
   }
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java

@@ -20,6 +20,7 @@
 
 import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -31,8 +32,7 @@
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.audit.Auditable;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
-    .BucketInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
 import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
 
 import com.google.common.base.Preconditions;
@@ -80,6 +80,8 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
 
   private final String sourceBucket;
 
+  private final LongAdder usedBytes = new LongAdder();
+
   /**
    * Private constructor, constructed via builder.
    * @param volumeName - Volume name.
@@ -93,6 +95,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
    * @param bekInfo - bucket encryption key info.
    * @param sourceVolume - source volume for bucket links, null otherwise
    * @param sourceBucket - source bucket for bucket links, null otherwise
+   * @param usedBytes - Bucket Quota Usage in bytes.
    */
   @SuppressWarnings("checkstyle:ParameterNumber")
   private OmBucketInfo(String volumeName,
@@ -107,7 +110,8 @@ private OmBucketInfo(String volumeName,
       Map<String, String> metadata,
       BucketEncryptionKeyInfo bekInfo,
       String sourceVolume,
-      String sourceBucket) {
+      String sourceBucket,
+      long usedBytes) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.acls = acls;
@@ -121,6 +125,7 @@ private OmBucketInfo(String volumeName,
     this.bekInfo = bekInfo;
     this.sourceVolume = sourceVolume;
     this.sourceBucket = sourceBucket;
+    this.usedBytes.add(usedBytes);
   }
 
   /**
@@ -226,6 +231,10 @@ public String getSourceBucket() {
     return sourceBucket;
   }
 
+  public LongAdder getUsedBytes() {
+    return usedBytes;
+  }
+
   public boolean isLink() {
     return sourceVolume != null && sourceBucket != null;
   }
@@ -261,6 +270,7 @@ public Map<String, String> toAuditMap() {
       auditMap.put(OzoneConsts.SOURCE_VOLUME, sourceVolume);
       auditMap.put(OzoneConsts.SOURCE_BUCKET, sourceBucket);
     }
+    auditMap.put(OzoneConsts.USED_BYTES, String.valueOf(this.usedBytes));
     return auditMap;
   }
 
@@ -296,7 +306,8 @@ public Builder toBuilder() {
         .setSourceVolume(sourceVolume)
         .setSourceBucket(sourceBucket)
         .setAcls(acls)
-        .addAllMetadata(metadata);
+        .addAllMetadata(metadata)
+        .setUsedBytes(usedBytes.sum());
   }
 
   /**
@@ -316,6 +327,7 @@ public static class Builder {
     private BucketEncryptionKeyInfo bekInfo;
     private String sourceVolume;
     private String sourceBucket;
+    private long usedBytes;
 
     public Builder() {
       //Default values
@@ -407,6 +419,11 @@ public Builder setSourceBucket(String bucket) {
       return this;
     }
 
+    public Builder setUsedBytes(long quotaUsage) {
+      this.usedBytes = quotaUsage;
+      return this;
+    }
+
     /**
      * Constructs the OmBucketInfo.
      * @return instance of OmBucketInfo.
@@ -420,7 +437,7 @@ public OmBucketInfo build() {
 
       return new OmBucketInfo(volumeName, bucketName, acls, isVersionEnabled,
           storageType, creationTime, modificationTime, objectID, updateID,
-          metadata, bekInfo, sourceVolume, sourceBucket);
+          metadata, bekInfo, sourceVolume, sourceBucket, usedBytes);
     }
   }
 
@@ -438,6 +455,7 @@ public BucketInfo getProtobuf() {
         .setModificationTime(modificationTime)
         .setObjectID(objectID)
         .setUpdateID(updateID)
+        .setUsedBytes(usedBytes.sum())
         .addAllMetadata(KeyValueUtil.toProtobuf(metadata));
     if (bekInfo != null && bekInfo.getKeyName() != null) {
       bib.setBeinfo(OMPBHelper.convert(bekInfo));
@@ -465,6 +483,7 @@ public static OmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
         .setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
         .setStorageType(StorageType.valueOf(bucketInfo.getStorageType()))
         .setCreationTime(bucketInfo.getCreationTime())
+        .setUsedBytes(bucketInfo.getUsedBytes())
         .setModificationTime(bucketInfo.getModificationTime());
     if (bucketInfo.hasObjectID()) {
       obib.setObjectID(bucketInfo.getObjectID());
@@ -500,6 +519,7 @@ public String getObjectInfo() {
         ", isVersionEnabled='" + isVersionEnabled + "'" +
         ", storageType='" + storageType + "'" +
         ", creationTime='" + creationTime + "'" +
+        ", usedBytes='" + usedBytes.sum() + '\'' +
         sourceInfo +
         '}';
   }
@@ -522,6 +542,7 @@ public boolean equals(Object o) {
         storageType == that.storageType &&
         objectID == that.objectID &&
         updateID == that.updateID &&
+        usedBytes.sum() == that.usedBytes.sum() &&
         Objects.equals(sourceVolume, that.sourceVolume) &&
         Objects.equals(sourceBucket, that.sourceBucket) &&
         Objects.equals(metadata, that.metadata) &&
@@ -548,6 +569,7 @@ public String toString() {
         ", objectID=" + objectID +
         ", updateID=" + updateID +
         ", metadata=" + metadata +
+        ", usedBytes=" + usedBytes.sum() +
         '}';
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java

@@ -708,10 +708,12 @@ public void testPutKey() throws IOException {
   }
 
   @Test
-  public void testVolumeUsedBytes() throws IOException {
+  @SuppressWarnings("methodlength")
+  public void testVolumeAndBucketUsedBytes() throws IOException {
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
     OzoneVolume volume = null;
+    OzoneBucket bucket = null;
 
     int blockSize = (int) ozoneManager.getConfiguration().getStorageSize(
         OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
@@ -727,13 +729,15 @@ public void testVolumeUsedBytes() throws IOException {
     // The initial value should be 0
     Assert.assertEquals(0L, volume.getUsedBytes());
     volume.createBucket(bucketName);
-    OzoneBucket bucket = volume.getBucket(bucketName);
+    bucket = volume.getBucket(bucketName);
 
     //Case1: Test the volumeUsedBytes of ONE replications.
     String keyName1 = UUID.randomUUID().toString();
     writeKey(bucket, keyName1, ONE, value, valueLength);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength, volume.getUsedBytes());
+    Assert.assertEquals(valueLength, bucket.getUsedBytes());
     currentQuotaUsage += valueLength;
 
     // Case2: Test overwrite the same KeyName under ONE Replicates, the
@@ -743,16 +747,22 @@ public void testVolumeUsedBytes() throws IOException {
     // Overwrite the keyName2
     writeKey(bucket, keyName2, ONE, value, valueLength);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 2;
 
     // Case3: Test the volumeUsedBytes of THREE replications.
     String keyName3 = UUID.randomUUID().toString();
     writeKey(bucket, keyName3, THREE, value, valueLength);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 3;
 
     // Case4: Test overwrite the same KeyName under THREE Replicates, the
@@ -762,17 +772,23 @@ public void testVolumeUsedBytes() throws IOException {
     // Overwrite the keyName4
     writeKey(bucket, keyName4, THREE, value, valueLength);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 3 * 2;
 
     //Case5: Do not specify the value Length, simulate HDFS api writing.
     // Test the volumeUsedBytes of ONE replications.
     String keyName5 = UUID.randomUUID().toString();
     writeFile(bucket, keyName5, ONE, value, 0);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength;
 
     // Case6: Do not specify the value Length, simulate HDFS api writing.
@@ -783,17 +799,23 @@ public void testVolumeUsedBytes() throws IOException {
     // Overwrite the keyName6
     writeFile(bucket, keyName6, ONE, value, 0);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 2;
 
     // Case7: Do not specify the value Length, simulate HDFS api writing.
     // Test the volumeUsedBytes of THREE replications.
     String keyName7 = UUID.randomUUID().toString();
     writeFile(bucket, keyName7, THREE, value, 0);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 3;
 
     // Case8: Do not specify the value Length, simulate HDFS api writing.
@@ -804,23 +826,32 @@ public void testVolumeUsedBytes() throws IOException {
     // Overwrite the keyName8
     writeFile(bucket, keyName8, THREE, value, 0);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
         volume.getUsedBytes());
+    Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
+        bucket.getUsedBytes());
     currentQuotaUsage += valueLength * 3 * 2;
 
     // Case9: Test volumeUsedBytes when delete key of ONE replications.
     bucket.deleteKey(keyName1);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(currentQuotaUsage - valueLength,
         volume.getUsedBytes());
+    Assert.assertEquals(currentQuotaUsage - valueLength,
+        bucket.getUsedBytes());
     currentQuotaUsage -= valueLength;
 
     // Case10: Test volumeUsedBytes when delete key of THREE
     // replications.
     bucket.deleteKey(keyName3);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(currentQuotaUsage - valueLength * 3,
         volume.getUsedBytes());
+    Assert.assertEquals(currentQuotaUsage - valueLength * 3,
+        bucket.getUsedBytes());
     currentQuotaUsage -= valueLength * 3;
 
     // Case11: Test volumeUsedBytes when Test Delete keys. At this
@@ -834,7 +865,9 @@ public void testVolumeUsedBytes() throws IOException {
     keyList.add(keyName8);
     bucket.deleteKeys(keyList);
     volume = store.getVolume(volumeName);
+    bucket = volume.getBucket(bucketName);
     Assert.assertEquals(0, volume.getUsedBytes());
+    Assert.assertEquals(0, bucket.getUsedBytes());
   }
 
   @Test
@@ -915,7 +948,7 @@ private void writeFile(OzoneBucket bucket, String keyName,
   }
 
   @Test
-  public void testVolumeQuotaWithUploadPart() throws IOException {
+  public void testUsedBytesWithUploadPart() throws IOException {
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
     String keyName = UUID.randomUUID().toString();
@@ -949,10 +982,14 @@ public void testVolumeQuotaWithUploadPart() throws IOException {
 
     Assert.assertEquals(valueLength, store.getVolume(volumeName)
         .getUsedBytes());
+    Assert.assertEquals(valueLength, store.getVolume(volumeName)
+        .getBucket(bucketName).getUsedBytes());
 
     // Abort uploaded partKey and the usedBytes of volume should be 0.
     bucket.abortMultipartUpload(keyName, uploadID);
     Assert.assertEquals(0, store.getVolume(volumeName).getUsedBytes());
+    Assert.assertEquals(0, store.getVolume(volumeName)
+        .getBucket(bucketName).getUsedBytes());
   }
 
   @Test

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -499,6 +499,7 @@ message BucketInfo {
     optional uint64 modificationTime = 11;
     optional string sourceVolume = 12;
     optional string sourceBucket = 13;
+    optional uint64 usedBytes = 14;
 }
 
 enum StorageTypeProto {