HDDS-4308. Fix issue with quota update

hadoop-hdds/docs/content/feature/Quota.md

@@ -31,7 +31,12 @@ So far, we know that Ozone allows users to create volumes, buckets, and keys. A
 ## Currently supported
 1. Storage Space level quota
 
-Administrators should be able to define how much storage space a Volume or Bucket can use.
+Administrators should be able to define how much storage space a Volume or Bucket can use. The following Settings for Storage space quota are currently supported：
+a. By default, the quota for volume and bucket is not enabled.
+b. When volume quota is enabled, the total size of bucket quota cannot exceed volume.
+c. Bucket quota can be set separately without enabling Volume quota. The size of bucket quota is unrestricted at this point.
+d. Volume quota is not currently supported separately, and volume quota takes effect only if bucket quota is set. Because ozone only check the usedBytes of the bucket when we write the key.
+
 
 ## Client usage
 ### Storage Space level quota
@@ -59,7 +64,7 @@ bin/ozone sh bucket setquota  --space-quota 10GB /volume1/bucket1
 ```
 This behavior changes the quota for Bucket1 to 10GB
 
-A bucket quota should not be greater than its Volume quota. Let's look at an example. If we have a 10MB Volume and create five buckets under that Volume with a quota of 5MB, the total quota is 25MB. In this case, the bucket creation will always succeed, and we check the quota for bucket and volume when the data is actually written. Each write needs to check whether the current bucket is exceeding the limit and the current total volume usage is exceeding the limit.
+Total bucket quota should not be greater than its Volume quota. If we have a 10MB Volume, The sum of the sizes of all buckets under this volume cannot exceed 10MB, otherwise the bucket set quota fails.
 
 #### Clear the quota for Volume1. The Bucket cleanup command is similar.
 ```shell

hadoop-hdds/docs/content/feature/Quota.zh.md

@@ -26,7 +26,11 @@ icon: user
 ## 目前支持的
 1. Storage space级别配额
 
- 管理员应该能够定义一个Volume或Bucket可以使用多少存储空间。
+ 管理员应该能够定义一个Volume或Bucket可以使用多少存储空间。目前支持以下storage space quota的设置:
+ a. 默认情况下volume和bucket的quota不启用。
+ b. 当volume quota启用时，bucket quota的总大小不能超过volume。
+ c. 可以在不启用volume quota的情况下单独给bucket设置quota。此时bucket quota的大小是不受限制的。
+ d. 目前不支持单独设置volume quota，只有在设置了bucket quota的情况下volume quota才会生效。因为ozone在写入key时只检查bucket的usedBytes。
 
 ## 客户端用法
 ### Storage space级别配额
@@ -53,7 +57,7 @@ bin/ozone sh bucket setquota  --space-quota 10GB /volume1/bucket1
 ```
  该行为将bucket1的配额更改为10GB
 
-一个bucket配额 不应大于其Volume的配额。让我们看一个例子，如果我们有一个10MB的Volume，并在该Volume下创建5个Bucket，配额为5MB，则总配额为25MB。在这种情况下，创建存储桶将始终成功，我们会在数据真正写入时检查bucket和volume的quota。每次写入需要检查当前bucket的是否超上限，当前总的volume使用量是否超上限。
+bucket的总配额 不应大于其Volume的配额。让我们看一个例子，如果我们有一个10MB的Volume，该volume下所有bucket的大小之和不能超过10MB，否则设置bucket quota将失败。
 
 #### 清除Volume1的配额, Bucket清除命令与此类似
 ```shell

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java

@@ -85,8 +85,6 @@ public class OzoneVolume extends WithMetadata {
 
   private int listCacheSize;
 
-  private long usedBytes;
-
   /**
    * Constructs OzoneVolume instance.
    * @param conf Configuration object.
@@ -135,17 +133,6 @@ public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy,
     this.modificationTime = Instant.ofEpochMilli(modificationTime);
   }
 
-  @SuppressWarnings("parameternumber")
-  public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy,
-      String name, String admin, String owner, long quotaInBytes,
-      long quotaInCounts, long creationTime, long modificationTime,
-      List<OzoneAcl> acls, Map<String, String> metadata,
-      long usedBytes) {
-    this(conf, proxy, name, admin, owner, quotaInBytes, quotaInCounts,
-        creationTime, acls, metadata);
-    this.usedBytes = usedBytes;
-  }
-
   @SuppressWarnings("parameternumber")
   public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy,
       String name, String admin, String owner, long quotaInBytes,
@@ -269,10 +256,6 @@ public List<OzoneAcl> getAcls() {
     return acls;
   }
 
-  public long getUsedBytes() {
-    return usedBytes;
-  }
-
   /**
    * Sets/Changes the owner of this Volume.
    * @param userName new owner

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -361,8 +361,7 @@ public OzoneVolume getVolumeDetails(String volumeName)
         volume.getModificationTime(),
         volume.getAclMap().ozoneAclGetProtobuf().stream().
             map(OzoneAcl::fromProtobuf).collect(Collectors.toList()),
-        volume.getMetadata(),
-        volume.getUsedBytes().sum());
+        volume.getMetadata());
   }
 
   @Override
@@ -418,8 +417,7 @@ public List<OzoneVolume> listVolumes(String user, String volumePrefix,
         volume.getModificationTime(),
         volume.getAclMap().ozoneAclGetProtobuf().stream().
             map(OzoneAcl::fromProtobuf).collect(Collectors.toList()),
-        volume.getMetadata(),
-        volume.getUsedBytes().sum()))
+        volume.getMetadata()))
         .collect(Collectors.toList());
   }
 
@@ -674,7 +672,7 @@ public OzoneBucket getBucketDetails(
             .getEncryptionKeyInfo().getKeyName() : null,
         bucketInfo.getSourceVolume(),
         bucketInfo.getSourceBucket(),
-        bucketInfo.getUsedBytes().sum(),
+        bucketInfo.getUsedBytes(),
         bucketInfo.getQuotaInBytes(),
         bucketInfo.getQuotaInCounts()
     );
@@ -701,7 +699,7 @@ public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
             .getEncryptionKeyInfo().getKeyName() : null,
         bucket.getSourceVolume(),
         bucket.getSourceBucket(),
-        bucket.getUsedBytes().sum(),
+        bucket.getUsedBytes(),
         bucket.getQuotaInBytes(),
         bucket.getQuotaInCounts()))
         .collect(Collectors.toList());

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java

@@ -20,7 +20,6 @@
 
 import java.util.ArrayList;
 import java.util.BitSet;
-import java.util.concurrent.atomic.LongAdder;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -80,7 +79,7 @@ public final class OmBucketInfo extends WithObjectID implements Auditable {
 
   private final String sourceBucket;
 
-  private final LongAdder usedBytes = new LongAdder();
+  private long usedBytes;
 
   private long quotaInBytes;
   private long quotaInCounts;
@@ -132,7 +131,7 @@ private OmBucketInfo(String volumeName,
     this.bekInfo = bekInfo;
     this.sourceVolume = sourceVolume;
     this.sourceBucket = sourceBucket;
-    this.usedBytes.add(usedBytes);
+    this.usedBytes = usedBytes;
     this.quotaInBytes = quotaInBytes;
     this.quotaInCounts = quotaInCounts;
   }
@@ -241,9 +240,14 @@ public String getSourceBucket() {
   }
 
 
-  public LongAdder getUsedBytes() {
+  public long getUsedBytes() {
     return usedBytes;
   }
+
+  public void incrUsedBytes(long bytes) {
+    this.usedBytes += bytes;
+  }
+
   public long getQuotaInBytes() {
     return quotaInBytes;
   }
@@ -324,7 +328,7 @@ public Builder toBuilder() {
         .setSourceBucket(sourceBucket)
         .setAcls(acls)
         .addAllMetadata(metadata)
-        .setUsedBytes(usedBytes.sum())
+        .setUsedBytes(usedBytes)
         .setQuotaInBytes(quotaInBytes)
         .setQuotaInCounts(quotaInCounts);
   }
@@ -489,7 +493,7 @@ public BucketInfo getProtobuf() {
         .setModificationTime(modificationTime)
         .setObjectID(objectID)
         .setUpdateID(updateID)
-        .setUsedBytes(usedBytes.sum())
+        .setUsedBytes(usedBytes)
         .addAllMetadata(KeyValueUtil.toProtobuf(metadata))
         .setQuotaInBytes(quotaInBytes)
         .setQuotaInCounts(quotaInCounts);
@@ -557,7 +561,7 @@ public String getObjectInfo() {
         ", isVersionEnabled='" + isVersionEnabled + "'" +
         ", storageType='" + storageType + "'" +
         ", creationTime='" + creationTime + "'" +
-        ", usedBytes='" + usedBytes.sum() + "'" +
+        ", usedBytes='" + usedBytes + "'" +
         ", quotaInBytes='" + quotaInBytes + "'" +
         ", quotaInCounts='" + quotaInCounts + '\'' +
         sourceInfo +
@@ -582,7 +586,7 @@ public boolean equals(Object o) {
         storageType == that.storageType &&
         objectID == that.objectID &&
         updateID == that.updateID &&
-        usedBytes.sum() == that.usedBytes.sum() &&
+        usedBytes == that.usedBytes &&
         Objects.equals(sourceVolume, that.sourceVolume) &&
         Objects.equals(sourceBucket, that.sourceBucket) &&
         Objects.equals(metadata, that.metadata) &&
@@ -609,7 +613,7 @@ public String toString() {
         ", objectID=" + objectID +
         ", updateID=" + updateID +
         ", metadata=" + metadata +
-        ", usedBytes=" + usedBytes.sum() +
+        ", usedBytes=" + usedBytes +
         ", quotaInBytes=" + quotaInBytes +
         ", quotaInCounts=" + quotaInCounts +
         '}';

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java

@@ -23,7 +23,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.atomic.LongAdder;
 
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -47,7 +46,6 @@ public final class OmVolumeArgs extends WithObjectID implements Auditable {
   private long quotaInBytes;
   private long quotaInCounts;
   private final OmOzoneAclMap aclMap;
-  private final LongAdder usedBytes = new LongAdder();
 
   /**
    * Private constructor, constructed via builder.
@@ -57,26 +55,24 @@ public final class OmVolumeArgs extends WithObjectID implements Auditable {
    * @param quotaInBytes - Volume Quota in bytes.
    * @param quotaInCounts - Volume Quota in counts.
    * @param metadata - metadata map for custom key/value data.
-   * @param usedBytes - Volume Quota Usage in bytes.
    * @param aclMap - User to access rights map.
    * @param creationTime - Volume creation time.
-   * @param  objectID - ID of this object.
+   * @param objectID - ID of this object.
    * @param updateID - A sequence number that denotes the last update on this
    * object. This is a monotonically increasing number.
    */
   @SuppressWarnings({"checkstyle:ParameterNumber", "This is invoked from a " +
       "builder."})
   private OmVolumeArgs(String adminName, String ownerName, String volume,
       long quotaInBytes, long quotaInCounts, Map<String, String> metadata,
-      long usedBytes, OmOzoneAclMap aclMap, long creationTime,
-      long modificationTime, long objectID, long updateID) {
+      OmOzoneAclMap aclMap, long creationTime, long modificationTime,
+      long objectID, long updateID) {
     this.adminName = adminName;
     this.ownerName = ownerName;
     this.volume = volume;
     this.quotaInBytes = quotaInBytes;
     this.quotaInCounts = quotaInCounts;
     this.metadata = metadata;
-    this.usedBytes.add(usedBytes);
     this.aclMap = aclMap;
     this.creationTime = creationTime;
     this.modificationTime = modificationTime;
@@ -177,10 +173,6 @@ public OmOzoneAclMap getAclMap() {
     return aclMap;
   }
 
-  public LongAdder getUsedBytes() {
-    return usedBytes;
-  }
-
   /**
    * Returns new builder class that builds a OmVolumeArgs.
    *
@@ -204,8 +196,6 @@ public Map<String, String> toAuditMap() {
         String.valueOf(this.quotaInCounts));
     auditMap.put(OzoneConsts.OBJECT_ID, String.valueOf(this.getObjectID()));
     auditMap.put(OzoneConsts.UPDATE_ID, String.valueOf(this.getUpdateID()));
-    auditMap.put(OzoneConsts.USED_BYTES,
-        String.valueOf(this.usedBytes));
     return auditMap;
   }
 
@@ -241,7 +231,6 @@ public static class Builder {
     private OmOzoneAclMap aclMap;
     private long objectID;
     private long updateID;
-    private long usedBytes;
 
     /**
      * Sets the Object ID for this Object.
@@ -319,11 +308,6 @@ public Builder addAllMetadata(Map<String, String> additionalMetaData) {
       return this;
     }
 
-    public Builder setUsedBytes(long quotaUsage) {
-      this.usedBytes = quotaUsage;
-      return this;
-    }
-
     public Builder addOzoneAcls(OzoneAclInfo acl) throws IOException {
       aclMap.addAcl(acl);
       return this;
@@ -338,8 +322,8 @@ public OmVolumeArgs build() {
       Preconditions.checkNotNull(ownerName);
       Preconditions.checkNotNull(volume);
       return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
-          quotaInCounts, metadata, usedBytes, aclMap, creationTime,
-          modificationTime, objectID, updateID);
+          quotaInCounts, metadata, aclMap, creationTime, modificationTime,
+          objectID, updateID);
     }
 
   }
@@ -359,7 +343,6 @@ public VolumeInfo getProtobuf() {
         .setModificationTime(modificationTime)
         .setObjectID(objectID)
         .setUpdateID(updateID)
-        .setUsedBytes(usedBytes.sum())
         .build();
   }
 
@@ -374,7 +357,6 @@ public static OmVolumeArgs getFromProtobuf(VolumeInfo volInfo)
         volInfo.getQuotaInBytes(),
         volInfo.getQuotaInCounts(),
         KeyValueUtil.getFromProtobuf(volInfo.getMetadataList()),
-        volInfo.getUsedBytes(),
         aclMap,
         volInfo.getCreationTime(),
         volInfo.getModificationTime(),
@@ -390,7 +372,6 @@ public String getObjectInfo() {
         ", owner='" + ownerName + '\'' +
         ", creationTime='" + creationTime + '\'' +
         ", quotaInBytes='" + quotaInBytes + '\'' +
-        ", usedBytes='" + usedBytes.sum() + '\'' +
         '}';
   }
 
@@ -406,7 +387,7 @@ public OmVolumeArgs copyObject() {
     OmOzoneAclMap cloneAclMap = aclMap.copyObject();
 
     return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
-        quotaInCounts, cloneMetadata, usedBytes.sum(), cloneAclMap,
-        creationTime, modificationTime, objectID, updateID);
+        quotaInCounts, cloneMetadata, cloneAclMap, creationTime,
+        modificationTime, objectID, updateID);
   }
 }