diff --git a/hadoop-hdds/docs/content/feature/Quota.md b/hadoop-hdds/docs/content/feature/Quota.md index 933bbb50aec3..b82c97897193 100644 --- a/hadoop-hdds/docs/content/feature/Quota.md +++ b/hadoop-hdds/docs/content/feature/Quota.md @@ -32,11 +32,18 @@ So far, we know that Ozone allows users to create volumes, buckets, and keys. A 1. Storage Space level quota Administrators should be able to define how much storage space a Volume or Bucket can use. The following Settings for Storage space quota are currently supported： + a. By default, the quota for volume and bucket is not enabled. -b. When volume quota is enabled, the total size of bucket quota cannot exceed volume. + +b. When volume quota is enabled, the total quota of the buckets, cannot exceed the volume quota. + c. Bucket quota can be set separately without enabling Volume quota. The size of bucket quota is unrestricted at this point. + d. Volume quota is not currently supported separately, and volume quota takes effect only if bucket quota is set. Because ozone only check the usedBytes of the bucket when we write the key. +e. If the cluster is upgraded from old version less than 1.1.0, use of quota on older volumes and buckets(We can confirm by looking at the info for the volume or bucket, and if the quota value is -2 the volume or bucket is old) is not recommended. Since the old key is not counted to the bucket's usedBytes, the quota setting is inaccurate at this point. + +f. If volume's quota is enabled then bucket's quota cannot be cleared. ## Client usage ### Storage Space level quota @@ -66,9 +73,10 @@ This behavior changes the quota for Bucket1 to 10GB Total bucket quota should not be greater than its Volume quota. If we have a 10MB Volume, The sum of the sizes of all buckets under this volume cannot exceed 10MB, otherwise the bucket set quota fails. -#### Clear the quota for Volume1. The Bucket cleanup command is similar. +#### Clear the quota for volume and bucket ```shell bin/ozone sh volume clrquota --space-quota /volume1 +bin/ozone sh bucket clrquota --space-quota /volume1/bucket1 ``` #### Check quota and usedBytes for volume and bucket diff --git a/hadoop-hdds/docs/content/feature/Quota.zh.md b/hadoop-hdds/docs/content/feature/Quota.zh.md index b3f0c3c3187e..3db9d9800803 100644 --- a/hadoop-hdds/docs/content/feature/Quota.zh.md +++ b/hadoop-hdds/docs/content/feature/Quota.zh.md @@ -30,10 +30,18 @@ menu: 1. Storage space级别配额 管理员应该能够定义一个Volume或Bucket可以使用多少存储空间。目前支持以下storage space quota的设置: + a. 默认情况下volume和bucket的quota不启用。 + b. 当volume quota启用时，bucket quota的总大小不能超过volume。 + c. 可以在不启用volume quota的情况下单独给bucket设置quota。此时bucket quota的大小是不受限制的。 + d. 目前不支持单独设置volume quota，只有在设置了bucket quota的情况下volume quota才会生效。因为ozone在写入key时只检查bucket的usedBytes。 + + e. 如果集群从小于1.1.0的旧版本升级而来，则不建议在旧volume和bucket(可以通过查看volume或者bucket的info确认，如果quota值是-2，那么这个volume或者bucket就是旧的)上使用配额。由于旧的key没有计算到bucket的usedBytes中，所以此时配额设置是不准确的。 + + f. 如果volume quota被启用，那么bucket quota将不能被清除。 ## 客户端用法 ### Storage space级别配额 @@ -62,9 +70,10 @@ bin/ozone sh bucket setquota --space-quota 10GB /volume1/bucket1 bucket的总配额 不应大于其Volume的配额。让我们看一个例子，如果我们有一个10MB的Volume，该volume下所有bucket的大小之和不能超过10MB，否则设置bucket quota将失败。 -#### 清除Volume1的配额, Bucket清除命令与此类似 +#### 清除volume和bucket的配额 ```shell bin/ozone sh volume clrquota --space-quota /volume1 +bin/ozone sh bucket clrquota --space-quota /volume1/bucket1 ``` #### 查看volume和bucket的quota值以及usedBytes ```shell diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index 661b9d2812c7..1c16c81ab921 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -65,9 +65,9 @@ source "${COMPOSE_DIR}/../testlib.sh" export OZONE_KEEP_RESULTS=true start_docker_env execute_robot_test scm -v PREFIX:pre freon/validate.robot -# TODO can be enabled with HDDS-4562: -# execute_robot_test scm -v PREFIX:post freon/generate.robot -# execute_robot_test scm -v PREFIX:post freon/validate.robot +# test write key to old bucket after upgrade +execute_robot_test scm -v PREFIX:post freon/generate.robot +execute_robot_test scm -v PREFIX:post freon/validate.robot stop_docker_env generate_report diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d25033709d5a..3be9f77ee8c8 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -363,7 +363,7 @@ message VolumeInfo { optional uint64 objectID = 8; optional uint64 updateID = 9; optional uint64 modificationTime = 10; - optional uint64 quotaInNamespace = 11; + optional int64 quotaInNamespace = 11 [default = -2]; optional uint64 usedNamespace = 12; } @@ -503,8 +503,8 @@ message BucketInfo { optional string sourceVolume = 12; optional string sourceBucket = 13; optional uint64 usedBytes = 14; - optional uint64 quotaInBytes = 15; - optional uint64 quotaInNamespace = 16; + optional int64 quotaInBytes = 15 [default = -2]; + optional int64 quotaInNamespace = 16 [default = -2]; } enum StorageTypeProto { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 7a9f6e03d6a5..1130f94e097f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -314,7 +314,7 @@ private BucketEncryptionInfoProto getBeinfo( */ private void checkQuotaInNamespace(OmVolumeArgs omVolumeArgs, long allocatedNamespace) throws IOException { - if (omVolumeArgs.getQuotaInNamespace() != OzoneConsts.QUOTA_RESET) { + if (omVolumeArgs.getQuotaInNamespace() > 0) { long usedNamespace = omVolumeArgs.getUsedNamespace(); long quotaInNamespace = omVolumeArgs.getQuotaInNamespace(); long toUseNamespaceInTotal = usedNamespace + allocatedNamespace; @@ -335,10 +335,10 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, long volumeQuotaInBytes = omVolumeArgs.getQuotaInBytes(); long totalBucketQuota = 0; - if (quotaInBytes == OzoneConsts.QUOTA_RESET || quotaInBytes == 0) { - return false; - } else if (quotaInBytes > OzoneConsts.QUOTA_RESET) { + if (quotaInBytes > 0) { totalBucketQuota = quotaInBytes; + } else { + return false; } List bucketList = metadataManager.listBuckets(