diff --git a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot index 8a79fd2fca47..1042f520b61c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot +++ b/hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot @@ -30,7 +30,7 @@ Create volume ${result} = Execute ozone sh volume create /${volume} --user hadoop --space-quota 100TB --namespace-quota 100 Should not contain ${result} Failed Create bucket - Execute ozone sh bucket create /${volume}/${bucket} + Execute ozone sh bucket create /${volume}/${bucket} --space-quota 1TB *** Test Cases *** Test ozone shell diff --git a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot index 79de78c8b224..d3049bd5ae8e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot +++ b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot @@ -32,7 +32,7 @@ Create volume ${result} = Execute ozone sh volume create /${volume} --user hadoop --space-quota 100TB --namespace-quota 100 Should not contain ${result} Failed Create bucket - Execute ozone sh bucket create /${volume}/${bucket} --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create /${volume}/${bucket} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED *** Test Cases *** Create test volume, bucket and key diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index f41635724134..c26e163a051b 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -31,7 +31,7 @@ ${TESTFILE} testfile *** Keywords *** Write keys Execute ozone sh volume create o3://om/${VOLUME} --space-quota 100TB --namespace-quota 100 - Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} + Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} --space-quota 1TB Execute dd if=/dev/urandom of=${TESTFILE} bs=100000 count=15 Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TESTFILE} diff --git a/hadoop-ozone/dist/src/main/smoketest/gdpr/gdpr.robot b/hadoop-ozone/dist/src/main/smoketest/gdpr/gdpr.robot index 4487fa998dba..1078d955405a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/gdpr/gdpr.robot +++ b/hadoop-ozone/dist/src/main/smoketest/gdpr/gdpr.robot @@ -47,7 +47,7 @@ Test GDPR -g=false Test GDPR(disabled) without explicit options [arguments] ${volume} Execute ozone sh volume create /${volume} --space-quota 100TB --namespace-quota 100 - Execute ozone sh bucket create /${volume}/mybucket1 + Execute ozone sh bucket create /${volume}/mybucket1 --space-quota 1TB ${result} = Execute ozone sh bucket info /${volume}/mybucket1 | jq -r '. | select(.name=="mybucket1") | .metadata | .gdprEnabled' Should Be Equal ${result} null Execute ozone sh key put /${volume}/mybucket1/mykey /opt/hadoop/NOTICE.txt @@ -58,7 +58,7 @@ Test GDPR(disabled) without explicit options Test GDPR with --enforcegdpr=true [arguments] ${volume} - Execute ozone sh bucket create --enforcegdpr=true /${volume}/mybucket2 + Execute ozone sh bucket create --enforcegdpr=true /${volume}/mybucket2 --space-quota 1TB ${result} = Execute ozone sh bucket info /${volume}/mybucket2 | jq -r '. | select(.name=="mybucket2") | .metadata | .gdprEnabled' Should Be Equal ${result} true Execute ozone sh key put /${volume}/mybucket2/mykey /opt/hadoop/NOTICE.txt @@ -69,7 +69,7 @@ Test GDPR with --enforcegdpr=true Test GDPR with -g=true [arguments] ${volume} - Execute ozone sh bucket create -g=true /${volume}/mybucket3 + Execute ozone sh bucket create -g=true /${volume}/mybucket3 --space-quota 1TB ${result} = Execute ozone sh bucket info /${volume}/mybucket3 | jq -r '. | select(.name=="mybucket3") | .metadata | .gdprEnabled' Should Be Equal ${result} true Execute ozone sh key put /${volume}/mybucket3/mykey /opt/hadoop/NOTICE.txt @@ -80,7 +80,7 @@ Test GDPR with -g=true Test GDPR with -g=false [arguments] ${volume} - Execute ozone sh bucket create /${volume}/mybucket4 + Execute ozone sh bucket create /${volume}/mybucket4 --space-quota 1TB ${result} = Execute ozone sh bucket info /${volume}/mybucket4 | jq -r '. | select(.name=="mybucket4") | .metadata | .gdprEnabled' Should Be Equal ${result} null Execute ozone sh key put /${volume}/mybucket4/mykey /opt/hadoop/NOTICE.txt diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot index eb94bd033d5b..e7aca825f35a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot @@ -41,16 +41,16 @@ Create volumes for FS test Execute And Ignore Error ozone sh volume create ${VOL2} --space-quota 100TB Create buckets for FS test - Execute ozone sh bucket create ${VOLUME}/${BUCKET} --layout FILE_SYSTEM_OPTIMIZED - Execute ozone sh bucket create ${VOLUME}/${BUCKET2} --layout FILE_SYSTEM_OPTIMIZED - Execute ozone sh bucket create ${VOL2}/${BUCKET_IN_VOL2} --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOLUME}/${BUCKET} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOLUME}/${BUCKET2} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOL2}/${BUCKET_IN_VOL2} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED Create links for FS test Execute And Ignore Error ozone sh volume create ${VOLUME}-src --space-quota 100TB Execute And Ignore Error ozone sh volume create ${VOL2}-src --space-quota 100TB - Execute ozone sh bucket create ${VOLUME}-src/${BUCKET}-src --layout FILE_SYSTEM_OPTIMIZED - Execute ozone sh bucket create ${VOLUME}-src/${BUCKET2}-src --layout FILE_SYSTEM_OPTIMIZED - Execute ozone sh bucket create ${VOL2}-src/${BUCKET_IN_VOL2}-src --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOLUME}-src/${BUCKET}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOLUME}-src/${BUCKET2}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create ${VOL2}-src/${BUCKET_IN_VOL2}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED Execute ozone sh bucket link ${VOLUME}-src/${BUCKET}-src ${VOLUME}/${BUCKET} Execute ozone sh bucket link ${VOLUME}-src/${BUCKET2}-src ${VOLUME}/${BUCKET2} Execute ozone sh bucket link ${VOL2}-src/${BUCKET_IN_VOL2}-src ${VOL2}/${BUCKET_IN_VOL2} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index cda335d6ba41..3dbec994fe9c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -1233,12 +1233,11 @@ public void testTempMount() throws IOException { // Construct VolumeArgs VolumeArgs volumeArgs = new VolumeArgs.Builder() .setAcls(Collections.singletonList(aclWorldAccess)) - .setQuotaInNamespace(1000) - .setQuotaInBytes(Long.MAX_VALUE).build(); + .setQuotaInNamespace(1000).build(); // Sanity check Assert.assertNull(volumeArgs.getOwner()); Assert.assertNull(volumeArgs.getAdmin()); - Assert.assertEquals(Long.MAX_VALUE, volumeArgs.getQuotaInBytes()); + Assert.assertEquals(-1, volumeArgs.getQuotaInBytes()); Assert.assertEquals(1000, volumeArgs.getQuotaInNamespace()); Assert.assertEquals(0, volumeArgs.getMetadata().size()); Assert.assertEquals(1, volumeArgs.getAcls().size()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 692ab2f77dea..e6bfca173df9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -345,12 +345,7 @@ public void testSetAndClrQuota() throws Exception { OzoneVolume volume = null; store.createVolume(volumeName); - store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota( - "10GB", "10000")); store.getVolume(volumeName).createBucket(bucketName); - volume = store.getVolume(volumeName); - Assert.assertEquals(10 * GB, volume.getQuotaInBytes()); - Assert.assertEquals(10000L, volume.getQuotaInNamespace()); OzoneBucket bucket = store.getVolume(volumeName).getBucket(bucketName); Assert.assertEquals(OzoneConsts.QUOTA_RESET, bucket.getQuotaInBytes()); Assert.assertEquals(OzoneConsts.QUOTA_RESET, bucket.getQuotaInNamespace()); @@ -369,6 +364,12 @@ public void testSetAndClrQuota() throws Exception { store.getVolume(volumeName).getBucket(bucketName2); Assert.assertEquals(1024L, ozoneBucket2.getQuotaInBytes()); + store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota( + "10GB", "10000")); + volume = store.getVolume(volumeName); + Assert.assertEquals(10 * GB, volume.getQuotaInBytes()); + Assert.assertEquals(10000L, volume.getQuotaInNamespace()); + LambdaTestUtils.intercept(IOException.class, "Can not clear bucket" + " spaceQuota because volume spaceQuota is not cleared.", () -> ozoneBucket.clearSpaceQuota()); @@ -407,8 +408,6 @@ public void testSetBucketQuotaIllegal() throws Exception { String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); store.createVolume(volumeName); - store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota( - "10GB", "1000")); store.getVolume(volumeName).createBucket(bucketName); // test bucket set quota 0 @@ -954,14 +953,15 @@ public void testCheckUsedBytesQuota() throws IOException { volume = store.getVolume(volumeName); volume.createBucket(bucketName); OzoneBucket bucket = volume.getBucket(bucketName); + bucket.setQuota(OzoneQuota.parseQuota("1 B", "100")); // Test bucket quota. - store.getVolume(volumeName).setQuota( - OzoneQuota.parseQuota(Long.MAX_VALUE + " B", "100")); bucketName = UUID.randomUUID().toString(); volume.createBucket(bucketName); bucket = volume.getBucket(bucketName); bucket.setQuota(OzoneQuota.parseQuota("1 B", "100")); + store.getVolume(volumeName).setQuota( + OzoneQuota.parseQuota(Long.MAX_VALUE + " B", "100")); // Test bucket quota: write key. // The remaining quota does not satisfy a block size, so the write fails. @@ -1310,8 +1310,8 @@ public void testVolumeUsedNamespace() throws IOException { Assert.assertEquals(0L, volumeWithLinkedBucket.getUsedNamespace()); // Reset volume quota, the original usedNamespace needs to remain the same - store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota( - 100 + " GB", "100")); + store.getVolume(volumeName).setQuota(OzoneQuota.parseNameSpaceQuota( + "100")); Assert.assertEquals(1L, store.getVolume(volumeName).getUsedNamespace()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 8c1653426bf0..a559266a880a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -863,29 +863,6 @@ public void testShQuota() throws Exception { () -> execute(ozoneShell, volumeArgs2)); out.reset(); - // Test set volume spaceQuota or nameSpaceQuota to normal value. - String[] volumeArgs3 = new String[]{"volume", "setquota", "vol4", - "--space-quota", "1000B"}; - execute(ozoneShell, volumeArgs3); - out.reset(); - assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes()); - assertEquals(-1, - objectStore.getVolume("vol4").getQuotaInNamespace()); - - String[] volumeArgs4 = new String[]{"volume", "setquota", "vol4", - "--namespace-quota", "100"}; - execute(ozoneShell, volumeArgs4); - out.reset(); - assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes()); - assertEquals(100, - objectStore.getVolume("vol4").getQuotaInNamespace()); - - // Test set volume quota without quota flag - String[] volumeArgs5 = new String[]{"volume", "setquota", "vol4"}; - executeWithError(ozoneShell, volumeArgs5, - "At least one of the quota set flag is required"); - out.reset(); - // Test set bucket quota to 0. String[] bucketArgs1 = new String[]{"bucket", "setquota", "vol4/buck4", "--space-quota", "0GB"}; @@ -936,6 +913,29 @@ public void testShQuota() throws Exception { "At least one of the quota set flag is required"); out.reset(); + // Test set volume spaceQuota or nameSpaceQuota to normal value. + String[] volumeArgs3 = new String[]{"volume", "setquota", "vol4", + "--space-quota", "1000B"}; + execute(ozoneShell, volumeArgs3); + out.reset(); + assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes()); + assertEquals(-1, + objectStore.getVolume("vol4").getQuotaInNamespace()); + + String[] volumeArgs4 = new String[]{"volume", "setquota", "vol4", + "--namespace-quota", "100"}; + execute(ozoneShell, volumeArgs4); + out.reset(); + assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes()); + assertEquals(100, + objectStore.getVolume("vol4").getQuotaInNamespace()); + + // Test set volume quota without quota flag + String[] volumeArgs5 = new String[]{"volume", "setquota", "vol4"}; + executeWithError(ozoneShell, volumeArgs5, + "At least one of the quota set flag is required"); + out.reset(); + objectStore.getVolume("vol").deleteBucket("buck"); objectStore.deleteVolume("vol"); objectStore.getVolume("vol1").deleteBucket("buck1"); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 2d913bf31938..2bb988069edc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -207,8 +207,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } //Check quotaInBytes to update - checkQuotaBytesValid(metadataManager, omVolumeArgs, omBucketInfo, - volumeKey); + if (!bucketInfo.hasSourceBucket()) { + checkQuotaBytesValid(metadataManager, omVolumeArgs, omBucketInfo, + volumeKey); + } // Add objectID and updateID omBucketInfo.setObjectID( @@ -388,6 +390,15 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, long quotaInBytes = omBucketInfo.getQuotaInBytes(); long volumeQuotaInBytes = omVolumeArgs.getQuotaInBytes(); + // When volume quota is set, then its mandatory to have bucket quota + if (volumeQuotaInBytes > 0) { + if (quotaInBytes <= 0) { + throw new OMException("Bucket space quota in this volume " + + "should be set as volume space quota is already set.", + OMException.ResultCodes.QUOTA_ERROR); + } + } + long totalBucketQuota = 0; if (quotaInBytes > 0) { totalBucketQuota = quotaInBytes; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 0b7decf52dbf..ff38668e5f53 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -302,6 +302,12 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, OMException.ResultCodes.QUOTA_ERROR); } } + + // avoid iteration of other bucket if quota set is less than previous set + if (quotaInBytes < dbBucketInfo.getQuotaInBytes()) { + return true; + } + List bucketList = metadataManager.listBuckets( omVolumeArgs.getVolume(), null, null, Integer.MAX_VALUE); for (OmBucketInfo bucketInfo : bucketList) { @@ -311,10 +317,6 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, long nextQuotaInBytes = bucketInfo.getQuotaInBytes(); if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) { totalBucketQuota += nextQuotaInBytes; - } else { - // consider used space for bucket where quota is not set - // This quota will be part of volume quota - totalBucketQuota += bucketInfo.getUsedBytes(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index dcf3999ac174..6f9b602bb748 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -538,40 +538,6 @@ protected void checkBucketQuotaInBytes( + allocateSize) + " Bytes.", OMException.ResultCodes.QUOTA_EXCEEDED); } - } else { - checkVolumeQuotaInBytes(metadataManager, omBucketInfo.getVolumeName(), - allocateSize); - } - } - - private void checkVolumeQuotaInBytes( - OMMetadataManager metadataManager, String volumeName, - long allocateSize) throws IOException { - String volumeKey = metadataManager.getVolumeKey(volumeName); - OmVolumeArgs omVolumeArgs = metadataManager.getVolumeTable() - .get(volumeKey); - if (omVolumeArgs.getQuotaInBytes() <= OzoneConsts.QUOTA_RESET) { - return; - } - - long totalBucketQuota = 0L; - List bucketList = metadataManager.listBuckets( - volumeName, null, null, Integer.MAX_VALUE); - for (OmBucketInfo bucketInfo : bucketList) { - long nextQuotaInBytes = bucketInfo.getQuotaInBytes(); - if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) { - totalBucketQuota += nextQuotaInBytes; - } else { - totalBucketQuota += bucketInfo.getUsedBytes(); - } - } - - if (omVolumeArgs.getQuotaInBytes() - totalBucketQuota < allocateSize) { - throw new OMException("The DiskSpace quota of volume:" - + volumeName + " exceeded: quotaInBytes: " - + omVolumeArgs.getQuotaInBytes() + " Bytes but diskspace consumed: " - + (totalBucketQuota + allocateSize) + " Bytes.", - OMException.ResultCodes.QUOTA_EXCEEDED); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java index d8848fc116dd..c1bf5a1c3305 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java @@ -194,15 +194,31 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, || volumeQuotaInBytes == 0) { return false; } + + // if volume quota is for reset, no need further check + if (volumeQuotaInBytes == OzoneConsts.QUOTA_RESET) { + return true; + } + boolean isBucketQuotaSet = true; List bucketList = metadataManager.listBuckets( volumeName, null, null, Integer.MAX_VALUE); for (OmBucketInfo bucketInfo : bucketList) { long nextQuotaInBytes = bucketInfo.getQuotaInBytes(); if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) { totalBucketQuota += nextQuotaInBytes; + } else { + isBucketQuotaSet = false; + break; } } + + if (!isBucketQuotaSet) { + throw new OMException("Can not set volume space quota on volume " + + "as some of buckets in this volume have no quota set.", + OMException.ResultCodes.QUOTA_ERROR); + } + if (volumeQuotaInBytes < totalBucketQuota && volumeQuotaInBytes != OzoneConsts.QUOTA_RESET) { throw new OMException("Total buckets quota in this volume " + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 7a7aafcfe401..dfabf0b577f2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -186,6 +186,30 @@ public void testValidateAndUpdateCacheCrossSpaceQuota() throws Exception { OMException.ResultCodes.QUOTA_EXCEEDED.toString()); } + @Test + public void testValidateAndUpdateCacheBucketWithNoQuotaWhenVolumeQuotaSet() + throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1000L); + + // create a bucket with no quota + OMRequest originalRequest = + OMRequestTestUtils.createBucketRequest(bucketName, volumeName, false, + StorageTypeProto.SSD); + OMBucketCreateRequest omBucketCreateRequest = + new OMBucketCreateRequest(originalRequest); + OMRequest modifiedRequest = omBucketCreateRequest.preExecute(ozoneManager); + OMBucketCreateRequest testRequest = + new OMBucketCreateRequest(modifiedRequest); + OMClientResponse resp = testRequest.validateAndUpdateCache( + ozoneManager, 1, ozoneManagerDoubleBufferHelper); + + Assert.assertEquals(resp.getOMResponse().getStatus().toString(), + OMException.ResultCodes.QUOTA_ERROR.toString()); + } + private OMBucketCreateRequest doPreExecute(String volumeName, String bucketName) throws Exception { addCreateVolumeToTable(volumeName, omMetadataManager); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java index a5d34aaa88c7..2d96aa9571e4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketSetPropertyRequest.java @@ -214,46 +214,6 @@ public void testValidateAndUpdateCacheWithQuota() throws Exception { "should not be greater than volume quota")); } - @Test - public void testValidateAndUpdateCacheWithOtherBucketQuotaUsedCrossLimit() - throws Exception { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String testBucketName = bucketName + "1"; - - OMRequestTestUtils.addVolumeToDB( - volumeName, omMetadataManager, 10 * GB); - OMRequestTestUtils.addBucketToDB(volumeName, bucketName, - omMetadataManager); - OMRequestTestUtils.addBucketToDB(volumeName, testBucketName, - omMetadataManager); - - // updated used size of one bucket - String bucketKey = omMetadataManager - .getBucketKey(volumeName, bucketName); - OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable() - .getCacheValue(new CacheKey<>(bucketKey)).getCacheValue(); - omBucketInfo.incrUsedBytes(8 * GB); - omMetadataManager.getBucketTable().put(bucketKey, omBucketInfo); - - // create request to set bucket size where other bucket have used quota - OMRequest omRequest = createSetBucketPropertyRequest(volumeName, - testBucketName, true, 3 * GB); - OMBucketSetPropertyRequest omBucketSetPropertyRequest = - new OMBucketSetPropertyRequest(omRequest); - - OMClientResponse omClientResponse = omBucketSetPropertyRequest - .validateAndUpdateCache(ozoneManager, 1, - ozoneManagerDoubleBufferHelper); - - Assert.assertFalse(omClientResponse.getOMResponse().getSuccess()); - Assert.assertEquals(omClientResponse.getOMResponse().getStatus(), - OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); - Assert.assertTrue(omClientResponse.getOMResponse().getMessage(). - contains("Total buckets quota in this volume " + - "should not be greater than volume quota")); - } - @Test public void rejectsSettingQuotaOnLink() throws Exception { // GIVEN diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java index b20247e8414e..3158e3597c00 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java @@ -194,29 +194,6 @@ public void testValidateAndUpdateCacheWithNamespaceQuotaExceeded() == OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); } - @Test - public void testValidateAndUpdateCacheWithOnlyVolumeQuotaExceeded() - throws Exception { - OMRequest omRequest = createFileRequest(volumeName, bucketName, keyName, - HddsProtos.ReplicationFactor.ONE, HddsProtos.ReplicationType.RATIS, - false, true); - - // add volume and create bucket with quota limit 1 - OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1L); - OMRequestTestUtils.addBucketToDB(volumeName, bucketName, - omMetadataManager, getBucketLayout()); - - OMFileCreateRequest omFileCreateRequest = getOMFileCreateRequest(omRequest); - OMRequest modifiedOmRequest = omFileCreateRequest.preExecute(ozoneManager); - - omFileCreateRequest = getOMFileCreateRequest(modifiedOmRequest); - OMClientResponse omFileCreateResponse = - omFileCreateRequest.validateAndUpdateCache(ozoneManager, 100L, - ozoneManagerDoubleBufferHelper); - Assert.assertTrue(omFileCreateResponse.getOMResponse().getStatus() - == OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); - } - @Test public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { OMRequest omRequest = createFileRequest(volumeName, bucketName, keyName, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java index 21f30068d68c..6f8bfd015b13 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java @@ -188,30 +188,6 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { } - @Test - public void testValidateAndUpdateCacheOnlyVolumeQuotaExceeded() - throws Exception { - // Add volume, bucket entries to DB. - // add volume and create bucket with quota limit 1 - OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1L); - OMRequestTestUtils.addBucketToDB(volumeName, bucketName, - omMetadataManager, getBucketLayout()); - addKeyToOpenKeyTable(volumeName, bucketName); - - OMRequest modifiedOmRequest = - doPreExecute(createAllocateBlockRequest()); - - OMAllocateBlockRequest omAllocateBlockRequest = - getOmAllocateBlockRequest(modifiedOmRequest); - - OMClientResponse omAllocateBlockResponse = - omAllocateBlockRequest.validateAndUpdateCache(ozoneManager, 100L, - ozoneManagerDoubleBufferHelper); - - Assert.assertTrue(omAllocateBlockResponse.getOMResponse().getStatus() - == OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); - } - /** * This method calls preExecute and verify the modified request. * @param originalOMRequest diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index d03689688979..10552e380d18 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -400,37 +400,6 @@ public void testValidateAndUpdateCacheWithBucketQuotaExceeds() Assert.assertEquals(0, bucketInfo.getUsedNamespace()); } - @Test - public void testValidateAndUpdateCacheWithOnlyVolumeQuotaExceeds() - throws Exception { - OMRequest modifiedOmRequest = - doPreExecute(createCommitKeyRequest()); - - OMKeyCommitRequest omKeyCommitRequest = - getOmKeyCommitRequest(modifiedOmRequest); - - KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); - - // Append new blocks - List allocatedLocationList = - keyArgs.getKeyLocationsList().stream() - .map(OmKeyLocationInfo::getFromProtobuf) - .collect(Collectors.toList()); - - OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1L); - OMRequestTestUtils.addBucketToDB(volumeName, bucketName, - omMetadataManager, omKeyCommitRequest.getBucketLayout()); - - addKeyToOpenKeyTable(allocatedLocationList); - - OMClientResponse omClientResponse = - omKeyCommitRequest.validateAndUpdateCache(ozoneManager, - 100L, ozoneManagerDoubleBufferHelper); - - Assert.assertEquals(OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED, - omClientResponse.getOMResponse().getStatus()); - } - @Test public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 17818e003671..fbd4bd9873ac 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -231,29 +231,6 @@ public void testValidateAndUpdateCacheWithNamespaceQuotaExceeded() == OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); } - @Test - public void testValidateAndUpdateCacheWithOnlyVolumeQuotaExceeded() - throws Exception { - when(ozoneManager.getOzoneLockProvider()).thenReturn( - new OzoneLockProvider(keyPathLockEnabled, enableFileSystemPaths)); - OMRequest modifiedOmRequest = - doPreExecute(createKeyRequest(false, 0, keyName)); - - OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest( - modifiedOmRequest, getBucketLayout()); - - // create bucket with quota limit 1 - OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1L); - OMRequestTestUtils.addBucketToDB(volumeName, bucketName, - omMetadataManager, omKeyCreateRequest.getBucketLayout()); - - OMClientResponse omKeyCreateResponse = - omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L, - ozoneManagerDoubleBufferHelper); - Assert.assertTrue(omKeyCreateResponse.getOMResponse().getStatus() - == OzoneManagerProtocolProtos.Status.QUOTA_EXCEEDED); - } - private void checkResponse(OMRequest modifiedOmRequest, OMClientResponse omKeyCreateResponse, long id, boolean override, BucketLayout bucketLayout) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequest.java index 8cb02cf77a7b..2e63633d421b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeSetQuotaRequest.java @@ -204,4 +204,27 @@ public void testValidateAndUpdateCacheWithQuota() throws Exception { contains("Total buckets quota in this volume " + "should not be greater than volume quota")); } + + @Test + public void testValidateAndUpdateCacheQuotaSetFailureWhenBucketQuotaNotSet() + throws Exception { + + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, + bucketName, omMetadataManager); + OMRequest originalRequest = + OMRequestTestUtils.createSetVolumePropertyRequest(volumeName, + 5 * GB, 100L); + + OMVolumeSetQuotaRequest omVolumeSetQuotaRequest = + new OMVolumeSetQuotaRequest(originalRequest); + + OMClientResponse omClientResponse = omVolumeSetQuotaRequest + .validateAndUpdateCache(ozoneManager, 1, + ozoneManagerDoubleBufferHelper); + Assert.assertEquals(omClientResponse.getOMResponse().getStatus(), + OzoneManagerProtocolProtos.Status.QUOTA_ERROR); + } }