HDDS-7751. Volume quota must have all bucket quota set

hadoop-ozone/dist/src/main/smoketest/createbucketenv.robot

@@ -30,7 +30,7 @@ Create volume
     ${result} =     Execute             ozone sh volume create /${volume} --user hadoop --space-quota 100TB --namespace-quota 100
                     Should not contain  ${result}       Failed
 Create bucket
-                    Execute             ozone sh bucket create /${volume}/${bucket}
+                    Execute             ozone sh bucket create /${volume}/${bucket} --space-quota 1TB
 
 *** Test Cases ***
 Test ozone shell

hadoop-ozone/dist/src/main/smoketest/createmrenv.robot

@@ -32,7 +32,7 @@ Create volume
     ${result} =     Execute             ozone sh volume create /${volume} --user hadoop --space-quota 100TB --namespace-quota 100
                     Should not contain  ${result}       Failed
 Create bucket
-                    Execute             ozone sh bucket create /${volume}/${bucket} --layout FILE_SYSTEM_OPTIMIZED
+                    Execute             ozone sh bucket create /${volume}/${bucket} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
 
 *** Test Cases ***
 Create test volume, bucket and key

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot

@@ -31,7 +31,7 @@ ${TESTFILE}         testfile
 *** Keywords ***
 Write keys
     Execute             ozone sh volume create o3://om/${VOLUME} --space-quota 100TB --namespace-quota 100
-    Execute             ozone sh bucket create o3://om/${VOLUME}/${BUCKET}
+    Execute             ozone sh bucket create o3://om/${VOLUME}/${BUCKET} --space-quota 1TB
     Execute             dd if=/dev/urandom of=${TESTFILE} bs=100000 count=15
     Execute             ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TESTFILE}
 

hadoop-ozone/dist/src/main/smoketest/gdpr/gdpr.robot

@@ -47,7 +47,7 @@ Test GDPR -g=false
 Test GDPR(disabled) without explicit options
     [arguments]     ${volume}
                     Execute             ozone sh volume create /${volume} --space-quota 100TB --namespace-quota 100
-                    Execute             ozone sh bucket create /${volume}/mybucket1
+                    Execute             ozone sh bucket create /${volume}/mybucket1 --space-quota 1TB
     ${result} =     Execute             ozone sh bucket info /${volume}/mybucket1 | jq -r '. | select(.name=="mybucket1") | .metadata | .gdprEnabled'
                     Should Be Equal     ${result}       null
                     Execute             ozone sh key put /${volume}/mybucket1/mykey /opt/hadoop/NOTICE.txt
@@ -58,7 +58,7 @@ Test GDPR(disabled) without explicit options
 
 Test GDPR with --enforcegdpr=true
     [arguments]     ${volume}
-                    Execute             ozone sh bucket create --enforcegdpr=true /${volume}/mybucket2
+                    Execute             ozone sh bucket create --enforcegdpr=true /${volume}/mybucket2 --space-quota 1TB
     ${result} =     Execute             ozone sh bucket info /${volume}/mybucket2 | jq -r '. | select(.name=="mybucket2") | .metadata | .gdprEnabled'
                     Should Be Equal     ${result}       true
                     Execute             ozone sh key put /${volume}/mybucket2/mykey /opt/hadoop/NOTICE.txt
@@ -69,7 +69,7 @@ Test GDPR with --enforcegdpr=true
 
 Test GDPR with -g=true
     [arguments]     ${volume}
-                    Execute             ozone sh bucket create -g=true /${volume}/mybucket3
+                    Execute             ozone sh bucket create -g=true /${volume}/mybucket3 --space-quota 1TB
     ${result} =     Execute             ozone sh bucket info /${volume}/mybucket3 | jq -r '. | select(.name=="mybucket3") | .metadata | .gdprEnabled'
                     Should Be Equal     ${result}       true
                     Execute             ozone sh key put /${volume}/mybucket3/mykey /opt/hadoop/NOTICE.txt
@@ -80,7 +80,7 @@ Test GDPR with -g=true
 
 Test GDPR with -g=false
     [arguments]     ${volume}
-                    Execute             ozone sh bucket create /${volume}/mybucket4
+                    Execute             ozone sh bucket create /${volume}/mybucket4 --space-quota 1TB
     ${result} =     Execute             ozone sh bucket info /${volume}/mybucket4 | jq -r '. | select(.name=="mybucket4") | .metadata | .gdprEnabled'
                     Should Be Equal     ${result}       null
                     Execute             ozone sh key put /${volume}/mybucket4/mykey /opt/hadoop/NOTICE.txt

hadoop-ozone/dist/src/main/smoketest/ozonefs/setup.robot

@@ -41,16 +41,16 @@ Create volumes for FS test
     Execute And Ignore Error    ozone sh volume create ${VOL2} --space-quota 100TB
 
 Create buckets for FS test
-    Execute                     ozone sh bucket create ${VOLUME}/${BUCKET} --layout FILE_SYSTEM_OPTIMIZED
-    Execute                     ozone sh bucket create ${VOLUME}/${BUCKET2} --layout FILE_SYSTEM_OPTIMIZED
-    Execute                     ozone sh bucket create ${VOL2}/${BUCKET_IN_VOL2} --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOLUME}/${BUCKET} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOLUME}/${BUCKET2} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOL2}/${BUCKET_IN_VOL2} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
 
 Create links for FS test
     Execute And Ignore Error    ozone sh volume create ${VOLUME}-src --space-quota 100TB
     Execute And Ignore Error    ozone sh volume create ${VOL2}-src --space-quota 100TB
-    Execute                     ozone sh bucket create ${VOLUME}-src/${BUCKET}-src --layout FILE_SYSTEM_OPTIMIZED
-    Execute                     ozone sh bucket create ${VOLUME}-src/${BUCKET2}-src --layout FILE_SYSTEM_OPTIMIZED
-    Execute                     ozone sh bucket create ${VOL2}-src/${BUCKET_IN_VOL2}-src --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOLUME}-src/${BUCKET}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOLUME}-src/${BUCKET2}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
+    Execute                     ozone sh bucket create ${VOL2}-src/${BUCKET_IN_VOL2}-src --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED
     Execute                     ozone sh bucket link ${VOLUME}-src/${BUCKET}-src ${VOLUME}/${BUCKET}
     Execute                     ozone sh bucket link ${VOLUME}-src/${BUCKET2}-src ${VOLUME}/${BUCKET2}
     Execute                     ozone sh bucket link ${VOL2}-src/${BUCKET_IN_VOL2}-src ${VOL2}/${BUCKET_IN_VOL2}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java

@@ -1233,12 +1233,11 @@ public void testTempMount() throws IOException {
     // Construct VolumeArgs
     VolumeArgs volumeArgs = new VolumeArgs.Builder()
         .setAcls(Collections.singletonList(aclWorldAccess))
-        .setQuotaInNamespace(1000)
-        .setQuotaInBytes(Long.MAX_VALUE).build();
+        .setQuotaInNamespace(1000).build();
     // Sanity check
     Assert.assertNull(volumeArgs.getOwner());
     Assert.assertNull(volumeArgs.getAdmin());
-    Assert.assertEquals(Long.MAX_VALUE, volumeArgs.getQuotaInBytes());
+    Assert.assertEquals(-1, volumeArgs.getQuotaInBytes());
     Assert.assertEquals(1000, volumeArgs.getQuotaInNamespace());
     Assert.assertEquals(0, volumeArgs.getMetadata().size());
     Assert.assertEquals(1, volumeArgs.getAcls().size());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java

@@ -345,12 +345,7 @@ public void testSetAndClrQuota() throws Exception {
     OzoneVolume volume = null;
     store.createVolume(volumeName);
 
-    store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota(
-        "10GB", "10000"));
     store.getVolume(volumeName).createBucket(bucketName);
-    volume = store.getVolume(volumeName);
-    Assert.assertEquals(10 * GB, volume.getQuotaInBytes());
-    Assert.assertEquals(10000L, volume.getQuotaInNamespace());
     OzoneBucket bucket = store.getVolume(volumeName).getBucket(bucketName);
     Assert.assertEquals(OzoneConsts.QUOTA_RESET, bucket.getQuotaInBytes());
     Assert.assertEquals(OzoneConsts.QUOTA_RESET, bucket.getQuotaInNamespace());
@@ -369,6 +364,12 @@ public void testSetAndClrQuota() throws Exception {
         store.getVolume(volumeName).getBucket(bucketName2);
     Assert.assertEquals(1024L, ozoneBucket2.getQuotaInBytes());
 
+    store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota(
+        "10GB", "10000"));
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(10 * GB, volume.getQuotaInBytes());
+    Assert.assertEquals(10000L, volume.getQuotaInNamespace());
+
     LambdaTestUtils.intercept(IOException.class, "Can not clear bucket" +
         " spaceQuota because volume spaceQuota is not cleared.",
         () -> ozoneBucket.clearSpaceQuota());
@@ -407,8 +408,6 @@ public void testSetBucketQuotaIllegal() throws Exception {
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
     store.createVolume(volumeName);
-    store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota(
-        "10GB", "1000"));
     store.getVolume(volumeName).createBucket(bucketName);
 
     // test bucket set quota 0
@@ -954,14 +953,15 @@ public void testCheckUsedBytesQuota() throws IOException {
     volume = store.getVolume(volumeName);
     volume.createBucket(bucketName);
     OzoneBucket bucket = volume.getBucket(bucketName);
+    bucket.setQuota(OzoneQuota.parseQuota("1 B", "100"));
 
     // Test bucket quota.
-    store.getVolume(volumeName).setQuota(
-        OzoneQuota.parseQuota(Long.MAX_VALUE + " B", "100"));
     bucketName = UUID.randomUUID().toString();
     volume.createBucket(bucketName);
     bucket = volume.getBucket(bucketName);
     bucket.setQuota(OzoneQuota.parseQuota("1 B", "100"));
+    store.getVolume(volumeName).setQuota(
+        OzoneQuota.parseQuota(Long.MAX_VALUE + " B", "100"));
 
     // Test bucket quota: write key.
     // The remaining quota does not satisfy a block size, so the write fails.
@@ -1310,8 +1310,8 @@ public void testVolumeUsedNamespace() throws IOException {
     Assert.assertEquals(0L, volumeWithLinkedBucket.getUsedNamespace());
 
     // Reset volume quota, the original usedNamespace needs to remain the same
-    store.getVolume(volumeName).setQuota(OzoneQuota.parseQuota(
-        100 + " GB", "100"));
+    store.getVolume(volumeName).setQuota(OzoneQuota.parseNameSpaceQuota(
+        "100"));
     Assert.assertEquals(1L,
         store.getVolume(volumeName).getUsedNamespace());
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java

@@ -863,29 +863,6 @@ public void testShQuota() throws Exception {
         () -> execute(ozoneShell, volumeArgs2));
     out.reset();
 
-    // Test set volume spaceQuota or nameSpaceQuota to normal value.
-    String[] volumeArgs3 = new String[]{"volume", "setquota", "vol4",
-        "--space-quota", "1000B"};
-    execute(ozoneShell, volumeArgs3);
-    out.reset();
-    assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes());
-    assertEquals(-1,
-        objectStore.getVolume("vol4").getQuotaInNamespace());
-
-    String[] volumeArgs4 = new String[]{"volume", "setquota", "vol4",
-        "--namespace-quota", "100"};
-    execute(ozoneShell, volumeArgs4);
-    out.reset();
-    assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes());
-    assertEquals(100,
-        objectStore.getVolume("vol4").getQuotaInNamespace());
-
-    // Test set volume quota without quota flag
-    String[] volumeArgs5 = new String[]{"volume", "setquota", "vol4"};
-    executeWithError(ozoneShell, volumeArgs5,
-        "At least one of the quota set flag is required");
-    out.reset();
-
     // Test set bucket quota to 0.
     String[] bucketArgs1 = new String[]{"bucket", "setquota", "vol4/buck4",
         "--space-quota", "0GB"};
@@ -936,6 +913,29 @@ public void testShQuota() throws Exception {
         "At least one of the quota set flag is required");
     out.reset();
 
+    // Test set volume spaceQuota or nameSpaceQuota to normal value.
+    String[] volumeArgs3 = new String[]{"volume", "setquota", "vol4",
+        "--space-quota", "1000B"};
+    execute(ozoneShell, volumeArgs3);
+    out.reset();
+    assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes());
+    assertEquals(-1,
+        objectStore.getVolume("vol4").getQuotaInNamespace());
+
+    String[] volumeArgs4 = new String[]{"volume", "setquota", "vol4",
+        "--namespace-quota", "100"};
+    execute(ozoneShell, volumeArgs4);
+    out.reset();
+    assertEquals(1000, objectStore.getVolume("vol4").getQuotaInBytes());
+    assertEquals(100,
+        objectStore.getVolume("vol4").getQuotaInNamespace());
+
+    // Test set volume quota without quota flag
+    String[] volumeArgs5 = new String[]{"volume", "setquota", "vol4"};
+    executeWithError(ozoneShell, volumeArgs5,
+        "At least one of the quota set flag is required");
+    out.reset();
+
     objectStore.getVolume("vol").deleteBucket("buck");
     objectStore.deleteVolume("vol");
     objectStore.getVolume("vol1").deleteBucket("buck1");

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java

@@ -207,8 +207,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
       }
 
       //Check quotaInBytes to update
-      checkQuotaBytesValid(metadataManager, omVolumeArgs, omBucketInfo,
-          volumeKey);
+      if (!bucketInfo.hasSourceBucket()) {
+        checkQuotaBytesValid(metadataManager, omVolumeArgs, omBucketInfo,
+            volumeKey);
+      }
 
       // Add objectID and updateID
       omBucketInfo.setObjectID(
@@ -388,6 +390,15 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager,
     long quotaInBytes = omBucketInfo.getQuotaInBytes();
     long volumeQuotaInBytes = omVolumeArgs.getQuotaInBytes();
 
+    // When volume quota is set, then its mandatory to have bucket quota
+    if (volumeQuotaInBytes > 0) {
+      if (quotaInBytes <= 0) {
+        throw new OMException("Bucket space quota in this volume " +
+            "should be set as volume space quota is already set.",
+            OMException.ResultCodes.QUOTA_ERROR);
+      }
+    }
+
     long totalBucketQuota = 0;
     if (quotaInBytes > 0) {
       totalBucketQuota = quotaInBytes;

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java

@@ -302,6 +302,12 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager,
             OMException.ResultCodes.QUOTA_ERROR);
       }
     }
+
+    // avoid iteration of other bucket if quota set is less than previous set
+    if (quotaInBytes < dbBucketInfo.getQuotaInBytes()) {
+      return true;
+    }
+
     List<OmBucketInfo> bucketList = metadataManager.listBuckets(
         omVolumeArgs.getVolume(), null, null, Integer.MAX_VALUE);
     for (OmBucketInfo bucketInfo : bucketList) {
@@ -311,10 +317,6 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager,
       long nextQuotaInBytes = bucketInfo.getQuotaInBytes();
       if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) {
         totalBucketQuota += nextQuotaInBytes;
-      } else {
-        // consider used space for bucket where quota is not set
-        // This quota will be part of volume quota
-        totalBucketQuota += bucketInfo.getUsedBytes();
       }
     }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java

@@ -538,40 +538,6 @@ protected void checkBucketQuotaInBytes(
             + allocateSize) + " Bytes.",
             OMException.ResultCodes.QUOTA_EXCEEDED);
       }
-    } else {
-      checkVolumeQuotaInBytes(metadataManager, omBucketInfo.getVolumeName(),
-          allocateSize);
-    }
-  }
-
-  private void checkVolumeQuotaInBytes(
-      OMMetadataManager metadataManager, String volumeName,
-      long allocateSize) throws IOException {
-    String volumeKey = metadataManager.getVolumeKey(volumeName);
-    OmVolumeArgs omVolumeArgs = metadataManager.getVolumeTable()
-        .get(volumeKey);
-    if (omVolumeArgs.getQuotaInBytes() <= OzoneConsts.QUOTA_RESET) {
-      return;
-    }
-
-    long totalBucketQuota = 0L;
-    List<OmBucketInfo> bucketList = metadataManager.listBuckets(
-        volumeName, null, null, Integer.MAX_VALUE);
-    for (OmBucketInfo bucketInfo : bucketList) {
-      long nextQuotaInBytes = bucketInfo.getQuotaInBytes();
-      if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) {
-        totalBucketQuota += nextQuotaInBytes;
-      } else {
-        totalBucketQuota += bucketInfo.getUsedBytes();
-      }
-    }
-
-    if (omVolumeArgs.getQuotaInBytes() - totalBucketQuota < allocateSize) {
-      throw new OMException("The DiskSpace quota of volume:"
-          + volumeName + " exceeded: quotaInBytes: "
-          + omVolumeArgs.getQuotaInBytes() + " Bytes but diskspace consumed: "
-          + (totalBucketQuota + allocateSize) + " Bytes.",
-          OMException.ResultCodes.QUOTA_EXCEEDED);
     }
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java

@@ -194,15 +194,31 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager,
         || volumeQuotaInBytes == 0) {
       return false;
     }
+
+    // if volume quota is for reset, no need further check
+    if (volumeQuotaInBytes == OzoneConsts.QUOTA_RESET) {
+      return true;
+    }
 
+    boolean isBucketQuotaSet = true;
     List<OmBucketInfo> bucketList = metadataManager.listBuckets(
         volumeName, null, null, Integer.MAX_VALUE);
     for (OmBucketInfo bucketInfo : bucketList) {
       long nextQuotaInBytes = bucketInfo.getQuotaInBytes();
       if (nextQuotaInBytes > OzoneConsts.QUOTA_RESET) {
         totalBucketQuota += nextQuotaInBytes;
+      } else {
+        isBucketQuotaSet = false;
+        break;
       }
     }
+
+    if (!isBucketQuotaSet) {
+      throw new OMException("Can not set volume space quota on volume " +
+          "as some of buckets in this volume have no quota set.",
+          OMException.ResultCodes.QUOTA_ERROR);
+    }
+
     if (volumeQuotaInBytes < totalBucketQuota &&
         volumeQuotaInBytes != OzoneConsts.QUOTA_RESET) {
       throw new OMException("Total buckets quota in this volume " +

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java

@@ -186,6 +186,30 @@ public void testValidateAndUpdateCacheCrossSpaceQuota() throws Exception {
         OMException.ResultCodes.QUOTA_EXCEEDED.toString());
   }
 
+  @Test
+  public void testValidateAndUpdateCacheBucketWithNoQuotaWhenVolumeQuotaSet()
+      throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+
+    OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager, 1000L);
+
+    // create a bucket with no quota
+    OMRequest originalRequest =
+        OMRequestTestUtils.createBucketRequest(bucketName, volumeName, false,
+            StorageTypeProto.SSD);
+    OMBucketCreateRequest omBucketCreateRequest =
+        new OMBucketCreateRequest(originalRequest);
+    OMRequest modifiedRequest = omBucketCreateRequest.preExecute(ozoneManager);
+    OMBucketCreateRequest testRequest =
+        new OMBucketCreateRequest(modifiedRequest);
+    OMClientResponse resp = testRequest.validateAndUpdateCache(
+        ozoneManager, 1, ozoneManagerDoubleBufferHelper);
+
+    Assert.assertEquals(resp.getOMResponse().getStatus().toString(),
+        OMException.ResultCodes.QUOTA_ERROR.toString());
+  }
+
   private OMBucketCreateRequest doPreExecute(String volumeName,
       String bucketName) throws Exception {
     addCreateVolumeToTable(volumeName, omMetadataManager);