Skip to content

HDDS-7503. CLI admin namespace -quota should return whole file system disk capacity consumed by all replicas #3976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jojochuang merged 5 commits into from
Dec 14, 2022
+24 −20
Merged

HDDS-7503. CLI admin namespace -quota should return whole file system disk capacity consumed by all replicas #3976

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d6b242
CLI admin namespace -quota should return whole file system disk capac…
Nov 17, 2022
e19a817
Update java import; revert timeout's change in wait.robot
Nov 30, 2022
1c572bd
Merge branch 'master' into HDDS-7503
Dec 7, 2022
6b20053
Update testQuotaUsage unit test
Dec 7, 2022
281617a
volume and bucket level quota does not change
Dec 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions ...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.recon.api.handlers;

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
Expand Down Expand Up @@ -126,15 +125,17 @@ public DUResponse getDuResponse(
public QuotaUsageResponse getQuotaResponse()
throws IOException {
QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse();
String[] names = getNames();
String bucketKey = getOmMetadataManager().getBucketKey(names[0], names[1]);
OmBucketInfo bucketInfo = getOmMetadataManager()
.getBucketTable().getSkipCache(bucketKey);
long bucketObjectId = bucketInfo.getObjectID();
long quotaInBytes = bucketInfo.getQuotaInBytes();
long quotaUsedInBytes = getTotalSize(bucketObjectId);
quotaUsageResponse.setQuota(quotaInBytes);
quotaUsageResponse.setQuotaUsed(quotaUsedInBytes);
RootEntityHandler rootEntityHandler = new RootEntityHandler(
getReconNamespaceSummaryManager(),
getOmMetadataManager(),
getReconSCM(),
"/"
);
QuotaUsageResponse rootQuotaUsageResponse =
rootEntityHandler.getQuotaResponse();

quotaUsageResponse.setQuota(rootQuotaUsageResponse.getQuota());
quotaUsageResponse.setQuotaUsed(rootQuotaUsageResponse.getQuotaUsed());
return quotaUsageResponse;
}

Expand Down
24 changes: 4 additions & 20 deletions ...one/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -127,26 +128,9 @@ public DUResponse getDuResponse(
public QuotaUsageResponse getQuotaResponse()
throws IOException {
QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse();
List<OmVolumeArgs> volumes = listVolumes();
List<OmBucketInfo> buckets = listBucketsUnderVolume(null);
long quotaInBytes = 0L;
long quotaUsedInBytes = 0L;

for (OmVolumeArgs volume: volumes) {
final long quota = volume.getQuotaInBytes();
assert (quota >= -1L);
if (quota == -1L) {
// If one volume has unlimited quota, the "root" quota is unlimited.
quotaInBytes = -1L;
break;
}
quotaInBytes += quota;
}
for (OmBucketInfo bucket: buckets) {
long bucketObjectId = bucket.getObjectID();
quotaUsedInBytes += getTotalSize(bucketObjectId);
}

SCMNodeStat stats = getReconSCM().getScmNodeManager().getStats();
long quotaInBytes = stats.getCapacity().get();
long quotaUsedInBytes = getDuResponse(true, true).getSizeWithReplica();
quotaUsageResponse.setQuota(quotaInBytes);
quotaUsageResponse.setQuotaUsed(quotaUsedInBytes);
return quotaUsageResponse;
Expand Down
26 changes: 10 additions & 16 deletions ...e/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.recon.ReconConstants;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
Expand Down Expand Up @@ -117,21 +116,16 @@ public DUResponse getDuResponse(
public QuotaUsageResponse getQuotaResponse()
throws IOException {
QuotaUsageResponse quotaUsageResponse = new QuotaUsageResponse();
String[] names = getNames();
List<OmBucketInfo> buckets = listBucketsUnderVolume(names[0]);
String volKey = getOmMetadataManager().getVolumeKey(names[0]);
OmVolumeArgs volumeArgs =
getOmMetadataManager().getVolumeTable().getSkipCache(volKey);
long quotaInBytes = volumeArgs.getQuotaInBytes();
long quotaUsedInBytes = 0L;

// Get the total data size used by all buckets
for (OmBucketInfo bucketInfo: buckets) {
long bucketObjectId = bucketInfo.getObjectID();
quotaUsedInBytes += getTotalSize(bucketObjectId);
}
quotaUsageResponse.setQuota(quotaInBytes);
quotaUsageResponse.setQuotaUsed(quotaUsedInBytes);
RootEntityHandler rootEntityHandler = new RootEntityHandler(
getReconNamespaceSummaryManager(),
getOmMetadataManager(),
getReconSCM(),
"/"
);
QuotaUsageResponse rootQuotaUsageResponse =
rootEntityHandler.getQuotaResponse();
quotaUsageResponse.setQuota(rootQuotaUsageResponse.getQuota());
quotaUsageResponse.setQuotaUsed(rootQuotaUsageResponse.getQuotaUsed());
return quotaUsageResponse;
}

Expand Down
22 changes: 16 additions & 6 deletions ...e/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
Comment thread
jojochuang marked this conversation as resolved.
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
Expand All @@ -50,6 +51,7 @@
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconNodeManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
Expand Down Expand Up @@ -657,20 +659,20 @@ public void testQuotaUsage() throws Exception {
// volume level quota usage
Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH);
QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity();
Assert.assertEquals(VOL_QUOTA, quVolRes.getQuota());
Assert.assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quVolRes.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quVolRes.getQuotaUsed());

// bucket level quota usage
Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH);
QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity();
Assert.assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota());
Assert.assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quBucketRes.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quBucketRes.getQuotaUsed());

Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH);
QuotaUsageResponse quBucketRes2 =
(QuotaUsageResponse) bucketRes2.getEntity();
Assert.assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota());
Assert.assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quBucketRes2.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quBucketRes2.getQuotaUsed());

// other level not applicable
Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH);
Expand Down Expand Up @@ -1262,10 +1264,18 @@ private static ReconStorageContainerManagerFacade getMockReconSCM()
.thenReturn(containerReplicas6);

when(reconSCM.getContainerManager()).thenReturn(containerManager);
ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class);
when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat());
when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager);
return reconSCM;
}

private static BucketLayout getBucketLayout() {
return BucketLayout.FILE_SYSTEM_OPTIMIZED;
}

private static SCMNodeStat getMockSCMRootStat() {
return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE,
ROOT_QUOTA - ROOT_DATA_SIZE);
}
}
22 changes: 16 additions & 6 deletions ...econ/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
Comment thread
jojochuang marked this conversation as resolved.
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
Expand All @@ -51,6 +52,7 @@
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.scm.ReconNodeManager;
import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
Expand Down Expand Up @@ -659,20 +661,20 @@ public void testQuotaUsage() throws Exception {
// volume level quota usage
Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH);
QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity();
Assert.assertEquals(VOL_QUOTA, quVolRes.getQuota());
Assert.assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quVolRes.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quVolRes.getQuotaUsed());

// bucket level quota usage
Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH);
QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity();
Assert.assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota());
Assert.assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quBucketRes.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quBucketRes.getQuotaUsed());

Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH);
QuotaUsageResponse quBucketRes2 =
(QuotaUsageResponse) bucketRes2.getEntity();
Assert.assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota());
Assert.assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed());
Assert.assertEquals(ROOT_QUOTA, quBucketRes2.getQuota());
Assert.assertEquals(ROOT_DATA_SIZE, quBucketRes2.getQuotaUsed());

// other level not applicable
Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH);
Expand Down Expand Up @@ -1301,10 +1303,18 @@ private static ReconStorageContainerManagerFacade getMockReconSCM()
.thenReturn(containerReplicas6);

when(reconSCM.getContainerManager()).thenReturn(containerManager);
ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class);
when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat());
when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager);
return reconSCM;
}

private static BucketLayout getBucketLayout() {
return BucketLayout.LEGACY;
}

private static SCMNodeStat getMockSCMRootStat() {
return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE,
ROOT_QUOTA - ROOT_DATA_SIZE);
}
}