Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.hadoop.hdds.client;

import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

Expand Down Expand Up @@ -84,25 +85,29 @@ public void appendTo(StringBuilder sb) {
sb.append(" bcsId: ").append(blockCommitSequenceId);
}

@JsonIgnore
public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() {
return ContainerProtos.DatanodeBlockID.newBuilder().
setContainerID(containerBlockID.getContainerID())
.setLocalID(containerBlockID.getLocalID())
.setBlockCommitSequenceId(blockCommitSequenceId).build();
}

@JsonIgnore
public static BlockID getFromProtobuf(
ContainerProtos.DatanodeBlockID blockID) {
return new BlockID(blockID.getContainerID(),
blockID.getLocalID(), blockID.getBlockCommitSequenceId());
}

@JsonIgnore
public HddsProtos.BlockID getProtobuf() {
return HddsProtos.BlockID.newBuilder()
.setContainerBlockID(containerBlockID.getProtobuf())
.setBlockCommitSequenceId(blockCommitSequenceId).build();
}

@JsonIgnore
public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) {
return new BlockID(
ContainerBlockID.getFromProtobuf(blockID.getContainerBlockID()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.hadoop.hdds.client;

import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

import java.util.Objects;
Expand Down Expand Up @@ -52,11 +53,13 @@ public void appendTo(StringBuilder sb) {
.append(" locID: ").append(localID);
}

@JsonIgnore
public HddsProtos.ContainerBlockID getProtobuf() {
return HddsProtos.ContainerBlockID.newBuilder().
setContainerID(containerID).setLocalID(localID).build();
}

@JsonIgnore
public static ContainerBlockID getFromProtobuf(
HddsProtos.ContainerBlockID containerBlockID) {
return new ContainerBlockID(containerBlockID.getContainerID(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,10 @@ public void testNamespaceSummaryAPI() throws Exception {
NamespaceSummaryResponse entity =
(NamespaceSummaryResponse) basicInfo.getEntity();
Assert.assertSame(entity.getEntityType(), EntityType.DIRECTORY);
Assert.assertEquals(1, entity.getNumTotalKey());
Assert.assertEquals(0, entity.getNumTotalDir());
Assert.assertEquals(1, entity.getCountStats().getNumTotalKey());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why aren't we asserting getNumVolume() and getNumBucket() here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @hemantk-12 for review. Changes done. Pls re-review. This was existing case, I added those missing as you mentioned.

Assert.assertEquals(0, entity.getCountStats().getNumTotalDir());
Assert.assertEquals(-1, entity.getCountStats().getNumVolume());
Assert.assertEquals(-1, entity.getCountStats().getNumBucket());
for (int i = 0; i < 10; i++) {
Assert.assertNotNull(impl.getOMMetadataManagerInstance()
.getVolumeTable().get("/vol" + i));
Expand All @@ -149,10 +151,10 @@ public void testNamespaceSummaryAPI() throws Exception {
(NamespaceSummaryResponse) rootBasicRes.getEntity();
Assert.assertSame(EntityType.ROOT, rootBasicEntity.getEntityType());
// one additional dummy volume at creation
Assert.assertEquals(13, rootBasicEntity.getNumVolume());
Assert.assertEquals(12, rootBasicEntity.getNumBucket());
Assert.assertEquals(12, rootBasicEntity.getNumTotalDir());
Assert.assertEquals(12, rootBasicEntity.getNumTotalKey());
Assert.assertEquals(13, rootBasicEntity.getCountStats().getNumVolume());
Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumBucket());
Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumTotalDir());
Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumTotalKey());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,10 @@ public Response getBasicInfo(
NamespaceSummaryResponse namespaceSummaryResponse;
if (!isInitializationComplete()) {
namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.UNKNOWN);
namespaceSummaryResponse.setStatus(ResponseStatus.INITIALIZING);
NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.UNKNOWN)
.setStatus(ResponseStatus.INITIALIZING)
.build();
return Response.ok(namespaceSummaryResponse).build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo;
import org.apache.hadoop.ozone.recon.api.types.CountStats;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse;
import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;

Expand All @@ -49,16 +52,37 @@ public BucketEntityHandler(
@Override
public NamespaceSummaryResponse getSummaryResponse()
throws IOException {
NamespaceSummaryResponse namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.BUCKET);

String[] names = getNames();
assert (names.length == 2);
long bucketObjectId = getBucketHandler().getBucketObjectId(names);
namespaceSummaryResponse
.setNumTotalDir(getTotalDirCount(bucketObjectId));
namespaceSummaryResponse.setNumTotalKey(getTotalKeyCount(bucketObjectId));

return namespaceSummaryResponse;
CountStats countStats = new CountStats(
-1, -1,
getTotalDirCount(bucketObjectId), getTotalKeyCount(bucketObjectId));
return NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.BUCKET)
.setCountStats(countStats)
.setObjectDBInfo(getBucketObjDbInfo(names))
.setStatus(ResponseStatus.OK)
.build();
}

private BucketObjectDBInfo getBucketObjDbInfo(String[] names)
throws IOException {
String volName = names[0];
String bucketName = names[1];
String bucketKey = getOmMetadataManager().
getBucketKey(volName, bucketName);
if (null == bucketKey) {
return new BucketObjectDBInfo();
}
OmBucketInfo omBucketInfo = getOmMetadataManager()
.getBucketTable().getSkipCache(bucketKey);
if (null == omBucketInfo) {
return new BucketObjectDBInfo();
}
return new BucketObjectDBInfo(omBucketInfo);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
Expand Down Expand Up @@ -97,6 +98,9 @@ public abstract long getDirObjectId(String[] names, int cutoff)
public abstract OmKeyInfo getKeyInfo(String[] names)
throws IOException;

public abstract OmDirectoryInfo getDirInfo(String[] names)
throws IOException;

/**
* Fixing the existing path and appending the next level entity to it.
* @param path
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@
package org.apache.hadoop.ozone.recon.api.handlers;

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.recon.api.types.CountStats;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo;
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
Expand Down Expand Up @@ -55,13 +58,24 @@ public NamespaceSummaryResponse getSummaryResponse()
throws IOException {
// path should exist so we don't need any extra verification/null check
long dirObjectId = getBucketHandler().getDirObjectId(getNames());
NamespaceSummaryResponse namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.DIRECTORY);
namespaceSummaryResponse
.setNumTotalDir(getTotalDirCount(dirObjectId));
namespaceSummaryResponse.setNumTotalKey(getTotalKeyCount(dirObjectId));
CountStats countStats = new CountStats(
-1, -1,
getTotalDirCount(dirObjectId), getTotalKeyCount(dirObjectId));
return NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.DIRECTORY)
.setCountStats(countStats)
.setObjectDBInfo(getDirectoryObjDbInfo(getNames()))
.setStatus(ResponseStatus.OK)
.build();
}

return namespaceSummaryResponse;
private ObjectDBInfo getDirectoryObjDbInfo(String[] names)
throws IOException {
OmDirectoryInfo omDirectoryInfo = getBucketHandler().getDirInfo(names);
if (null == omDirectoryInfo) {
return new ObjectDBInfo();
}
return new ObjectDBInfo(omDirectoryInfo);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -276,4 +276,18 @@ public OmKeyInfo getKeyInfo(String[] names) throws IOException {
parentObjectId, fileName);
return getOmMetadataManager().getFileTable().getSkipCache(ozoneKey);
}

@Override
public OmDirectoryInfo getDirInfo(String[] names) throws IOException {
long parentObjectId = getDirObjectId(names, names.length - 1);
String dirKey = getOmMetadataManager().getOzonePathKey(
getVolumeObjectId(names),
getBucketObjectId(names),
parentObjectId,
names[names.length - 1]);
OmDirectoryInfo dirInfo = getOmMetadataManager()
.getDirectoryTable().getSkipCache(dirKey);
return dirInfo;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.recon.api.types.CountStats;
import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo;
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse;
Expand All @@ -46,10 +49,24 @@ public KeyEntityHandler(
@Override
public NamespaceSummaryResponse getSummaryResponse()
throws IOException {
NamespaceSummaryResponse namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.KEY);
CountStats countStats = new CountStats(
-1, -1,
-1, 0);
return NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.KEY)
.setCountStats(countStats)
.setObjectDBInfo(getKeyDbObjectInfo(getNames()))
.setStatus(ResponseStatus.OK)
.build();
}

return namespaceSummaryResponse;
private ObjectDBInfo getKeyDbObjectInfo(String[] names)
throws IOException {
OmKeyInfo omKeyInfo = getBucketHandler().getKeyInfo(names);
if (null == omKeyInfo) {
return new KeyObjectDBInfo();
}
return new KeyObjectDBInfo(omKeyInfo);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
Expand Down Expand Up @@ -322,4 +323,11 @@ public Table<String, OmKeyInfo> getKeyTable() {
getOmMetadataManager().getKeyTable(getBucketLayout());
return keyTable;
}

@Override
public OmDirectoryInfo getDirInfo(String[] names) throws IOException {
return OmDirectoryInfo.newBuilder()
.setName(names[2])

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If names has less than 3 elements, iirc it throws ArrayIndexOutOfBoundsException which is a RuntimeException and won't be caught by IOException.

Add a Preconditions.checkArgument check here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@smengcl - this check is not needed here as above code flow hits only when the path will be directory else not.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you explain more, what guarantees that we can't call this getDirInfo method only when the path is a directory? this is inside BucketHandler and a legacy bucket doesn't necessarily have a directory inside.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@devmadhuu I get that the only caller (at the moment) calling this method won't cause issues. I'm fine with either adding or not adding the check but I would prefer the former.

Same goes for the method in FSOBucketHandler.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dombizita - getDirInfo method has caller only through DirectoryEntityHandler and call stack for DirectoryEntityHandler will be determined after validating the path , if path is directory, then only DirectoryEntityHandler will be returned by EntityHandler.getEntityHandler , so there is no way the current code flow will hit getDirInfo method if path is not directory. First two indexes will be taken by Volume and Bucket in path and 3rd index will be surely a directory. Please check the caller hierarchy of this method. We can have zoom call to explain by sharing my screen.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I get that currently this is only called when it is guaranteed that the path is a directory. but if someone later adds something and doesn't know how this method works the scenario can happen which @smengcl wrote above. all the classes that are extending the EntityHandler abstract class can do a OmDirectoryInfo omDirectoryInfo = getBucketHandler().getDirInfo(names), even if the names length is less than 3. I am also fine with either way, just wanted to clarify what I meant.

.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,19 @@
package org.apache.hadoop.ozone.recon.api.handlers;

import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.recon.ReconConstants;
import org.apache.hadoop.ozone.recon.api.types.CountStats;
import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse;
import org.apache.hadoop.ozone.recon.api.types.EntityType;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo;
import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse;
import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse;
import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;

Expand All @@ -48,24 +53,35 @@ public RootEntityHandler(
@Override
public NamespaceSummaryResponse getSummaryResponse()
throws IOException {
NamespaceSummaryResponse namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.ROOT);

List<OmVolumeArgs> volumes = listVolumes();
namespaceSummaryResponse.setNumVolume(volumes.size());
List<OmBucketInfo> allBuckets = listBucketsUnderVolume(null);
namespaceSummaryResponse.setNumBucket(allBuckets.size());
int totalNumDir = 0;
long totalNumKey = 0L;
for (OmBucketInfo bucket : allBuckets) {
long bucketObjectId = bucket.getObjectID();
totalNumDir += getTotalDirCount(bucketObjectId);
totalNumKey += getTotalKeyCount(bucketObjectId);
}
CountStats countStats = new CountStats(
volumes.size(), allBuckets.size(), totalNumDir, totalNumKey);

return NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.ROOT)
.setCountStats(countStats)
.setObjectDBInfo(getPrefixObjDbInfo())
.setStatus(ResponseStatus.OK)
.build();
}

namespaceSummaryResponse.setNumTotalDir(totalNumDir);
namespaceSummaryResponse.setNumTotalKey(totalNumKey);

return namespaceSummaryResponse;
private ObjectDBInfo getPrefixObjDbInfo()
throws IOException {
OmPrefixInfo omPrefixInfo = getOmMetadataManager().getPrefixTable()
.getSkipCache(OzoneConsts.OM_KEY_PREFIX);
if (null == omPrefixInfo) {
return new ObjectDBInfo();
}
return new ObjectDBInfo(omPrefixInfo);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,9 @@ public UnknownEntityHandler(
@Override
public NamespaceSummaryResponse getSummaryResponse()
throws IOException {
NamespaceSummaryResponse namespaceSummaryResponse =
new NamespaceSummaryResponse(EntityType.UNKNOWN);
namespaceSummaryResponse.setStatus(ResponseStatus.PATH_NOT_FOUND);

return namespaceSummaryResponse;
return NamespaceSummaryResponse.newBuilder()
.setEntityType(EntityType.UNKNOWN)
.setStatus(ResponseStatus.PATH_NOT_FOUND).build();
}

@Override
Expand Down
Loading