From 8125de839e8231ef3a5d45d7e827a0bc0d0fdd80 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Mon, 5 Dec 2022 13:10:48 +0530 Subject: [PATCH 1/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../apache/hadoop/hdds/client/BlockID.java | 5 + .../hadoop/hdds/client/ContainerBlockID.java | 3 + .../recon/TestReconWithOzoneManagerFSO.java | 12 +- .../ozone/recon/api/NSSummaryEndpoint.java | 3 +- .../api/handlers/BucketEntityHandler.java | 55 +++++- .../recon/api/handlers/BucketHandler.java | 4 + .../api/handlers/DirectoryEntityHandler.java | 30 +++- .../recon/api/handlers/FSOBucketHandler.java | 14 ++ .../recon/api/handlers/KeyEntityHandler.java | 38 ++++- .../api/handlers/LegacyBucketHandler.java | 8 + .../recon/api/handlers/RootEntityHandler.java | 35 +++- .../api/handlers/UnknownEntityHandler.java | 8 +- .../api/handlers/VolumeEntityHandler.java | 40 ++++- .../recon/api/types/BucketObjectDBInfo.java | 159 ++++++++++++++++++ .../ozone/recon/api/types/CountStats.java | 66 ++++++++ .../recon/api/types/KeyObjectDBInfo.java | 132 +++++++++++++++ .../api/types/NamespaceSummaryResponse.java | 138 ++++++++++----- .../ozone/recon/api/types/ObjectDBInfo.java | 126 ++++++++++++++ .../recon/api/types/VolumeObjectDBInfo.java | 50 ++++++ .../api/TestNSSummaryEndpointWithFSO.java | 115 +++++++++++-- .../api/TestNSSummaryEndpointWithLegacy.java | 115 +++++++++++-- 21 files changed, 1053 insertions(+), 103 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java index 1a979f1eb2a1..3b39817bab51 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hdds.client; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -84,6 +85,7 @@ public void appendTo(StringBuilder sb) { sb.append(" bcsId: ").append(blockCommitSequenceId); } + @JsonIgnore public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() { return ContainerProtos.DatanodeBlockID.newBuilder(). setContainerID(containerBlockID.getContainerID()) @@ -91,18 +93,21 @@ public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() { .setBlockCommitSequenceId(blockCommitSequenceId).build(); } + @JsonIgnore public static BlockID getFromProtobuf( ContainerProtos.DatanodeBlockID blockID) { return new BlockID(blockID.getContainerID(), blockID.getLocalID(), blockID.getBlockCommitSequenceId()); } + @JsonIgnore public HddsProtos.BlockID getProtobuf() { return HddsProtos.BlockID.newBuilder() .setContainerBlockID(containerBlockID.getProtobuf()) .setBlockCommitSequenceId(blockCommitSequenceId).build(); } + @JsonIgnore public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) { return new BlockID( ContainerBlockID.getFromProtobuf(blockID.getContainerBlockID()), diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java index 94a1c87d4154..57980363850f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hdds.client; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import java.util.Objects; @@ -52,11 +53,13 @@ public void appendTo(StringBuilder sb) { .append(" locID: ").append(localID); } + @JsonIgnore public HddsProtos.ContainerBlockID getProtobuf() { return HddsProtos.ContainerBlockID.newBuilder(). setContainerID(containerID).setLocalID(localID).build(); } + @JsonIgnore public static ContainerBlockID getFromProtobuf( HddsProtos.ContainerBlockID containerBlockID) { return new ContainerBlockID(containerBlockID.getContainerID(), diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index a893ee58134e..d3681459ce2e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -128,8 +128,8 @@ public void testNamespaceSummaryAPI() throws Exception { NamespaceSummaryResponse entity = (NamespaceSummaryResponse) basicInfo.getEntity(); Assert.assertSame(entity.getEntityType(), EntityType.DIRECTORY); - Assert.assertEquals(1, entity.getNumTotalKey()); - Assert.assertEquals(0, entity.getNumTotalDir()); + Assert.assertEquals(1, entity.getCountStats().getNumTotalKey()); + Assert.assertEquals(0, entity.getCountStats().getNumTotalDir()); for (int i = 0; i < 10; i++) { Assert.assertNotNull(impl.getOMMetadataManagerInstance() .getVolumeTable().get("/vol" + i)); @@ -149,10 +149,10 @@ public void testNamespaceSummaryAPI() throws Exception { (NamespaceSummaryResponse) rootBasicRes.getEntity(); Assert.assertSame(EntityType.ROOT, rootBasicEntity.getEntityType()); // one additional dummy volume at creation - Assert.assertEquals(13, rootBasicEntity.getNumVolume()); - Assert.assertEquals(12, rootBasicEntity.getNumBucket()); - Assert.assertEquals(12, rootBasicEntity.getNumTotalDir()); - Assert.assertEquals(12, rootBasicEntity.getNumTotalKey()); + Assert.assertEquals(13, rootBasicEntity.getCountStats().getNumVolume()); + Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumBucket()); + Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumTotalDir()); + Assert.assertEquals(12, rootBasicEntity.getCountStats().getNumTotalKey()); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index aa2699f8bbdd..abad204313ba 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -80,7 +80,8 @@ public Response getBasicInfo( NamespaceSummaryResponse namespaceSummaryResponse; if (!isInitializationComplete()) { namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.UNKNOWN); + NamespaceSummaryResponse.newBuilder(). + setEntityType(EntityType.UNKNOWN).build(); namespaceSummaryResponse.setStatus(ResponseStatus.INITIALIZING); return Response.ok(namespaceSummaryResponse).build(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 36f1636af0f6..67e3caa15e07 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -19,12 +19,15 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.CountStats; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -49,18 +52,60 @@ public BucketEntityHandler( @Override public NamespaceSummaryResponse getSummaryResponse() throws IOException { - NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.BUCKET); + String[] names = getNames(); assert (names.length == 2); long bucketObjectId = getBucketHandler().getBucketObjectId(names); - namespaceSummaryResponse - .setNumTotalDir(getTotalDirCount(bucketObjectId)); - namespaceSummaryResponse.setNumTotalKey(getTotalKeyCount(bucketObjectId)); + + CountStats countStats = new CountStats( + -1, -1, + getTotalDirCount(bucketObjectId), getTotalKeyCount(bucketObjectId)); + NamespaceSummaryResponse namespaceSummaryResponse = + NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.BUCKET) + .setCountStats(countStats) + .setObjectDBInfo(getBucketObjDbInfo(names)) + .setStatus(ResponseStatus.OK) + .build(); return namespaceSummaryResponse; } + private BucketObjectDBInfo getBucketObjDbInfo(String[] names) + throws IOException { + String volName = names[0]; + String bucketName = names[1]; + String bucketKey = getOmMetadataManager(). + getBucketKey(volName, bucketName); + OmBucketInfo omBucketInfo = + getOmMetadataManager().getBucketTable().getSkipCache(bucketKey); + BucketObjectDBInfo bucketObjectDBInfo = new BucketObjectDBInfo(); + if (null != omBucketInfo) { + bucketObjectDBInfo.setMetadata(omBucketInfo.getMetadata()); + bucketObjectDBInfo.setVolumeName(omBucketInfo.getVolumeName()); + bucketObjectDBInfo.setName(omBucketInfo.getBucketName()); + bucketObjectDBInfo.setQuotaInBytes(omBucketInfo.getQuotaInBytes()); + bucketObjectDBInfo.setQuotaInNamespace( + omBucketInfo.getQuotaInNamespace()); + bucketObjectDBInfo.setUsedNamespace(omBucketInfo.getUsedNamespace()); + bucketObjectDBInfo.setCreationTime(omBucketInfo.getCreationTime()); + bucketObjectDBInfo.setModificationTime( + omBucketInfo.getModificationTime()); + bucketObjectDBInfo.setAcls(omBucketInfo.getAcls()); + bucketObjectDBInfo.setSourceBucket(omBucketInfo.getSourceBucket()); + bucketObjectDBInfo.setSourceVolume(omBucketInfo.getSourceVolume()); + bucketObjectDBInfo.setBekInfo(omBucketInfo.getEncryptionKeyInfo()); + bucketObjectDBInfo.setVersioningEnabled( + omBucketInfo.getIsVersionEnabled()); + bucketObjectDBInfo.setStorageType(omBucketInfo.getStorageType()); + bucketObjectDBInfo.setDefaultReplicationConfig( + omBucketInfo.getDefaultReplicationConfig()); + bucketObjectDBInfo.setBucketLayout(omBucketInfo.getBucketLayout()); + bucketObjectDBInfo.setOwner(omBucketInfo.getOwner()); + } + return bucketObjectDBInfo; + } + @Override public DUResponse getDuResponse( boolean listFile, boolean withReplica) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java index d12c3cdb6ad3..205e9879a6ab 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -97,6 +98,9 @@ public abstract long getDirObjectId(String[] names, int cutoff) public abstract OmKeyInfo getKeyInfo(String[] names) throws IOException; + public abstract OmDirectoryInfo getDirInfo(String[] names) + throws IOException; + /** * Fixing the existing path and appending the next level entity to it. * @param path diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index 0cfa6f1b47fc..e7027609df3d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -18,8 +18,11 @@ package org.apache.hadoop.ozone.recon.api.handlers; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.recon.api.types.CountStats; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.NSSummary; @@ -55,15 +58,34 @@ public NamespaceSummaryResponse getSummaryResponse() throws IOException { // path should exist so we don't need any extra verification/null check long dirObjectId = getBucketHandler().getDirObjectId(getNames()); + CountStats countStats = new CountStats( + -1, -1, + getTotalDirCount(dirObjectId), getTotalKeyCount(dirObjectId)); NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.DIRECTORY); - namespaceSummaryResponse - .setNumTotalDir(getTotalDirCount(dirObjectId)); - namespaceSummaryResponse.setNumTotalKey(getTotalKeyCount(dirObjectId)); + NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.DIRECTORY) + .setCountStats(countStats) + .setObjectDBInfo(getDirectoryObjDbInfo(getNames())) + .setStatus(ResponseStatus.OK) + .build(); return namespaceSummaryResponse; } + private ObjectDBInfo getDirectoryObjDbInfo(String[] names) + throws IOException { + OmDirectoryInfo dirInfo = getBucketHandler().getDirInfo(names); + ObjectDBInfo objectDBInfo = new ObjectDBInfo(); + if (null != dirInfo) { + objectDBInfo.setName(dirInfo.getName()); + objectDBInfo.setCreationTime(dirInfo.getCreationTime()); + objectDBInfo.setModificationTime(dirInfo.getModificationTime()); + objectDBInfo.setAcls(dirInfo.getAcls()); + objectDBInfo.setMetadata(dirInfo.getMetadata()); + } + return objectDBInfo; + } + @Override public DUResponse getDuResponse( boolean listFile, boolean withReplica) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java index ba188b70bdb6..cb6ff098ae88 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java @@ -276,4 +276,18 @@ public OmKeyInfo getKeyInfo(String[] names) throws IOException { parentObjectId, fileName); return getOmMetadataManager().getFileTable().getSkipCache(ozoneKey); } + + @Override + public OmDirectoryInfo getDirInfo(String[] names) throws IOException { + long parentObjectId = getDirObjectId(names, names.length - 1); + String dirKey = getOmMetadataManager().getOzonePathKey( + getVolumeObjectId(names), + getBucketObjectId(names), + parentObjectId, + names[names.length - 1]); + OmDirectoryInfo dirInfo = getOmMetadataManager() + .getDirectoryTable().getSkipCache(dirKey); + return dirInfo; + } + } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java index f70945462a39..0e5621531147 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java @@ -19,8 +19,11 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.api.types.CountStats; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; @@ -46,10 +49,39 @@ public KeyEntityHandler( @Override public NamespaceSummaryResponse getSummaryResponse() throws IOException { - NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.KEY); + CountStats countStats = new CountStats( + -1, -1, + -1, 0); + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.KEY) + .setCountStats(countStats) + .setObjectDBInfo(getKeyDbObjectInfo(getNames())) + .setStatus(ResponseStatus.OK) + .build(); - return namespaceSummaryResponse; + } + + private ObjectDBInfo getKeyDbObjectInfo(String[] names) + throws IOException { + OmKeyInfo keyInfo = getBucketHandler().getKeyInfo(names); + KeyObjectDBInfo keyObjectDBInfo = new KeyObjectDBInfo(); + if (null != keyInfo) { + keyObjectDBInfo.setVolumeName(keyInfo.getVolumeName()); + keyObjectDBInfo.setBucketName(keyInfo.getBucketName()); + keyObjectDBInfo.setName(keyInfo.getKeyName()); + keyObjectDBInfo.setKeyName(keyInfo.getKeyName()); + keyObjectDBInfo.setDataSize(keyInfo.getDataSize()); + keyObjectDBInfo.setKeyLocationVersions(keyInfo.getKeyLocationVersions()); + keyObjectDBInfo.setCreationTime(keyInfo.getCreationTime()); + keyObjectDBInfo.setModificationTime(keyInfo.getModificationTime()); + keyObjectDBInfo.setReplicationConfig(keyInfo.getReplicationConfig()); + keyObjectDBInfo.setEncInfo(keyInfo.getFileEncryptionInfo()); + keyObjectDBInfo.setFileName(keyInfo.getFileName()); + keyObjectDBInfo.setFile(keyInfo.isFile()); + keyObjectDBInfo.setAcls(keyInfo.getAcls()); + keyObjectDBInfo.setMetadata(keyInfo.getMetadata()); + } + return keyObjectDBInfo; } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java index e4d218fed9b3..9105eb147f41 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.recon.api.types.DUResponse; @@ -322,4 +323,11 @@ public Table getKeyTable() { getOmMetadataManager().getKeyTable(getBucketLayout()); return keyTable; } + + @Override + public OmDirectoryInfo getDirInfo(String[] names) throws IOException { + return OmDirectoryInfo.newBuilder() + .setName(names[2]) + .build(); + } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index 357f3a30c267..651b440c806b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -18,14 +18,19 @@ package org.apache.hadoop.ozone.recon.api.handlers; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.api.types.CountStats; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.ObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -48,12 +53,9 @@ public RootEntityHandler( @Override public NamespaceSummaryResponse getSummaryResponse() throws IOException { - NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.ROOT); + List volumes = listVolumes(); - namespaceSummaryResponse.setNumVolume(volumes.size()); List allBuckets = listBucketsUnderVolume(null); - namespaceSummaryResponse.setNumBucket(allBuckets.size()); int totalNumDir = 0; long totalNumKey = 0L; for (OmBucketInfo bucket : allBuckets) { @@ -61,13 +63,34 @@ public NamespaceSummaryResponse getSummaryResponse() totalNumDir += getTotalDirCount(bucketObjectId); totalNumKey += getTotalKeyCount(bucketObjectId); } + CountStats countStats = new CountStats( + volumes.size(), allBuckets.size(), totalNumDir, totalNumKey); - namespaceSummaryResponse.setNumTotalDir(totalNumDir); - namespaceSummaryResponse.setNumTotalKey(totalNumKey); + NamespaceSummaryResponse namespaceSummaryResponse = + NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.ROOT) + .setCountStats(countStats) + .setObjectDBInfo(getPrefixObjDbInfo()) + .setStatus(ResponseStatus.OK) + .build(); return namespaceSummaryResponse; } + private ObjectDBInfo getPrefixObjDbInfo() + throws IOException { + OmPrefixInfo omPrefixInfo = + getOmMetadataManager().getPrefixTable(). + getSkipCache(OzoneConsts.OM_KEY_PREFIX); + ObjectDBInfo objectDBInfo = new ObjectDBInfo(); + if (null != omPrefixInfo) { + objectDBInfo.setMetadata(omPrefixInfo.getMetadata()); + objectDBInfo.setName(omPrefixInfo.getName()); + objectDBInfo.setAcls(omPrefixInfo.getAcls()); + } + return objectDBInfo; + } + @Override public DUResponse getDuResponse( boolean listFile, boolean withReplica) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java index c1d5dfe4ad1f..b5a5bd9a0be9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/UnknownEntityHandler.java @@ -44,11 +44,9 @@ public UnknownEntityHandler( @Override public NamespaceSummaryResponse getSummaryResponse() throws IOException { - NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.UNKNOWN); - namespaceSummaryResponse.setStatus(ResponseStatus.PATH_NOT_FOUND); - - return namespaceSummaryResponse; + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.UNKNOWN) + .setStatus(ResponseStatus.PATH_NOT_FOUND).build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index e8c4c03bc06b..8481cd95af8e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -21,11 +21,14 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.api.types.CountStats; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -48,11 +51,9 @@ public VolumeEntityHandler( @Override public NamespaceSummaryResponse getSummaryResponse() throws IOException { - NamespaceSummaryResponse namespaceSummaryResponse = - new NamespaceSummaryResponse(EntityType.VOLUME); + String[] names = getNames(); List buckets = listBucketsUnderVolume(names[0]); - namespaceSummaryResponse.setNumBucket(buckets.size()); int totalDir = 0; long totalKey = 0L; @@ -63,12 +64,41 @@ public NamespaceSummaryResponse getSummaryResponse() totalKey += getTotalKeyCount(bucketObjectId); } - namespaceSummaryResponse.setNumTotalDir(totalDir); - namespaceSummaryResponse.setNumTotalKey(totalKey); + CountStats countStats = new CountStats( + -1, buckets.size(), totalDir, totalKey); + + NamespaceSummaryResponse namespaceSummaryResponse = + NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.VOLUME) + .setCountStats(countStats) + .setObjectDBInfo(getVolumeObjDbInfo(names)) + .setStatus(ResponseStatus.OK) + .build(); return namespaceSummaryResponse; } + private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) + throws IOException { + String dbVolumeKey = getOmMetadataManager().getVolumeKey(names[0]); + OmVolumeArgs volumeArgs = + getOmMetadataManager().getVolumeTable().getSkipCache(dbVolumeKey); + VolumeObjectDBInfo objectDBInfo = new VolumeObjectDBInfo(); + if (null != volumeArgs) { + objectDBInfo.setMetadata(volumeArgs.getMetadata()); + objectDBInfo.setName(volumeArgs.getVolume()); + objectDBInfo.setAdmin(volumeArgs.getAdminName()); + objectDBInfo.setOwner(volumeArgs.getOwnerName()); + objectDBInfo.setQuotaInBytes(volumeArgs.getQuotaInBytes()); + objectDBInfo.setQuotaInNamespace(volumeArgs.getQuotaInNamespace()); + objectDBInfo.setUsedNamespace(volumeArgs.getUsedNamespace()); + objectDBInfo.setCreationTime(volumeArgs.getCreationTime()); + objectDBInfo.setModificationTime(volumeArgs.getModificationTime()); + objectDBInfo.setAcls(volumeArgs.getAcls()); + } + return objectDBInfo; + } + @Override public DUResponse getDuResponse( boolean listFile, boolean withReplica) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java new file mode 100644 index 000000000000..807da37f71c3 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; + +/** + * Encapsulates the low level bucket info. + */ +public class BucketObjectDBInfo extends ObjectDBInfo { + /** volume name from om db. */ + @JsonProperty("volumeName") + private String volumeName; + + /** storage type for the object. */ + @JsonProperty("storageType") + private StorageType storageType; + + /** is versioning enabled for the object. */ + @JsonProperty("versioning") + private boolean isVersioningEnabled; + + /** used bytes for the object. */ + @JsonProperty("usedBytes") + private String usedBytes; + + /** + * Bucket encryption key info if encryption is enabled. + */ + @JsonProperty("encryptionInfo") + private BucketEncryptionKeyInfo bekInfo; + + /** + * Optional default replication for bucket. + */ + @JsonProperty("replicationConfigInfo") + private DefaultReplicationConfig defaultReplicationConfig; + + /** source volume. */ + @JsonProperty("sourceVolume") + private String sourceVolume; + + /** source bucket. */ + @JsonProperty("sourceBucket") + private String sourceBucket; + + /** + * Bucket Layout. + */ + @JsonProperty("bucketLayout") + private BucketLayout bucketLayout; + + /** + * bucket owner. + */ + @JsonProperty("owner") + private String owner; + + public String getVolumeName() { + return volumeName; + } + + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + + public StorageType getStorageType() { + return storageType; + } + + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + public String getUsedBytes() { + return usedBytes; + } + + public void setUsedBytes(String usedBytes) { + this.usedBytes = usedBytes; + } + + public BucketEncryptionKeyInfo getBekInfo() { + return bekInfo; + } + + public void setBekInfo(BucketEncryptionKeyInfo bekInfo) { + this.bekInfo = bekInfo; + } + + public DefaultReplicationConfig getDefaultReplicationConfig() { + return defaultReplicationConfig; + } + + public void setDefaultReplicationConfig( + DefaultReplicationConfig defaultReplicationConfig) { + this.defaultReplicationConfig = defaultReplicationConfig; + } + + public String getSourceVolume() { + return sourceVolume; + } + + public void setSourceVolume(String sourceVolume) { + this.sourceVolume = sourceVolume; + } + + public String getSourceBucket() { + return sourceBucket; + } + + public void setSourceBucket(String sourceBucket) { + this.sourceBucket = sourceBucket; + } + + public boolean isVersioningEnabled() { + return isVersioningEnabled; + } + + public void setVersioningEnabled(boolean versioningEnabled) { + isVersioningEnabled = versioningEnabled; + } + + public BucketLayout getBucketLayout() { + return bucketLayout; + } + + public void setBucketLayout(BucketLayout bucketLayout) { + this.bucketLayout = bucketLayout; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java new file mode 100644 index 000000000000..950ef5ff5886 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Count stats which tells the number of + * volumes/buckets/dir/files etc. + * */ +public class CountStats { + /** Total number of volumes under root, -1 for other types. */ + @JsonProperty("numVolume") + private int numVolume; + + /** Total number of buckets for root/volume, -1 for other types. */ + @JsonProperty("numBucket") + private int numBucket; + + /** Total number of directories for all types except key, -1 for key. */ + @JsonProperty("numDir") + private int numTotalDir; + + /** Total number of keys. */ + @JsonProperty("numKey") + private long numTotalKey; + + public CountStats(int numVolume, int numBucket, + int numTotalDir, long numTotalKey) { + this.numVolume = numVolume; + this.numBucket = numBucket; + this.numTotalDir = numTotalDir; + this.numTotalKey = numTotalKey; + } + + public int getNumVolume() { + return numVolume; + } + + public int getNumBucket() { + return numBucket; + } + + public int getNumTotalDir() { + return numTotalDir; + } + + public long getNumTotalKey() { + return numTotalKey; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java new file mode 100644 index 000000000000..bf6c88988f16 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; + +import java.util.List; + +/** + * Encapsulates the low level key info. + */ +public class KeyObjectDBInfo extends ObjectDBInfo { + /** volume name from om db. */ + @JsonProperty("volumeName") + private String volumeName; + @JsonProperty("bucketName") + private String bucketName; + @JsonProperty("keyName") + private String keyName; + @JsonProperty("dataSize") + private long dataSize; + @JsonProperty("keyLocationVersions") + private List keyLocationVersions; + @JsonProperty("replicationConfig") + private ReplicationConfig replicationConfig; + @JsonProperty("encInfo") + private FileEncryptionInfo encInfo; + + /** + * Support OFS use-case to identify if the key is a file or a directory. + */ + private boolean isFile; + + /** + * Represents leaf node name. This also will be used when the keyName is + * created on a FileSystemOptimized(FSO) bucket. For example, the user given + * keyName is "a/b/key1" then the fileName stores "key1". + */ + private String fileName; + + public String getVolumeName() { + return volumeName; + } + + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getKeyName() { + return keyName; + } + + public void setKeyName(String keyName) { + this.keyName = keyName; + } + + public long getDataSize() { + return dataSize; + } + + public void setDataSize(long dataSize) { + this.dataSize = dataSize; + } + + public List getKeyLocationVersions() { + return keyLocationVersions; + } + + public void setKeyLocationVersions( + List keyLocationVersions) { + this.keyLocationVersions = keyLocationVersions; + } + + public ReplicationConfig getReplicationConfig() { + return replicationConfig; + } + + public void setReplicationConfig(ReplicationConfig replicationConfig) { + this.replicationConfig = replicationConfig; + } + + public boolean isFile() { + return isFile; + } + + public void setFile(boolean file) { + isFile = file; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public FileEncryptionInfo getEncInfo() { + return encInfo; + } + + public void setEncInfo(FileEncryptionInfo encInfo) { + this.encInfo = encInfo; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java index 2a2d9f6de746..5ccfd9887314 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NamespaceSummaryResponse.java @@ -18,62 +18,67 @@ package org.apache.hadoop.ozone.recon.api.types; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; /** * HTTP Response wrapped for a 'summary' request. */ public class NamespaceSummaryResponse { + /** Path for metadata summary. */ + @JsonProperty("path") + private String path; + /** The namespace the request path is on. */ @JsonProperty("type") private EntityType entityType; - /** Total number of volumes under root, -1 for other types. */ - @JsonProperty("numVolume") - private int numVolume; - - /** Total number of buckets for root/volume, -1 for other types. */ - @JsonProperty("numBucket") - private int numBucket; + /** Count stats which tells the number of volumes/buckets/dir/files etc. */ + @JsonProperty("countStats") + private CountStats countStats; - /** Total number of directories for all types except key, -1 for key. */ - @JsonProperty("numDir") - private int numTotalDir; - - /** Total number of keys. */ - @JsonProperty("numKey") - private long numTotalKey; + @JsonProperty("objectInfo") + private ObjectDBInfo objectDBInfo; /** Path Status. */ @JsonProperty("status") private ResponseStatus status; - public NamespaceSummaryResponse(EntityType entityType) { - this.entityType = entityType; - this.numVolume = -1; - this.numBucket = -1; - this.numTotalDir = -1; - this.numTotalKey = 0; - this.status = ResponseStatus.OK; + /** + * Returns new builder class that builds a NamespaceSummaryResponse. + * + * @return Builder + */ + public static NamespaceSummaryResponse.Builder newBuilder() { + return new NamespaceSummaryResponse.Builder(); } - public EntityType getEntityType() { - return this.entityType; + public NamespaceSummaryResponse(Builder b) { + this.path = b.path; + this.entityType = b.entityType; + this.countStats = b.countStats; + this.objectDBInfo = b.objectDBInfo; + this.status = b.status; } - public int getNumVolume() { - return this.numVolume; + public String getPath() { + return path; } - public int getNumBucket() { - return this.numBucket; + public void setPath(String path) { + this.path = path; } - public int getNumTotalDir() { - return this.numTotalDir; + public CountStats getCountStats() { + return countStats; } - public long getNumTotalKey() { - return this.numTotalKey; + public void setCountStats(CountStats countStats) { + this.countStats = countStats; + } + + public EntityType getEntityType() { + return this.entityType; } public ResponseStatus getStatus() { @@ -84,23 +89,72 @@ public void setEntityType(EntityType entityType) { this.entityType = entityType; } - public void setNumVolume(int numVolume) { - this.numVolume = numVolume; + public void setStatus(ResponseStatus status) { + this.status = status; } - public void setNumBucket(int numBucket) { - this.numBucket = numBucket; + public ObjectDBInfo getObjectDBInfo() { + return objectDBInfo; } - public void setNumTotalDir(int numTotalDir) { - this.numTotalDir = numTotalDir; + public void setObjectDBInfo(ObjectDBInfo objectDBInfo) { + this.objectDBInfo = objectDBInfo; } - public void setNumTotalKey(long numTotalKey) { - this.numTotalKey = numTotalKey; + /** + * Builder for NamespaceSummaryResponse. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class Builder { + private String path; + private EntityType entityType; + private CountStats countStats; + private ObjectDBInfo objectDBInfo; + private ResponseStatus status; + + + public Builder() { + // Default values + this.path = StringUtils.EMPTY; + this.entityType = EntityType.ROOT; + } + + public NamespaceSummaryResponse.Builder setPath(String path) { + this.path = path; + return this; + } + + public NamespaceSummaryResponse.Builder setEntityType( + EntityType entityType) { + this.entityType = entityType; + return this; + } + + public NamespaceSummaryResponse.Builder setCountStats( + CountStats countStats) { + this.countStats = countStats; + return this; + } + + public NamespaceSummaryResponse.Builder setObjectDBInfo( + ObjectDBInfo objectDBInfo) { + this.objectDBInfo = objectDBInfo; + return this; + } + + public NamespaceSummaryResponse.Builder setStatus( + ResponseStatus status) { + this.status = status; + return this; + } + + public NamespaceSummaryResponse build() { + Preconditions.checkNotNull(this.path); + Preconditions.checkNotNull(this.entityType); + Preconditions.checkNotNull(this.status); + + return new NamespaceSummaryResponse(this); + } } - public void setStatus(ResponseStatus status) { - this.status = status; - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java new file mode 100644 index 000000000000..b0db053cebaa --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.ozone.OzoneAcl; + +import java.util.List; +import java.util.Map; + +/** + * Encapsulates the low level volume/bucket/dir info. + */ +public class ObjectDBInfo { + /** object metadata from om db. */ + @JsonProperty("metadata") + private Map metadata; + + /** name of the object. */ + @JsonProperty("name") + private String name; + + /** quota in bytes. */ + @JsonProperty("quotaInBytes") + private long quotaInBytes; + + /** quota in namespace. */ + @JsonProperty("quotaInNamespace") + private long quotaInNamespace; + + /** used namespace. */ + @JsonProperty("usedNamespace") + private long usedNamespace; + + /** creation time. */ + @JsonProperty("creationTime") + private long creationTime; + + /** modification time. */ + @JsonProperty("modificationTime") + private long modificationTime; + + /** acl list. */ + @JsonProperty("acls") + private List acls; + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getQuotaInBytes() { + return quotaInBytes; + } + + public void setQuotaInBytes(long quotaInBytes) { + this.quotaInBytes = quotaInBytes; + } + + public long getQuotaInNamespace() { + return quotaInNamespace; + } + + public void setQuotaInNamespace(long quotaInNamespace) { + this.quotaInNamespace = quotaInNamespace; + } + + public long getUsedNamespace() { + return usedNamespace; + } + + public void setUsedNamespace(long usedNamespace) { + this.usedNamespace = usedNamespace; + } + + public long getCreationTime() { + return creationTime; + } + + public void setCreationTime(long creationTime) { + this.creationTime = creationTime; + } + + public long getModificationTime() { + return modificationTime; + } + + public void setModificationTime(long modificationTime) { + this.modificationTime = modificationTime; + } + + public List getAcls() { + return acls; + } + + public void setAcls(List acls) { + this.acls = acls; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java new file mode 100644 index 000000000000..6d8691857f3d --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Encapsulates the low level volume info. + */ +public class VolumeObjectDBInfo extends ObjectDBInfo { + /** object admin from om db. */ + @JsonProperty("admin") + private String admin; + + /** owner of the object. */ + @JsonProperty("owner") + private String owner; + + public String getAdmin() { + return admin; + } + + public void setAdmin(String admin) { + this.admin = admin; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index cdfb3ab28fda..83da65ca10e2 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -21,16 +21,20 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -39,12 +43,15 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -52,6 +59,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -64,12 +72,14 @@ import java.io.IOException; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; @@ -390,14 +400,46 @@ public void testUtility() { @Test public void testGetBasicInfoRoot() throws Exception { // Test root basics + String username = "myuser"; + OmPrefixInfo omPrefixInfo = getOmPrefixInfoForTest(ROOT_PATH, + IAccessAuthorizer.ACLIdentityType.USER, + username, + IAccessAuthorizer.ACLType.WRITE, + ACCESS); + omPrefixInfo.getMetadata().put("key", "value"); + reconOMMetadataManager.getPrefixTable() + .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); NamespaceSummaryResponse rootResponseObj = (NamespaceSummaryResponse) rootResponse.getEntity(); Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - Assert.assertEquals(2, rootResponseObj.getNumVolume()); - Assert.assertEquals(4, rootResponseObj.getNumBucket()); - Assert.assertEquals(5, rootResponseObj.getNumTotalDir()); - Assert.assertEquals(10, rootResponseObj.getNumTotalKey()); + Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); + Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals(IAccessAuthorizer.ACLIdentityType.USER, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); + Assert.assertEquals(IAccessAuthorizer.ACLType.WRITE.toString(), + rootResponseObj.getObjectDBInfo().getAcls().get(0) + .getAclList().get(0).toString()); + Assert.assertEquals(username, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); + Assert.assertEquals("value", + rootResponseObj.getObjectDBInfo().getMetadata().get("key")); + Assert.assertEquals(ACCESS, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getAclScope()); + } + + private OmPrefixInfo getOmPrefixInfoForTest( + String path, + IAccessAuthorizer.ACLIdentityType identityType, + String identityString, + IAccessAuthorizer.ACLType aclType, + OzoneAcl.AclScope scope) { + return new OmPrefixInfo(path, + Collections.singletonList(new OzoneAcl( + identityType, identityString, + aclType, scope)), new HashMap<>(), 10, 100); } @Test @@ -407,9 +449,18 @@ public void testGetBasicInfoVol() throws Exception { NamespaceSummaryResponse volResponseObj = (NamespaceSummaryResponse) volResponse.getEntity(); Assert.assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); - Assert.assertEquals(2, volResponseObj.getNumBucket()); - Assert.assertEquals(4, volResponseObj.getNumTotalDir()); - Assert.assertEquals(6, volResponseObj.getNumTotalKey()); + Assert.assertEquals(2, volResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(6, volResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getAdmin()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getOwner()); + Assert.assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); + Assert.assertEquals(2097152, + volResponseObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(-1, + volResponseObj.getObjectDBInfo().getQuotaInNamespace()); } @Test @@ -420,8 +471,18 @@ public void testGetBasicInfoBucketOne() throws Exception { NamespaceSummaryResponse bucketOneObj = (NamespaceSummaryResponse) bucketOneResponse.getEntity(); Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - Assert.assertEquals(4, bucketOneObj.getNumTotalDir()); - Assert.assertEquals(4, bucketOneObj.getNumTotalKey()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket1", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); } @Test @@ -432,8 +493,18 @@ public void testGetBasicInfoBucketTwo() throws Exception { NamespaceSummaryResponse bucketTwoObj = (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - Assert.assertEquals(0, bucketTwoObj.getNumTotalDir()); - Assert.assertEquals(2, bucketTwoObj.getNumTotalKey()); + Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket2", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); } @Test @@ -443,8 +514,13 @@ public void testGetBasicInfoDir() throws Exception { NamespaceSummaryResponse dirOneObj = (NamespaceSummaryResponse) dirOneResponse.getEntity(); Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - Assert.assertEquals(3, dirOneObj.getNumTotalDir()); - Assert.assertEquals(3, dirOneObj.getNumTotalKey()); + Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); } @Test @@ -455,6 +531,8 @@ public void testGetBasicInfoNoPath() throws Exception { (NamespaceSummaryResponse) invalidResponse.getEntity(); Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + Assert.assertEquals(null, invalidObj.getCountStats()); + Assert.assertEquals(null, invalidObj.getObjectDBInfo()); } @Test @@ -464,6 +542,17 @@ public void testGetBasicInfoKey() throws Exception { NamespaceSummaryResponse keyResObj = (NamespaceSummaryResponse) keyResponse.getEntity(); Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); + Assert.assertEquals("vol", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals("bucket2", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); + Assert.assertEquals("file4", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); + Assert.assertEquals(2049, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); + Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). + getReplicationConfig().getReplicationType()); } @Test diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index ccbdd3619577..0bf2432fb401 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -21,17 +21,21 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -40,12 +44,15 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.DUResponse; import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -53,6 +60,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -65,12 +73,14 @@ import java.io.IOException; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; @@ -389,15 +399,47 @@ public void testUtility() { @Test public void testGetBasicInfoRoot() throws Exception { + String username = "myuser"; + OmPrefixInfo omPrefixInfo = getOmPrefixInfoForTest(ROOT_PATH, + IAccessAuthorizer.ACLIdentityType.USER, + username, + IAccessAuthorizer.ACLType.WRITE, + ACCESS); + omPrefixInfo.getMetadata().put("key", "value"); + reconOMMetadataManager.getPrefixTable() + .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); // Test root basics Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); NamespaceSummaryResponse rootResponseObj = (NamespaceSummaryResponse) rootResponse.getEntity(); Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - Assert.assertEquals(2, rootResponseObj.getNumVolume()); - Assert.assertEquals(4, rootResponseObj.getNumBucket()); - Assert.assertEquals(5, rootResponseObj.getNumTotalDir()); - Assert.assertEquals(10, rootResponseObj.getNumTotalKey()); + Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); + Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals(IAccessAuthorizer.ACLIdentityType.USER, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); + Assert.assertEquals(IAccessAuthorizer.ACLType.WRITE.toString(), + rootResponseObj.getObjectDBInfo().getAcls().get(0) + .getAclList().get(0).toString()); + Assert.assertEquals(username, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); + Assert.assertEquals("value", + rootResponseObj.getObjectDBInfo().getMetadata().get("key")); + Assert.assertEquals(ACCESS, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getAclScope()); + } + + private OmPrefixInfo getOmPrefixInfoForTest( + String path, + IAccessAuthorizer.ACLIdentityType identityType, + String identityString, + IAccessAuthorizer.ACLType aclType, + OzoneAcl.AclScope scope) { + return new OmPrefixInfo(path, + Collections.singletonList(new OzoneAcl( + identityType, identityString, + aclType, scope)), new HashMap<>(), 10, 100); } @Test @@ -407,9 +449,18 @@ public void testGetBasicInfoVol() throws Exception { NamespaceSummaryResponse volResponseObj = (NamespaceSummaryResponse) volResponse.getEntity(); Assert.assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); - Assert.assertEquals(2, volResponseObj.getNumBucket()); - Assert.assertEquals(4, volResponseObj.getNumTotalDir()); - Assert.assertEquals(6, volResponseObj.getNumTotalKey()); + Assert.assertEquals(2, volResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(6, volResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getAdmin()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getOwner()); + Assert.assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); + Assert.assertEquals(2097152, + volResponseObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(-1, + volResponseObj.getObjectDBInfo().getQuotaInNamespace()); } @Test @@ -420,8 +471,18 @@ public void testGetBasicInfoBucketOne() throws Exception { NamespaceSummaryResponse bucketOneObj = (NamespaceSummaryResponse) bucketOneResponse.getEntity(); Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - Assert.assertEquals(4, bucketOneObj.getNumTotalDir()); - Assert.assertEquals(4, bucketOneObj.getNumTotalKey()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket1", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); } @Test @@ -432,8 +493,18 @@ public void testGetBasicInfoBucketTwo() throws Exception { NamespaceSummaryResponse bucketTwoObj = (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - Assert.assertEquals(0, bucketTwoObj.getNumTotalDir()); - Assert.assertEquals(2, bucketTwoObj.getNumTotalKey()); + Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket2", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); } @Test @@ -443,8 +514,13 @@ public void testGetBasicInfoDir() throws Exception { NamespaceSummaryResponse dirOneObj = (NamespaceSummaryResponse) dirOneResponse.getEntity(); Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - Assert.assertEquals(3, dirOneObj.getNumTotalDir()); - Assert.assertEquals(3, dirOneObj.getNumTotalKey()); + Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); + Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); } @Test @@ -455,6 +531,8 @@ public void testGetBasicInfoNoPath() throws Exception { (NamespaceSummaryResponse) invalidResponse.getEntity(); Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + Assert.assertEquals(null, invalidObj.getCountStats()); + Assert.assertEquals(null, invalidObj.getObjectDBInfo()); } @Test @@ -464,6 +542,17 @@ public void testGetBasicInfoKey() throws Exception { NamespaceSummaryResponse keyResObj = (NamespaceSummaryResponse) keyResponse.getEntity(); Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); + Assert.assertEquals("vol", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals("bucket2", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); + Assert.assertEquals("file4", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); + Assert.assertEquals(2049, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); + Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). + getReplicationConfig().getReplicationType()); } @Test From a11acd6d76477eb8ff683ab1f9e9accca4146182 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Mon, 5 Dec 2022 15:28:07 +0530 Subject: [PATCH 2/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../ozone/recon/api/TestNSSummaryEndpointWithLegacy.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index 0bf2432fb401..2c61fd78c3c0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -478,7 +478,7 @@ public void testGetBasicInfoBucketOne() throws Exception { Assert.assertEquals(StorageType.DISK, ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + Assert.assertEquals(BucketLayout.LEGACY, ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getBucketLayout()); Assert.assertEquals("bucket1", @@ -500,7 +500,7 @@ public void testGetBasicInfoBucketTwo() throws Exception { Assert.assertEquals(StorageType.DISK, ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + Assert.assertEquals(BucketLayout.LEGACY, ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getBucketLayout()); Assert.assertEquals("bucket2", From 7e64e58d1928a7b8f55c798c5c4d0f1b4c0a4825 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Wed, 7 Dec 2022 17:32:59 +0530 Subject: [PATCH 3/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../recon/TestReconWithOzoneManagerFSO.java | 2 + .../api/handlers/BucketEntityHandler.java | 32 +-- .../api/handlers/DirectoryEntityHandler.java | 16 +- .../recon/api/handlers/KeyEntityHandler.java | 22 +- .../recon/api/handlers/RootEntityHandler.java | 14 +- .../api/handlers/VolumeEntityHandler.java | 21 +- .../recon/api/types/BucketObjectDBInfo.java | 79 ++++-- .../recon/api/types/KeyObjectDBInfo.java | 54 +++++ .../ozone/recon/api/types/ObjectDBInfo.java | 100 +++++++- .../recon/api/types/VolumeObjectDBInfo.java | 52 +++- .../api/TestNSSummaryEndpointWithFSO.java | 150 ++---------- .../api/TestNSSummaryEndpointWithLegacy.java | 151 ++---------- .../ozone/recon/common/CommonUtils.java | 225 ++++++++++++++++++ 13 files changed, 533 insertions(+), 385 deletions(-) create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index d3681459ce2e..7af632830bb5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -130,6 +130,8 @@ public void testNamespaceSummaryAPI() throws Exception { Assert.assertSame(entity.getEntityType(), EntityType.DIRECTORY); Assert.assertEquals(1, entity.getCountStats().getNumTotalKey()); Assert.assertEquals(0, entity.getCountStats().getNumTotalDir()); + Assert.assertEquals(-1, entity.getCountStats().getNumVolume()); + Assert.assertEquals(-1, entity.getCountStats().getNumBucket()); for (int i = 0; i < 10; i++) { Assert.assertNotNull(impl.getOMMetadataManagerInstance() .getVolumeTable().get("/vol" + i)); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 67e3caa15e07..9878d9a441b3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -60,15 +60,13 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, -1, getTotalDirCount(bucketObjectId), getTotalKeyCount(bucketObjectId)); - NamespaceSummaryResponse namespaceSummaryResponse = + return NamespaceSummaryResponse.newBuilder() .setEntityType(EntityType.BUCKET) .setCountStats(countStats) .setObjectDBInfo(getBucketObjDbInfo(names)) .setStatus(ResponseStatus.OK) .build(); - - return namespaceSummaryResponse; } private BucketObjectDBInfo getBucketObjDbInfo(String[] names) @@ -79,31 +77,9 @@ private BucketObjectDBInfo getBucketObjDbInfo(String[] names) getBucketKey(volName, bucketName); OmBucketInfo omBucketInfo = getOmMetadataManager().getBucketTable().getSkipCache(bucketKey); - BucketObjectDBInfo bucketObjectDBInfo = new BucketObjectDBInfo(); - if (null != omBucketInfo) { - bucketObjectDBInfo.setMetadata(omBucketInfo.getMetadata()); - bucketObjectDBInfo.setVolumeName(omBucketInfo.getVolumeName()); - bucketObjectDBInfo.setName(omBucketInfo.getBucketName()); - bucketObjectDBInfo.setQuotaInBytes(omBucketInfo.getQuotaInBytes()); - bucketObjectDBInfo.setQuotaInNamespace( - omBucketInfo.getQuotaInNamespace()); - bucketObjectDBInfo.setUsedNamespace(omBucketInfo.getUsedNamespace()); - bucketObjectDBInfo.setCreationTime(omBucketInfo.getCreationTime()); - bucketObjectDBInfo.setModificationTime( - omBucketInfo.getModificationTime()); - bucketObjectDBInfo.setAcls(omBucketInfo.getAcls()); - bucketObjectDBInfo.setSourceBucket(omBucketInfo.getSourceBucket()); - bucketObjectDBInfo.setSourceVolume(omBucketInfo.getSourceVolume()); - bucketObjectDBInfo.setBekInfo(omBucketInfo.getEncryptionKeyInfo()); - bucketObjectDBInfo.setVersioningEnabled( - omBucketInfo.getIsVersionEnabled()); - bucketObjectDBInfo.setStorageType(omBucketInfo.getStorageType()); - bucketObjectDBInfo.setDefaultReplicationConfig( - omBucketInfo.getDefaultReplicationConfig()); - bucketObjectDBInfo.setBucketLayout(omBucketInfo.getBucketLayout()); - bucketObjectDBInfo.setOwner(omBucketInfo.getOwner()); - } - return bucketObjectDBInfo; + return BucketObjectDBInfo.newBuilder() + .setOmBucketInfo(omBucketInfo) + .build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index e7027609df3d..79ae192a462e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -61,29 +61,21 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, -1, getTotalDirCount(dirObjectId), getTotalKeyCount(dirObjectId)); - NamespaceSummaryResponse namespaceSummaryResponse = + return NamespaceSummaryResponse.newBuilder() .setEntityType(EntityType.DIRECTORY) .setCountStats(countStats) .setObjectDBInfo(getDirectoryObjDbInfo(getNames())) .setStatus(ResponseStatus.OK) .build(); - - return namespaceSummaryResponse; } private ObjectDBInfo getDirectoryObjDbInfo(String[] names) throws IOException { OmDirectoryInfo dirInfo = getBucketHandler().getDirInfo(names); - ObjectDBInfo objectDBInfo = new ObjectDBInfo(); - if (null != dirInfo) { - objectDBInfo.setName(dirInfo.getName()); - objectDBInfo.setCreationTime(dirInfo.getCreationTime()); - objectDBInfo.setModificationTime(dirInfo.getModificationTime()); - objectDBInfo.setAcls(dirInfo.getAcls()); - objectDBInfo.setMetadata(dirInfo.getMetadata()); - } - return objectDBInfo; + return ObjectDBInfo.newDirObjectDbInfoBuilder() + .setDirInfo(dirInfo) + .build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java index 0e5621531147..4faff5856531 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java @@ -58,30 +58,14 @@ public NamespaceSummaryResponse getSummaryResponse() .setObjectDBInfo(getKeyDbObjectInfo(getNames())) .setStatus(ResponseStatus.OK) .build(); - } private ObjectDBInfo getKeyDbObjectInfo(String[] names) throws IOException { OmKeyInfo keyInfo = getBucketHandler().getKeyInfo(names); - KeyObjectDBInfo keyObjectDBInfo = new KeyObjectDBInfo(); - if (null != keyInfo) { - keyObjectDBInfo.setVolumeName(keyInfo.getVolumeName()); - keyObjectDBInfo.setBucketName(keyInfo.getBucketName()); - keyObjectDBInfo.setName(keyInfo.getKeyName()); - keyObjectDBInfo.setKeyName(keyInfo.getKeyName()); - keyObjectDBInfo.setDataSize(keyInfo.getDataSize()); - keyObjectDBInfo.setKeyLocationVersions(keyInfo.getKeyLocationVersions()); - keyObjectDBInfo.setCreationTime(keyInfo.getCreationTime()); - keyObjectDBInfo.setModificationTime(keyInfo.getModificationTime()); - keyObjectDBInfo.setReplicationConfig(keyInfo.getReplicationConfig()); - keyObjectDBInfo.setEncInfo(keyInfo.getFileEncryptionInfo()); - keyObjectDBInfo.setFileName(keyInfo.getFileName()); - keyObjectDBInfo.setFile(keyInfo.isFile()); - keyObjectDBInfo.setAcls(keyInfo.getAcls()); - keyObjectDBInfo.setMetadata(keyInfo.getMetadata()); - } - return keyObjectDBInfo; + return KeyObjectDBInfo.newBuilder() + .setOmKeyInfo(keyInfo) + .build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index 651b440c806b..4b2be3492941 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -66,15 +66,13 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( volumes.size(), allBuckets.size(), totalNumDir, totalNumKey); - NamespaceSummaryResponse namespaceSummaryResponse = + return NamespaceSummaryResponse.newBuilder() .setEntityType(EntityType.ROOT) .setCountStats(countStats) .setObjectDBInfo(getPrefixObjDbInfo()) .setStatus(ResponseStatus.OK) .build(); - - return namespaceSummaryResponse; } private ObjectDBInfo getPrefixObjDbInfo() @@ -82,13 +80,9 @@ private ObjectDBInfo getPrefixObjDbInfo() OmPrefixInfo omPrefixInfo = getOmMetadataManager().getPrefixTable(). getSkipCache(OzoneConsts.OM_KEY_PREFIX); - ObjectDBInfo objectDBInfo = new ObjectDBInfo(); - if (null != omPrefixInfo) { - objectDBInfo.setMetadata(omPrefixInfo.getMetadata()); - objectDBInfo.setName(omPrefixInfo.getName()); - objectDBInfo.setAcls(omPrefixInfo.getAcls()); - } - return objectDBInfo; + return ObjectDBInfo.newPrefixObjectDbInfoBuilder() + .setOmPrefixInfo(omPrefixInfo) + .build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index 8481cd95af8e..72c6bf66c96f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -67,15 +67,13 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, buckets.size(), totalDir, totalKey); - NamespaceSummaryResponse namespaceSummaryResponse = + return NamespaceSummaryResponse.newBuilder() .setEntityType(EntityType.VOLUME) .setCountStats(countStats) .setObjectDBInfo(getVolumeObjDbInfo(names)) .setStatus(ResponseStatus.OK) .build(); - - return namespaceSummaryResponse; } private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) @@ -83,20 +81,9 @@ private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) String dbVolumeKey = getOmMetadataManager().getVolumeKey(names[0]); OmVolumeArgs volumeArgs = getOmMetadataManager().getVolumeTable().getSkipCache(dbVolumeKey); - VolumeObjectDBInfo objectDBInfo = new VolumeObjectDBInfo(); - if (null != volumeArgs) { - objectDBInfo.setMetadata(volumeArgs.getMetadata()); - objectDBInfo.setName(volumeArgs.getVolume()); - objectDBInfo.setAdmin(volumeArgs.getAdminName()); - objectDBInfo.setOwner(volumeArgs.getOwnerName()); - objectDBInfo.setQuotaInBytes(volumeArgs.getQuotaInBytes()); - objectDBInfo.setQuotaInNamespace(volumeArgs.getQuotaInNamespace()); - objectDBInfo.setUsedNamespace(volumeArgs.getUsedNamespace()); - objectDBInfo.setCreationTime(volumeArgs.getCreationTime()); - objectDBInfo.setModificationTime(volumeArgs.getModificationTime()); - objectDBInfo.setAcls(volumeArgs.getAcls()); - } - return objectDBInfo; + return VolumeObjectDBInfo.newBuilder() + .setVolumeArgs(volumeArgs) + .build(); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java index 807da37f71c3..d8f64cfeb9dc 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java @@ -23,59 +23,74 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; /** * Encapsulates the low level bucket info. */ public class BucketObjectDBInfo extends ObjectDBInfo { - /** volume name from om db. */ @JsonProperty("volumeName") private String volumeName; - /** storage type for the object. */ @JsonProperty("storageType") private StorageType storageType; - /** is versioning enabled for the object. */ @JsonProperty("versioning") private boolean isVersioningEnabled; - /** used bytes for the object. */ @JsonProperty("usedBytes") private String usedBytes; - /** - * Bucket encryption key info if encryption is enabled. - */ @JsonProperty("encryptionInfo") private BucketEncryptionKeyInfo bekInfo; - /** - * Optional default replication for bucket. - */ @JsonProperty("replicationConfigInfo") private DefaultReplicationConfig defaultReplicationConfig; - /** source volume. */ @JsonProperty("sourceVolume") private String sourceVolume; - /** source bucket. */ @JsonProperty("sourceBucket") private String sourceBucket; - /** - * Bucket Layout. - */ @JsonProperty("bucketLayout") private BucketLayout bucketLayout; - /** - * bucket owner. - */ @JsonProperty("owner") private String owner; + public static BucketObjectDBInfo.Builder newBuilder() { + return new BucketObjectDBInfo.Builder(); + } + + public BucketObjectDBInfo() { + + } + + public BucketObjectDBInfo(Builder b) { + this.setMetadata(b.getOmBucketInfo().getMetadata()); + this.setVolumeName(b.getOmBucketInfo().getVolumeName()); + this.setName(b.getOmBucketInfo().getBucketName()); + this.setQuotaInBytes(b.getOmBucketInfo().getQuotaInBytes()); + this.setQuotaInNamespace( + b.getOmBucketInfo().getQuotaInNamespace()); + this.setUsedNamespace(b.getOmBucketInfo().getUsedNamespace()); + this.setCreationTime(b.getOmBucketInfo().getCreationTime()); + this.setModificationTime( + b.getOmBucketInfo().getModificationTime()); + this.setAcls(b.getOmBucketInfo().getAcls()); + this.setSourceBucket(b.getOmBucketInfo().getSourceBucket()); + this.setSourceVolume(b.getOmBucketInfo().getSourceVolume()); + this.setBekInfo(b.getOmBucketInfo().getEncryptionKeyInfo()); + this.setVersioningEnabled( + b.getOmBucketInfo().getIsVersionEnabled()); + this.setStorageType(b.getOmBucketInfo().getStorageType()); + this.setDefaultReplicationConfig( + b.getOmBucketInfo().getDefaultReplicationConfig()); + this.setBucketLayout(b.getOmBucketInfo().getBucketLayout()); + this.setOwner(b.getOmBucketInfo().getOwner()); + } + public String getVolumeName() { return volumeName; } @@ -156,4 +171,32 @@ public String getOwner() { public void setOwner(String owner) { this.owner = owner; } + + /** + * Builder for BucketObjectDBInfo. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class Builder { + private OmBucketInfo omBucketInfo; + public Builder() { + + } + + public BucketObjectDBInfo.Builder setOmBucketInfo( + OmBucketInfo omBucketInfo) { + this.omBucketInfo = omBucketInfo; + return this; + } + + public OmBucketInfo getOmBucketInfo() { + return omBucketInfo; + } + + public BucketObjectDBInfo build() { + if (null == this.omBucketInfo) { + return new BucketObjectDBInfo(); + } + return new BucketObjectDBInfo(this); + } + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java index bf6c88988f16..17e2286da834 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import java.util.List; @@ -57,6 +58,31 @@ public class KeyObjectDBInfo extends ObjectDBInfo { */ private String fileName; + public static KeyObjectDBInfo.Builder newBuilder() { + return new KeyObjectDBInfo.Builder(); + } + + public KeyObjectDBInfo() { + + } + + public KeyObjectDBInfo(Builder b) { + this.setVolumeName(b.getOmKeyInfo().getVolumeName()); + this.setBucketName(b.getOmKeyInfo().getBucketName()); + this.setName(b.getOmKeyInfo().getKeyName()); + this.setKeyName(b.getOmKeyInfo().getKeyName()); + this.setDataSize(b.getOmKeyInfo().getDataSize()); + this.setKeyLocationVersions(b.getOmKeyInfo().getKeyLocationVersions()); + this.setCreationTime(b.getOmKeyInfo().getCreationTime()); + this.setModificationTime(b.getOmKeyInfo().getModificationTime()); + this.setReplicationConfig(b.getOmKeyInfo().getReplicationConfig()); + this.setEncInfo(b.getOmKeyInfo().getFileEncryptionInfo()); + this.setFileName(b.getOmKeyInfo().getFileName()); + this.setFile(b.getOmKeyInfo().isFile()); + this.setAcls(b.getOmKeyInfo().getAcls()); + this.setMetadata(b.getOmKeyInfo().getMetadata()); + } + public String getVolumeName() { return volumeName; } @@ -129,4 +155,32 @@ public FileEncryptionInfo getEncInfo() { public void setEncInfo(FileEncryptionInfo encInfo) { this.encInfo = encInfo; } + + /** + * Builder for KeyObjectDBInfo. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class Builder { + private OmKeyInfo omKeyInfo; + public Builder() { + + } + + public KeyObjectDBInfo.Builder setOmKeyInfo( + OmKeyInfo omKeyInfo) { + this.omKeyInfo = omKeyInfo; + return this; + } + + public OmKeyInfo getOmKeyInfo() { + return omKeyInfo; + } + + public KeyObjectDBInfo build() { + if (null == this.omKeyInfo) { + return new KeyObjectDBInfo(); + } + return new KeyObjectDBInfo(this); + } + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java index b0db053cebaa..8d7d65269bbe 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import java.util.List; import java.util.Map; @@ -28,38 +30,62 @@ * Encapsulates the low level volume/bucket/dir info. */ public class ObjectDBInfo { - /** object metadata from om db. */ @JsonProperty("metadata") private Map metadata; - /** name of the object. */ @JsonProperty("name") private String name; - /** quota in bytes. */ @JsonProperty("quotaInBytes") private long quotaInBytes; - /** quota in namespace. */ @JsonProperty("quotaInNamespace") private long quotaInNamespace; - /** used namespace. */ @JsonProperty("usedNamespace") private long usedNamespace; - /** creation time. */ @JsonProperty("creationTime") private long creationTime; - /** modification time. */ @JsonProperty("modificationTime") private long modificationTime; - /** acl list. */ @JsonProperty("acls") private List acls; + public static ObjectDBInfo.PrefixObjectDbInfoBuilder + newPrefixObjectDbInfoBuilder() { + return new ObjectDBInfo.PrefixObjectDbInfoBuilder(); + } + + public ObjectDBInfo(PrefixObjectDbInfoBuilder + prefixObjectDbInfoBuilder) { + this.setMetadata(prefixObjectDbInfoBuilder. + getOmPrefixInfo().getMetadata()); + this.setName(prefixObjectDbInfoBuilder. + getOmPrefixInfo().getName()); + this.setAcls(prefixObjectDbInfoBuilder. + getOmPrefixInfo().getAcls()); + } + + public static ObjectDBInfo.DirObjectDbInfoBuilder + newDirObjectDbInfoBuilder() { + return new ObjectDBInfo.DirObjectDbInfoBuilder(); + } + + public ObjectDBInfo() { + + } + + public ObjectDBInfo(DirObjectDbInfoBuilder b) { + this.setName(b.getDirInfo().getName()); + this.setCreationTime(b.getDirInfo().getCreationTime()); + this.setModificationTime(b.getDirInfo().getModificationTime()); + this.setAcls(b.getDirInfo().getAcls()); + this.setMetadata(b.getDirInfo().getMetadata()); + } + public Map getMetadata() { return metadata; } @@ -123,4 +149,62 @@ public List getAcls() { public void setAcls(List acls) { this.acls = acls; } + + /** + * Builder for Directory ObjectDBInfo. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class DirObjectDbInfoBuilder { + private OmDirectoryInfo dirInfo; + + public DirObjectDbInfoBuilder() { + + } + + public DirObjectDbInfoBuilder setDirInfo( + OmDirectoryInfo dirInfo) { + this.dirInfo = dirInfo; + return this; + } + + public OmDirectoryInfo getDirInfo() { + return dirInfo; + } + + public ObjectDBInfo build() { + if (null == this.dirInfo) { + return new ObjectDBInfo(); + } + return new ObjectDBInfo(this); + } + } + + /** + * Builder for Prefix ObjectDBInfo. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class PrefixObjectDbInfoBuilder { + private OmPrefixInfo omPrefixInfo; + + public PrefixObjectDbInfoBuilder() { + + } + + public PrefixObjectDbInfoBuilder setOmPrefixInfo( + OmPrefixInfo omPrefixInfo) { + this.omPrefixInfo = omPrefixInfo; + return this; + } + + public OmPrefixInfo getOmPrefixInfo() { + return omPrefixInfo; + } + + public ObjectDBInfo build() { + if (null == this.omPrefixInfo) { + return new ObjectDBInfo(); + } + return new ObjectDBInfo(this); + } + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java index 6d8691857f3d..cba539be5df7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java @@ -19,19 +19,39 @@ package org.apache.hadoop.ozone.recon.api.types; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; /** * Encapsulates the low level volume info. */ public class VolumeObjectDBInfo extends ObjectDBInfo { - /** object admin from om db. */ @JsonProperty("admin") private String admin; - /** owner of the object. */ @JsonProperty("owner") private String owner; + public static VolumeObjectDBInfo.Builder newBuilder() { + return new VolumeObjectDBInfo.Builder(); + } + + public VolumeObjectDBInfo() { + + } + + public VolumeObjectDBInfo(Builder b) { + this.setMetadata(b.getVolumeArgs().getMetadata()); + this.setName(b.getVolumeArgs().getVolume()); + this.setAdmin(b.getVolumeArgs().getAdminName()); + this.setOwner(b.getVolumeArgs().getOwnerName()); + this.setQuotaInBytes(b.getVolumeArgs().getQuotaInBytes()); + this.setQuotaInNamespace(b.getVolumeArgs().getQuotaInNamespace()); + this.setUsedNamespace(b.getVolumeArgs().getUsedNamespace()); + this.setCreationTime(b.getVolumeArgs().getCreationTime()); + this.setModificationTime(b.getVolumeArgs().getModificationTime()); + this.setAcls(b.getVolumeArgs().getAcls()); + } + public String getAdmin() { return admin; } @@ -47,4 +67,32 @@ public String getOwner() { public void setOwner(String owner) { this.owner = owner; } + + /** + * Builder for VolumeObjectDBInfo. + */ + @SuppressWarnings("checkstyle:hiddenfield") + public static final class Builder { + private OmVolumeArgs volumeArgs; + public Builder() { + + } + + public VolumeObjectDBInfo.Builder setVolumeArgs( + OmVolumeArgs volumeArgs) { + this.volumeArgs = volumeArgs; + return this; + } + + public OmVolumeArgs getVolumeArgs() { + return volumeArgs; + } + + public VolumeObjectDBInfo build() { + if (null == this.volumeArgs) { + return new VolumeObjectDBInfo(); + } + return new VolumeObjectDBInfo(this); + } + } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index 83da65ca10e2..ab92c8261707 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -21,20 +21,16 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -43,15 +39,11 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.common.CommonUtils; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -59,7 +51,6 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; -import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -72,14 +63,12 @@ import java.io.IOException; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; @@ -119,6 +108,7 @@ public class TestNSSummaryEndpointWithFSO { private ReconOMMetadataManager reconOMMetadataManager; private NSSummaryEndpoint nsSummaryEndpoint; private OzoneConfiguration ozoneConfiguration; + private CommonUtils commonUtils; private static final String TEST_PATH_UTILITY = "/vol1/buck1/a/b/c/d/e/file1.txt"; @@ -385,6 +375,7 @@ public void setUp() throws Exception { new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, reconOMMetadataManager, ozoneConfiguration); nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); + commonUtils = new CommonUtils(); } @Test @@ -400,159 +391,48 @@ public void testUtility() { @Test public void testGetBasicInfoRoot() throws Exception { // Test root basics - String username = "myuser"; - OmPrefixInfo omPrefixInfo = getOmPrefixInfoForTest(ROOT_PATH, - IAccessAuthorizer.ACLIdentityType.USER, - username, - IAccessAuthorizer.ACLType.WRITE, - ACCESS); - omPrefixInfo.getMetadata().put("key", "value"); - reconOMMetadataManager.getPrefixTable() - .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); - Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals(IAccessAuthorizer.ACLIdentityType.USER, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); - Assert.assertEquals(IAccessAuthorizer.ACLType.WRITE.toString(), - rootResponseObj.getObjectDBInfo().getAcls().get(0) - .getAclList().get(0).toString()); - Assert.assertEquals(username, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); - Assert.assertEquals("value", - rootResponseObj.getObjectDBInfo().getMetadata().get("key")); - Assert.assertEquals(ACCESS, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getAclScope()); - } - - private OmPrefixInfo getOmPrefixInfoForTest( - String path, - IAccessAuthorizer.ACLIdentityType identityType, - String identityString, - IAccessAuthorizer.ACLType aclType, - OzoneAcl.AclScope scope) { - return new OmPrefixInfo(path, - Collections.singletonList(new OzoneAcl( - identityType, identityString, - aclType, scope)), new HashMap<>(), 10, 100); + commonUtils.testNSSummaryBasicInfoRoot( + nsSummaryEndpoint, reconOMMetadataManager); } @Test public void testGetBasicInfoVol() throws Exception { // Test volume basics - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - Assert.assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); - Assert.assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(6, volResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getAdmin()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getOwner()); - Assert.assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); - Assert.assertEquals(2097152, - volResponseObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(-1, - volResponseObj.getObjectDBInfo().getQuotaInNamespace()); + commonUtils.testNSSummaryBasicInfoVolume(nsSummaryEndpoint); } @Test public void testGetBasicInfoBucketOne() throws Exception { // Test bucket 1's basics - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket1", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); + commonUtils.testNSSummaryBasicInfoBucketOne( + BucketLayout.FILE_SYSTEM_OPTIMIZED, + nsSummaryEndpoint); } @Test public void testGetBasicInfoBucketTwo() throws Exception { // Test bucket 2's basics - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket2", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); + commonUtils.testNSSummaryBasicInfoBucketTwo( + BucketLayout.FILE_SYSTEM_OPTIMIZED, + nsSummaryEndpoint); } @Test public void testGetBasicInfoDir() throws Exception { // Test intermediate directory basics - Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); - NamespaceSummaryResponse dirOneObj = - (NamespaceSummaryResponse) dirOneResponse.getEntity(); - Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); + commonUtils.testNSSummaryBasicInfoDir(nsSummaryEndpoint); } @Test public void testGetBasicInfoNoPath() throws Exception { // Test invalid path - Response invalidResponse = nsSummaryEndpoint.getBasicInfo(INVALID_PATH); - NamespaceSummaryResponse invalidObj = - (NamespaceSummaryResponse) invalidResponse.getEntity(); - Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - Assert.assertEquals(null, invalidObj.getCountStats()); - Assert.assertEquals(null, invalidObj.getObjectDBInfo()); + commonUtils.testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); } @Test public void testGetBasicInfoKey() throws Exception { // Test key - Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); - NamespaceSummaryResponse keyResObj = - (NamespaceSummaryResponse) keyResponse.getEntity(); - Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); - Assert.assertEquals("vol", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals("bucket2", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); - Assert.assertEquals("file4", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); - Assert.assertEquals(2049, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); - Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). - getReplicationConfig().getReplicationType()); + commonUtils.testNSSummaryBasicInfoKey(nsSummaryEndpoint); } @Test diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java index 2c61fd78c3c0..43587ede281f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -21,21 +21,17 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -44,15 +40,11 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.EntityType; import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.common.CommonUtils; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; @@ -60,7 +52,6 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; -import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -73,14 +64,12 @@ import java.io.IOException; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; @@ -120,6 +109,7 @@ public class TestNSSummaryEndpointWithLegacy { private ReconOMMetadataManager reconOMMetadataManager; private NSSummaryEndpoint nsSummaryEndpoint; private OzoneConfiguration conf; + private CommonUtils commonUtils; private static final String TEST_PATH_UTILITY = "/vol1/buck1/a/b/c/d/e/file1.txt"; @@ -385,6 +375,7 @@ public void setUp() throws Exception { new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, reconOMMetadataManager, conf); nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); + commonUtils = new CommonUtils(); } @Test @@ -399,160 +390,48 @@ public void testUtility() { @Test public void testGetBasicInfoRoot() throws Exception { - String username = "myuser"; - OmPrefixInfo omPrefixInfo = getOmPrefixInfoForTest(ROOT_PATH, - IAccessAuthorizer.ACLIdentityType.USER, - username, - IAccessAuthorizer.ACLType.WRITE, - ACCESS); - omPrefixInfo.getMetadata().put("key", "value"); - reconOMMetadataManager.getPrefixTable() - .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); - // Test root basics - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); - Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals(IAccessAuthorizer.ACLIdentityType.USER, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); - Assert.assertEquals(IAccessAuthorizer.ACLType.WRITE.toString(), - rootResponseObj.getObjectDBInfo().getAcls().get(0) - .getAclList().get(0).toString()); - Assert.assertEquals(username, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); - Assert.assertEquals("value", - rootResponseObj.getObjectDBInfo().getMetadata().get("key")); - Assert.assertEquals(ACCESS, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getAclScope()); - } - - private OmPrefixInfo getOmPrefixInfoForTest( - String path, - IAccessAuthorizer.ACLIdentityType identityType, - String identityString, - IAccessAuthorizer.ACLType aclType, - OzoneAcl.AclScope scope) { - return new OmPrefixInfo(path, - Collections.singletonList(new OzoneAcl( - identityType, identityString, - aclType, scope)), new HashMap<>(), 10, 100); + commonUtils.testNSSummaryBasicInfoRoot( + nsSummaryEndpoint, reconOMMetadataManager); } @Test public void testGetBasicInfoVol() throws Exception { // Test volume basics - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - Assert.assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); - Assert.assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - Assert.assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(6, volResponseObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getAdmin()); - Assert.assertEquals("TestUser", - ((VolumeObjectDBInfo) volResponseObj.getObjectDBInfo()).getOwner()); - Assert.assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); - Assert.assertEquals(2097152, - volResponseObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(-1, - volResponseObj.getObjectDBInfo().getQuotaInNamespace()); + commonUtils.testNSSummaryBasicInfoVolume(nsSummaryEndpoint); } @Test public void testGetBasicInfoBucketOne() throws Exception { // Test bucket 1's basics - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.LEGACY, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket1", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); + commonUtils.testNSSummaryBasicInfoBucketOne( + BucketLayout.LEGACY, + nsSummaryEndpoint); } @Test public void testGetBasicInfoBucketTwo() throws Exception { // Test bucket 2's basics - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("vol", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - Assert.assertEquals(BucketLayout.LEGACY, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - Assert.assertEquals("bucket2", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); + commonUtils.testNSSummaryBasicInfoBucketTwo( + BucketLayout.LEGACY, + nsSummaryEndpoint); } @Test public void testGetBasicInfoDir() throws Exception { // Test intermediate directory basics - Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); - NamespaceSummaryResponse dirOneObj = - (NamespaceSummaryResponse) dirOneResponse.getEntity(); - Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); - Assert.assertEquals(3, dirOneObj.getCountStats().getNumTotalKey()); - Assert.assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); - Assert.assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); + commonUtils.testNSSummaryBasicInfoDir(nsSummaryEndpoint); } @Test public void testGetBasicInfoNoPath() throws Exception { // Test invalid path - Response invalidResponse = nsSummaryEndpoint.getBasicInfo(INVALID_PATH); - NamespaceSummaryResponse invalidObj = - (NamespaceSummaryResponse) invalidResponse.getEntity(); - Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - Assert.assertEquals(null, invalidObj.getCountStats()); - Assert.assertEquals(null, invalidObj.getObjectDBInfo()); + commonUtils.testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); } @Test public void testGetBasicInfoKey() throws Exception { // Test key - Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); - NamespaceSummaryResponse keyResObj = - (NamespaceSummaryResponse) keyResponse.getEntity(); - Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); - Assert.assertEquals("vol", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); - Assert.assertEquals("bucket2", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); - Assert.assertEquals("file4", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); - Assert.assertEquals(2049, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); - Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). - getReplicationConfig().getReplicationType()); + commonUtils.testNSSummaryBasicInfoKey(nsSummaryEndpoint); } @Test diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java new file mode 100644 index 000000000000..fe1b29c94f18 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.common; + +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.junit.Assert; + +import javax.ws.rs.core.Response; + +import java.util.Collections; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; + +/** + * This is a utility class for common code for test cases. + */ +public class CommonUtils { + private static final String ROOT_PATH = "/"; + private static final String VOL_PATH = "/vol"; + private static final String BUCKET_ONE_PATH = "/vol/bucket1"; + private static final String BUCKET_TWO_PATH = "/vol/bucket2"; + private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; + private static final String INVALID_PATH = "/vol/path/not/found"; + private static final String KEY_PATH = "/vol/bucket2/file4"; + + private OmPrefixInfo getOmPrefixInfoForTest( + String path, + IAccessAuthorizer.ACLIdentityType identityType, + String identityString, + IAccessAuthorizer.ACLType aclType, + OzoneAcl.AclScope scope) { + return new OmPrefixInfo(path, + Collections.singletonList(new OzoneAcl( + identityType, identityString, + aclType, scope)), new HashMap<>(), 10, 100); + } + + public void testNSSummaryBasicInfoRoot( + NSSummaryEndpoint nsSummaryEndpoint, + ReconOMMetadataManager reconOMMetadataManager) throws Exception { + String username = "myuser"; + OmPrefixInfo omPrefixInfo = getOmPrefixInfoForTest(ROOT_PATH, + IAccessAuthorizer.ACLIdentityType.USER, + username, + IAccessAuthorizer.ACLType.WRITE, + ACCESS); + omPrefixInfo.getMetadata().put("key", "value"); + reconOMMetadataManager.getPrefixTable() + .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); + // Test root basics + Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); + NamespaceSummaryResponse rootResponseObj = + (NamespaceSummaryResponse) rootResponse.getEntity(); + Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); + Assert.assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); + Assert.assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(5, rootResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(10, rootResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals(IAccessAuthorizer.ACLIdentityType.USER, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); + Assert.assertEquals(IAccessAuthorizer.ACLType.WRITE.toString(), + rootResponseObj.getObjectDBInfo().getAcls().get(0) + .getAclList().get(0).toString()); + Assert.assertEquals(username, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); + Assert.assertEquals("value", + rootResponseObj.getObjectDBInfo().getMetadata().get("key")); + Assert.assertEquals(ACCESS, + rootResponseObj.getObjectDBInfo().getAcls().get(0).getAclScope()); + } + + public void testNSSummaryBasicInfoVolume( + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); + NamespaceSummaryResponse volResponseObj = + (NamespaceSummaryResponse) volResponse.getEntity(); + Assert.assertEquals(EntityType.VOLUME, + volResponseObj.getEntityType()); + Assert.assertEquals(2, + volResponseObj.getCountStats().getNumBucket()); + Assert.assertEquals(4, + volResponseObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(6, + volResponseObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj. + getObjectDBInfo()).getAdmin()); + Assert.assertEquals("TestUser", + ((VolumeObjectDBInfo) volResponseObj. + getObjectDBInfo()).getOwner()); + Assert.assertEquals("vol", + volResponseObj.getObjectDBInfo().getName()); + Assert.assertEquals(2097152, + volResponseObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(-1, + volResponseObj.getObjectDBInfo().getQuotaInNamespace()); + } + + public void testNSSummaryBasicInfoBucketOne(BucketLayout bucketLayout, + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response bucketOneResponse = + nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); + NamespaceSummaryResponse bucketOneObj = + (NamespaceSummaryResponse) bucketOneResponse.getEntity(); + Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(4, bucketOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(bucketLayout, + ((BucketObjectDBInfo) + bucketOneObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket1", + ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); + } + + public void testNSSummaryBasicInfoBucketTwo( + BucketLayout bucketLayout, + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response bucketTwoResponse = + nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); + NamespaceSummaryResponse bucketTwoObj = + (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); + Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); + Assert.assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("vol", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getStorageType()); + Assert.assertEquals(bucketLayout, + ((BucketObjectDBInfo) + bucketTwoObj.getObjectDBInfo()).getBucketLayout()); + Assert.assertEquals("bucket2", + ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); + } + + public void testNSSummaryBasicInfoDir( + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); + NamespaceSummaryResponse dirOneObj = + (NamespaceSummaryResponse) dirOneResponse.getEntity(); + Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); + Assert.assertEquals(3, + dirOneObj.getCountStats().getNumTotalDir()); + Assert.assertEquals(3, + dirOneObj.getCountStats().getNumTotalKey()); + Assert.assertEquals("dir1", + dirOneObj.getObjectDBInfo().getName()); + Assert.assertEquals(0, + dirOneObj.getObjectDBInfo().getMetadata().size()); + Assert.assertEquals(0, + dirOneObj.getObjectDBInfo().getQuotaInBytes()); + Assert.assertEquals(0, + dirOneObj.getObjectDBInfo().getQuotaInNamespace()); + Assert.assertEquals(0, + dirOneObj.getObjectDBInfo().getUsedNamespace()); + } + + public void testNSSummaryBasicInfoNoPath( + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response invalidResponse = nsSummaryEndpoint + .getBasicInfo(INVALID_PATH); + NamespaceSummaryResponse invalidObj = + (NamespaceSummaryResponse) invalidResponse.getEntity(); + Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, + invalidObj.getStatus()); + Assert.assertEquals(null, invalidObj.getCountStats()); + Assert.assertEquals(null, invalidObj.getObjectDBInfo()); + } + + public void testNSSummaryBasicInfoKey( + NSSummaryEndpoint nsSummaryEndpoint) throws Exception { + Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); + NamespaceSummaryResponse keyResObj = + (NamespaceSummaryResponse) keyResponse.getEntity(); + Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); + Assert.assertEquals("vol", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); + Assert.assertEquals("bucket2", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); + Assert.assertEquals("file4", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); + Assert.assertEquals(2049, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); + Assert.assertEquals(HddsProtos.ReplicationType.STAND_ALONE, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). + getReplicationConfig().getReplicationType()); + } +} From 4045ed397eba3a906ba727aaebcc15f8b250cd6d Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Thu, 8 Dec 2022 12:15:42 +0530 Subject: [PATCH 4/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../ozone/recon/api/NSSummaryEndpoint.java | 7 +- .../api/handlers/BucketEntityHandler.java | 27 +++--- .../api/handlers/DirectoryEntityHandler.java | 22 ++--- .../recon/api/handlers/KeyEntityHandler.java | 9 +- .../recon/api/handlers/RootEntityHandler.java | 25 +++-- .../api/handlers/VolumeEntityHandler.java | 23 +++-- .../recon/api/types/BucketObjectDBInfo.java | 72 ++++---------- .../recon/api/types/KeyObjectDBInfo.java | 62 +++--------- .../ozone/recon/api/types/ObjectDBInfo.java | 94 +++---------------- .../recon/api/types/VolumeObjectDBInfo.java | 58 ++++-------- 10 files changed, 125 insertions(+), 274 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java index abad204313ba..5b104c461158 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpoint.java @@ -80,9 +80,10 @@ public Response getBasicInfo( NamespaceSummaryResponse namespaceSummaryResponse; if (!isInitializationComplete()) { namespaceSummaryResponse = - NamespaceSummaryResponse.newBuilder(). - setEntityType(EntityType.UNKNOWN).build(); - namespaceSummaryResponse.setStatus(ResponseStatus.INITIALIZING); + NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.UNKNOWN) + .setStatus(ResponseStatus.INITIALIZING) + .build(); return Response.ok(namespaceSummaryResponse).build(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java index 9878d9a441b3..7ad961195ee7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketEntityHandler.java @@ -60,13 +60,12 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, -1, getTotalDirCount(bucketObjectId), getTotalKeyCount(bucketObjectId)); - return - NamespaceSummaryResponse.newBuilder() - .setEntityType(EntityType.BUCKET) - .setCountStats(countStats) - .setObjectDBInfo(getBucketObjDbInfo(names)) - .setStatus(ResponseStatus.OK) - .build(); + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.BUCKET) + .setCountStats(countStats) + .setObjectDBInfo(getBucketObjDbInfo(names)) + .setStatus(ResponseStatus.OK) + .build(); } private BucketObjectDBInfo getBucketObjDbInfo(String[] names) @@ -75,11 +74,15 @@ private BucketObjectDBInfo getBucketObjDbInfo(String[] names) String bucketName = names[1]; String bucketKey = getOmMetadataManager(). getBucketKey(volName, bucketName); - OmBucketInfo omBucketInfo = - getOmMetadataManager().getBucketTable().getSkipCache(bucketKey); - return BucketObjectDBInfo.newBuilder() - .setOmBucketInfo(omBucketInfo) - .build(); + if (null == bucketKey) { + return new BucketObjectDBInfo(); + } + OmBucketInfo omBucketInfo = getOmMetadataManager() + .getBucketTable().getSkipCache(bucketKey); + if (null == omBucketInfo) { + return new BucketObjectDBInfo(); + } + return new BucketObjectDBInfo(omBucketInfo); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index 79ae192a462e..fc7022e2dab2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -61,21 +61,21 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, -1, getTotalDirCount(dirObjectId), getTotalKeyCount(dirObjectId)); - return - NamespaceSummaryResponse.newBuilder() - .setEntityType(EntityType.DIRECTORY) - .setCountStats(countStats) - .setObjectDBInfo(getDirectoryObjDbInfo(getNames())) - .setStatus(ResponseStatus.OK) - .build(); + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.DIRECTORY) + .setCountStats(countStats) + .setObjectDBInfo(getDirectoryObjDbInfo(getNames())) + .setStatus(ResponseStatus.OK) + .build(); } private ObjectDBInfo getDirectoryObjDbInfo(String[] names) throws IOException { - OmDirectoryInfo dirInfo = getBucketHandler().getDirInfo(names); - return ObjectDBInfo.newDirObjectDbInfoBuilder() - .setDirInfo(dirInfo) - .build(); + OmDirectoryInfo omDirectoryInfo = getBucketHandler().getDirInfo(names); + if (null == omDirectoryInfo) { + return new ObjectDBInfo(); + } + return new ObjectDBInfo(omDirectoryInfo); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java index 4faff5856531..fb31ad0bb62d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/KeyEntityHandler.java @@ -62,10 +62,11 @@ public NamespaceSummaryResponse getSummaryResponse() private ObjectDBInfo getKeyDbObjectInfo(String[] names) throws IOException { - OmKeyInfo keyInfo = getBucketHandler().getKeyInfo(names); - return KeyObjectDBInfo.newBuilder() - .setOmKeyInfo(keyInfo) - .build(); + OmKeyInfo omKeyInfo = getBucketHandler().getKeyInfo(names); + if (null == omKeyInfo) { + return new KeyObjectDBInfo(); + } + return new KeyObjectDBInfo(omKeyInfo); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java index 4b2be3492941..907aeed3bad9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/RootEntityHandler.java @@ -66,23 +66,22 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( volumes.size(), allBuckets.size(), totalNumDir, totalNumKey); - return - NamespaceSummaryResponse.newBuilder() - .setEntityType(EntityType.ROOT) - .setCountStats(countStats) - .setObjectDBInfo(getPrefixObjDbInfo()) - .setStatus(ResponseStatus.OK) - .build(); + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.ROOT) + .setCountStats(countStats) + .setObjectDBInfo(getPrefixObjDbInfo()) + .setStatus(ResponseStatus.OK) + .build(); } private ObjectDBInfo getPrefixObjDbInfo() throws IOException { - OmPrefixInfo omPrefixInfo = - getOmMetadataManager().getPrefixTable(). - getSkipCache(OzoneConsts.OM_KEY_PREFIX); - return ObjectDBInfo.newPrefixObjectDbInfoBuilder() - .setOmPrefixInfo(omPrefixInfo) - .build(); + OmPrefixInfo omPrefixInfo = getOmMetadataManager().getPrefixTable() + .getSkipCache(OzoneConsts.OM_KEY_PREFIX); + if (null == omPrefixInfo) { + return new ObjectDBInfo(); + } + return new ObjectDBInfo(omPrefixInfo); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java index 72c6bf66c96f..8c9ae62c0a9a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/VolumeEntityHandler.java @@ -67,23 +67,26 @@ public NamespaceSummaryResponse getSummaryResponse() CountStats countStats = new CountStats( -1, buckets.size(), totalDir, totalKey); - return - NamespaceSummaryResponse.newBuilder() - .setEntityType(EntityType.VOLUME) - .setCountStats(countStats) - .setObjectDBInfo(getVolumeObjDbInfo(names)) - .setStatus(ResponseStatus.OK) - .build(); + return NamespaceSummaryResponse.newBuilder() + .setEntityType(EntityType.VOLUME) + .setCountStats(countStats) + .setObjectDBInfo(getVolumeObjDbInfo(names)) + .setStatus(ResponseStatus.OK) + .build(); } private VolumeObjectDBInfo getVolumeObjDbInfo(String[] names) throws IOException { String dbVolumeKey = getOmMetadataManager().getVolumeKey(names[0]); + if (null == dbVolumeKey) { + return new VolumeObjectDBInfo(); + } OmVolumeArgs volumeArgs = getOmMetadataManager().getVolumeTable().getSkipCache(dbVolumeKey); - return VolumeObjectDBInfo.newBuilder() - .setVolumeArgs(volumeArgs) - .build(); + if (null == volumeArgs) { + return new VolumeObjectDBInfo(); + } + return new VolumeObjectDBInfo(volumeArgs); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java index d8f64cfeb9dc..ef2ae7dddfe9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; + /** * Encapsulates the low level bucket info. */ @@ -59,36 +60,28 @@ public class BucketObjectDBInfo extends ObjectDBInfo { @JsonProperty("owner") private String owner; - public static BucketObjectDBInfo.Builder newBuilder() { - return new BucketObjectDBInfo.Builder(); - } - public BucketObjectDBInfo() { } - public BucketObjectDBInfo(Builder b) { - this.setMetadata(b.getOmBucketInfo().getMetadata()); - this.setVolumeName(b.getOmBucketInfo().getVolumeName()); - this.setName(b.getOmBucketInfo().getBucketName()); - this.setQuotaInBytes(b.getOmBucketInfo().getQuotaInBytes()); - this.setQuotaInNamespace( - b.getOmBucketInfo().getQuotaInNamespace()); - this.setUsedNamespace(b.getOmBucketInfo().getUsedNamespace()); - this.setCreationTime(b.getOmBucketInfo().getCreationTime()); - this.setModificationTime( - b.getOmBucketInfo().getModificationTime()); - this.setAcls(b.getOmBucketInfo().getAcls()); - this.setSourceBucket(b.getOmBucketInfo().getSourceBucket()); - this.setSourceVolume(b.getOmBucketInfo().getSourceVolume()); - this.setBekInfo(b.getOmBucketInfo().getEncryptionKeyInfo()); - this.setVersioningEnabled( - b.getOmBucketInfo().getIsVersionEnabled()); - this.setStorageType(b.getOmBucketInfo().getStorageType()); - this.setDefaultReplicationConfig( - b.getOmBucketInfo().getDefaultReplicationConfig()); - this.setBucketLayout(b.getOmBucketInfo().getBucketLayout()); - this.setOwner(b.getOmBucketInfo().getOwner()); + public BucketObjectDBInfo(OmBucketInfo omBucketInfo) { + super.setMetadata(omBucketInfo.getMetadata()); + super.setName(omBucketInfo.getBucketName()); + super.setQuotaInBytes(omBucketInfo.getQuotaInBytes()); + super.setQuotaInNamespace(omBucketInfo.getQuotaInNamespace()); + super.setUsedNamespace(omBucketInfo.getUsedNamespace()); + super.setCreationTime(omBucketInfo.getCreationTime()); + super.setModificationTime(omBucketInfo.getModificationTime()); + super.setAcls(omBucketInfo.getAcls()); + this.volumeName = omBucketInfo.getVolumeName(); + this.sourceBucket = omBucketInfo.getSourceBucket(); + this.sourceVolume = omBucketInfo.getSourceVolume(); + this.isVersioningEnabled = omBucketInfo.getIsVersionEnabled(); + this.storageType = omBucketInfo.getStorageType(); + this.defaultReplicationConfig = omBucketInfo.getDefaultReplicationConfig(); + this.bucketLayout = omBucketInfo.getBucketLayout(); + this.owner = omBucketInfo.getOwner(); + this.bekInfo = omBucketInfo.getEncryptionKeyInfo(); } public String getVolumeName() { @@ -172,31 +165,4 @@ public void setOwner(String owner) { this.owner = owner; } - /** - * Builder for BucketObjectDBInfo. - */ - @SuppressWarnings("checkstyle:hiddenfield") - public static final class Builder { - private OmBucketInfo omBucketInfo; - public Builder() { - - } - - public BucketObjectDBInfo.Builder setOmBucketInfo( - OmBucketInfo omBucketInfo) { - this.omBucketInfo = omBucketInfo; - return this; - } - - public OmBucketInfo getOmBucketInfo() { - return omBucketInfo; - } - - public BucketObjectDBInfo build() { - if (null == this.omBucketInfo) { - return new BucketObjectDBInfo(); - } - return new BucketObjectDBInfo(this); - } - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java index 17e2286da834..f8cba210a598 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java @@ -58,29 +58,25 @@ public class KeyObjectDBInfo extends ObjectDBInfo { */ private String fileName; - public static KeyObjectDBInfo.Builder newBuilder() { - return new KeyObjectDBInfo.Builder(); - } - public KeyObjectDBInfo() { } - public KeyObjectDBInfo(Builder b) { - this.setVolumeName(b.getOmKeyInfo().getVolumeName()); - this.setBucketName(b.getOmKeyInfo().getBucketName()); - this.setName(b.getOmKeyInfo().getKeyName()); - this.setKeyName(b.getOmKeyInfo().getKeyName()); - this.setDataSize(b.getOmKeyInfo().getDataSize()); - this.setKeyLocationVersions(b.getOmKeyInfo().getKeyLocationVersions()); - this.setCreationTime(b.getOmKeyInfo().getCreationTime()); - this.setModificationTime(b.getOmKeyInfo().getModificationTime()); - this.setReplicationConfig(b.getOmKeyInfo().getReplicationConfig()); - this.setEncInfo(b.getOmKeyInfo().getFileEncryptionInfo()); - this.setFileName(b.getOmKeyInfo().getFileName()); - this.setFile(b.getOmKeyInfo().isFile()); - this.setAcls(b.getOmKeyInfo().getAcls()); - this.setMetadata(b.getOmKeyInfo().getMetadata()); + public KeyObjectDBInfo(OmKeyInfo omKeyInfo) { + super.setName(omKeyInfo.getKeyName()); + super.setCreationTime(omKeyInfo.getCreationTime()); + super.setModificationTime(omKeyInfo.getModificationTime()); + super.setAcls(omKeyInfo.getAcls()); + super.setMetadata(omKeyInfo.getMetadata()); + this.setVolumeName(omKeyInfo.getVolumeName()); + this.setBucketName(omKeyInfo.getBucketName()); + this.setKeyName(omKeyInfo.getKeyName()); + this.setDataSize(omKeyInfo.getDataSize()); + this.setKeyLocationVersions(omKeyInfo.getKeyLocationVersions()); + this.setReplicationConfig(omKeyInfo.getReplicationConfig()); + this.setEncInfo(omKeyInfo.getFileEncryptionInfo()); + this.setFileName(omKeyInfo.getFileName()); + this.setFile(omKeyInfo.isFile()); } public String getVolumeName() { @@ -155,32 +151,4 @@ public FileEncryptionInfo getEncInfo() { public void setEncInfo(FileEncryptionInfo encInfo) { this.encInfo = encInfo; } - - /** - * Builder for KeyObjectDBInfo. - */ - @SuppressWarnings("checkstyle:hiddenfield") - public static final class Builder { - private OmKeyInfo omKeyInfo; - public Builder() { - - } - - public KeyObjectDBInfo.Builder setOmKeyInfo( - OmKeyInfo omKeyInfo) { - this.omKeyInfo = omKeyInfo; - return this; - } - - public OmKeyInfo getOmKeyInfo() { - return omKeyInfo; - } - - public KeyObjectDBInfo build() { - if (null == this.omKeyInfo) { - return new KeyObjectDBInfo(); - } - return new KeyObjectDBInfo(this); - } - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java index 8d7d65269bbe..acd034a4869d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java @@ -27,7 +27,7 @@ import java.util.Map; /** - * Encapsulates the low level volume/bucket/dir info. + * Encapsulates the low level DB info common to volume or bucket or dir. */ public class ObjectDBInfo { @JsonProperty("metadata") @@ -54,37 +54,24 @@ public class ObjectDBInfo { @JsonProperty("acls") private List acls; - public static ObjectDBInfo.PrefixObjectDbInfoBuilder - newPrefixObjectDbInfoBuilder() { - return new ObjectDBInfo.PrefixObjectDbInfoBuilder(); - } + public ObjectDBInfo() { - public ObjectDBInfo(PrefixObjectDbInfoBuilder - prefixObjectDbInfoBuilder) { - this.setMetadata(prefixObjectDbInfoBuilder. - getOmPrefixInfo().getMetadata()); - this.setName(prefixObjectDbInfoBuilder. - getOmPrefixInfo().getName()); - this.setAcls(prefixObjectDbInfoBuilder. - getOmPrefixInfo().getAcls()); } - public static ObjectDBInfo.DirObjectDbInfoBuilder - newDirObjectDbInfoBuilder() { - return new ObjectDBInfo.DirObjectDbInfoBuilder(); + public ObjectDBInfo(OmDirectoryInfo omDirectoryInfo) { + this.setName(omDirectoryInfo.getName()); + this.setCreationTime(omDirectoryInfo.getCreationTime()); + this.setModificationTime(omDirectoryInfo.getModificationTime()); + this.setAcls(omDirectoryInfo.getAcls()); + this.setMetadata(omDirectoryInfo.getMetadata()); } - public ObjectDBInfo() { - + public ObjectDBInfo(OmPrefixInfo omPrefixInfo) { + this.setName(omPrefixInfo.getName()); + this.setAcls(omPrefixInfo.getAcls()); + this.setMetadata(omPrefixInfo.getMetadata()); } - public ObjectDBInfo(DirObjectDbInfoBuilder b) { - this.setName(b.getDirInfo().getName()); - this.setCreationTime(b.getDirInfo().getCreationTime()); - this.setModificationTime(b.getDirInfo().getModificationTime()); - this.setAcls(b.getDirInfo().getAcls()); - this.setMetadata(b.getDirInfo().getMetadata()); - } public Map getMetadata() { return metadata; @@ -150,61 +137,4 @@ public void setAcls(List acls) { this.acls = acls; } - /** - * Builder for Directory ObjectDBInfo. - */ - @SuppressWarnings("checkstyle:hiddenfield") - public static final class DirObjectDbInfoBuilder { - private OmDirectoryInfo dirInfo; - - public DirObjectDbInfoBuilder() { - - } - - public DirObjectDbInfoBuilder setDirInfo( - OmDirectoryInfo dirInfo) { - this.dirInfo = dirInfo; - return this; - } - - public OmDirectoryInfo getDirInfo() { - return dirInfo; - } - - public ObjectDBInfo build() { - if (null == this.dirInfo) { - return new ObjectDBInfo(); - } - return new ObjectDBInfo(this); - } - } - - /** - * Builder for Prefix ObjectDBInfo. - */ - @SuppressWarnings("checkstyle:hiddenfield") - public static final class PrefixObjectDbInfoBuilder { - private OmPrefixInfo omPrefixInfo; - - public PrefixObjectDbInfoBuilder() { - - } - - public PrefixObjectDbInfoBuilder setOmPrefixInfo( - OmPrefixInfo omPrefixInfo) { - this.omPrefixInfo = omPrefixInfo; - return this; - } - - public OmPrefixInfo getOmPrefixInfo() { - return omPrefixInfo; - } - - public ObjectDBInfo build() { - if (null == this.omPrefixInfo) { - return new ObjectDBInfo(); - } - return new ObjectDBInfo(this); - } - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java index cba539be5df7..7bb759ca4910 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java @@ -31,25 +31,25 @@ public class VolumeObjectDBInfo extends ObjectDBInfo { @JsonProperty("owner") private String owner; - public static VolumeObjectDBInfo.Builder newBuilder() { - return new VolumeObjectDBInfo.Builder(); - } + @JsonProperty("volume") + private String volume; public VolumeObjectDBInfo() { } - public VolumeObjectDBInfo(Builder b) { - this.setMetadata(b.getVolumeArgs().getMetadata()); - this.setName(b.getVolumeArgs().getVolume()); - this.setAdmin(b.getVolumeArgs().getAdminName()); - this.setOwner(b.getVolumeArgs().getOwnerName()); - this.setQuotaInBytes(b.getVolumeArgs().getQuotaInBytes()); - this.setQuotaInNamespace(b.getVolumeArgs().getQuotaInNamespace()); - this.setUsedNamespace(b.getVolumeArgs().getUsedNamespace()); - this.setCreationTime(b.getVolumeArgs().getCreationTime()); - this.setModificationTime(b.getVolumeArgs().getModificationTime()); - this.setAcls(b.getVolumeArgs().getAcls()); + public VolumeObjectDBInfo(OmVolumeArgs omVolumeArgs) { + super.setMetadata(omVolumeArgs.getMetadata()); + super.setName(omVolumeArgs.getVolume()); + super.setQuotaInBytes(omVolumeArgs.getQuotaInBytes()); + super.setQuotaInNamespace(omVolumeArgs.getQuotaInNamespace()); + super.setUsedNamespace(omVolumeArgs.getUsedNamespace()); + super.setCreationTime(omVolumeArgs.getCreationTime()); + super.setModificationTime(omVolumeArgs.getModificationTime()); + super.setAcls(omVolumeArgs.getAcls()); + this.setAdmin(omVolumeArgs.getAdminName()); + this.setOwner(omVolumeArgs.getOwnerName()); + this.setVolume(omVolumeArgs.getVolume()); } public String getAdmin() { @@ -68,31 +68,11 @@ public void setOwner(String owner) { this.owner = owner; } - /** - * Builder for VolumeObjectDBInfo. - */ - @SuppressWarnings("checkstyle:hiddenfield") - public static final class Builder { - private OmVolumeArgs volumeArgs; - public Builder() { - - } - - public VolumeObjectDBInfo.Builder setVolumeArgs( - OmVolumeArgs volumeArgs) { - this.volumeArgs = volumeArgs; - return this; - } - - public OmVolumeArgs getVolumeArgs() { - return volumeArgs; - } + public String getVolume() { + return volume; + } - public VolumeObjectDBInfo build() { - if (null == this.volumeArgs) { - return new VolumeObjectDBInfo(); - } - return new VolumeObjectDBInfo(this); - } + public void setVolume(String volume) { + this.volume = volume; } } From 4b07e59e3461af624bdcff118d6784931ed5eee4 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Thu, 22 Dec 2022 11:55:20 +0530 Subject: [PATCH 5/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../recon/schema/MetricsSchemaDefinition.java | 156 +++++++ .../ozone/recon/api/types/CountStats.java | 2 +- .../ozone/recon/api/types/VolumeMetrics.java | 99 +++++ .../EntityMetricsGenTaskDbEventHandler.java | 228 ++++++++++ .../tasks/EntityMetricsGenTaskWithFSO.java | 218 ++++++++++ .../tasks/EntityMetricsGenTaskWithLegacy.java | 310 ++++++++++++++ .../tasks/EntityMetricsGeneratorTask.java | 399 ++++++++++++++++++ 7 files changed, 1411 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java new file mode 100644 index 000000000000..ea3e5cedd25a --- /dev/null +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.hadoop.ozone.recon.schema; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; + +import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; + +/** + * Programmatic schema definition of Recon generated metrics + * at volume, bucket, key, file, directory level. + */ +@Singleton +public class MetricsSchemaDefinition implements ReconSchemaDefinition { + + private static final Logger LOG = + LoggerFactory.getLogger(MetricsSchemaDefinition.class); + + private final DataSource dataSource; + private DSLContext dslContext; + + public static final String VOLUME_METRICS_TABLE_NAME = + "VOLUME_METRICS"; + public static final String BUCKET_METRICS_TABLE_NAME = + "BUCKET_METRICS"; + public static final String KEY_METRICS_TABLE_NAME = + "KEY_METRICS"; + public static final String FILE_METRICS_TABLE_NAME = + "FILE_METRICS"; + public static final String DIR_METRICS_TABLE_NAME = + "DIR_METRICS"; + + @Inject + MetricsSchemaDefinition(DataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + @Transactional + public void initializeSchema() throws SQLException { + Connection conn = dataSource.getConnection(); + dslContext = DSL.using(conn); + if (!TABLE_EXISTS_CHECK.test(conn, VOLUME_METRICS_TABLE_NAME)) { + createVolumeMetricsTable(); + } + if (!TABLE_EXISTS_CHECK.test(conn, BUCKET_METRICS_TABLE_NAME)) { + createBucketMetricsTable(); + } + if (!TABLE_EXISTS_CHECK.test(conn, KEY_METRICS_TABLE_NAME)) { + createKeyMetricsTable(); + } + if (!TABLE_EXISTS_CHECK.test(conn, FILE_METRICS_TABLE_NAME)) { + createFileMetricsTable(); + } + if (!TABLE_EXISTS_CHECK.test(conn, DIR_METRICS_TABLE_NAME)) { + createDirMetricsTable(); + } + } + + private void createVolumeMetricsTable() { + dslContext.createTableIfNotExists(VOLUME_METRICS_TABLE_NAME) + .column("volume_objectid", SQLDataType.BIGINT.nullable(false)) + .column("bucket_count", SQLDataType.INTEGER.nullable(false)) + .column("access_count", SQLDataType.INTEGER.nullable(false)) + .column("size", SQLDataType.INTEGER.nullable(false)) + .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .constraint(DSL.constraint("pk_volume_objectid") + .primaryKey("volume_objectid")) + .execute(); + } + + private void createBucketMetricsTable() { + dslContext.createTableIfNotExists(BUCKET_METRICS_TABLE_NAME) + .column("bucket_objectid", SQLDataType.BIGINT.nullable(false)) + .column("key_count", SQLDataType.INTEGER.nullable(false)) + .column("access_count", SQLDataType.INTEGER.nullable(false)) + .column("size", SQLDataType.INTEGER.nullable(false)) + .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .constraint(DSL.constraint("pk_bucket_objectid") + .primaryKey("bucket_objectid")) + .execute(); + } + + private void createKeyMetricsTable() { + dslContext.createTableIfNotExists(KEY_METRICS_TABLE_NAME) + .column("key_objectid", SQLDataType.BIGINT.nullable(false)) + .column("access_count", SQLDataType.INTEGER.nullable(false)) + .column("size", SQLDataType.INTEGER.nullable(false)) + .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .constraint(DSL.constraint("pk_key_objectid") + .primaryKey("key_objectid")) + .execute(); + } + + private void createFileMetricsTable() { + dslContext.createTableIfNotExists(FILE_METRICS_TABLE_NAME) + .column("file_objectid", SQLDataType.BIGINT.nullable(false)) + .column("access_count", SQLDataType.INTEGER.nullable(false)) + .column("size", SQLDataType.INTEGER.nullable(false)) + .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .constraint(DSL.constraint("pk_file_objectid") + .primaryKey("file_objectid")) + .execute(); + } + + private void createDirMetricsTable() { + dslContext.createTableIfNotExists(DIR_METRICS_TABLE_NAME) + .column("dir_objectid", SQLDataType.BIGINT.nullable(false)) + .column("access_count", SQLDataType.INTEGER.nullable(false)) + .column("size", SQLDataType.INTEGER.nullable(false)) + .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) + .constraint(DSL.constraint("pk_dir_objectid") + .primaryKey("dir_objectid")) + .execute(); + } + + /** + * Returns the DSL context. + * + * @return dslContext + */ + public DSLContext getDSLContext() { + return dslContext; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java index 950ef5ff5886..1d88831d807e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java @@ -24,7 +24,7 @@ * volumes/buckets/dir/files etc. * */ public class CountStats { - /** Total number of volumes under root, -1 for other types. */ + /** Total number of volumes under root, -1 for other types. */ @JsonProperty("numVolume") private int numVolume; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java new file mode 100644 index 000000000000..bc6e8da6a9c2 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +/** + * Class to encapsulate volume level metrics from OM. + */ + +public final class VolumeMetrics { + private final long objectID; + private final long bucketCount; + private final long accessCount; + private final long size; + private final long creationTimeStamp; + private final long lastUpdatedTimeStamp; + + private VolumeMetrics() { + this(0L, 0L, 0L, 0L, + 0L, 0L); + } + + private VolumeMetrics(long objectID, + long bucketCount, + long accessCount, + long size, + long creationTimeStamp, + long lastUpdatedTimeStamp) { + this.objectID = objectID; + this.bucketCount = bucketCount; + this.accessCount = accessCount; + this.size = size; + this.creationTimeStamp = creationTimeStamp; + this.lastUpdatedTimeStamp = lastUpdatedTimeStamp; + } + + /** + * Builder of VolumeMetrics. + */ + public static class Builder { + + private long objectID; + private long bucketCount; + private long accessCount; + private long size; + private long creationTimeStamp; + private long lastUpdatedTimeStamp; + + public VolumeMetrics build() { + return new VolumeMetrics(objectID, bucketCount, + accessCount, size, creationTimeStamp, lastUpdatedTimeStamp); + } + + public Builder setObjectID(long objectID) { + this.objectID = objectID; + return this; + } + + public Builder setBucketCount(long bucketCount) { + this.bucketCount = bucketCount; + return this; + } + + public Builder setAccessCount(long accessCount) { + this.accessCount = accessCount; + return this; + } + + public Builder setSize(long size) { + this.size = size; + return this; + } + + public Builder setCreationTimeStamp(long creationTimeStamp) { + this.creationTimeStamp = creationTimeStamp; + return this; + } + + public Builder setLastUpdatedTimeStamp(long lastUpdatedTimeStamp) { + this.lastUpdatedTimeStamp = lastUpdatedTimeStamp; + return this; + } + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java new file mode 100644 index 000000000000..648f6811007c --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT; + +/** + * Class for holding all NSSummaryTask methods + * related to DB operations so that they can commonly be + * used in NSSummaryTaskWithFSO and NSSummaryTaskWithLegacy. + */ +public class NSSummaryTaskDbEventHandler { + + private static final Logger LOG = + LoggerFactory.getLogger(NSSummaryTaskDbEventHandler.class); + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private ReconOMMetadataManager reconOMMetadataManager; + + private final long nsSummaryFlushToDBMaxThreshold; + + public NSSummaryTaskDbEventHandler(ReconNamespaceSummaryManager + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager, + OzoneConfiguration + ozoneConfiguration) { + this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; + this.reconOMMetadataManager = reconOMMetadataManager; + nsSummaryFlushToDBMaxThreshold = ozoneConfiguration.getLong( + OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, + OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT); + } + + public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { + return reconNamespaceSummaryManager; + } + + public ReconOMMetadataManager getReconOMMetadataManager() { + return reconOMMetadataManager; + } + + protected void writeNSSummariesToDB(Map nsSummaryMap) + throws IOException { + try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { + nsSummaryMap.keySet().forEach((Long key) -> { + try { + reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, + key, nsSummaryMap.get(key)); + } catch (IOException e) { + LOG.error("Unable to write Namespace Summary data in Recon DB.", + e); + } + }); + reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); + } + } + + protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map nsSummaryMap) throws IOException { + long parentObjectId = keyInfo.getParentObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + if (nsSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + nsSummary = new NSSummary(); + } + int numOfFile = nsSummary.getNumOfFiles(); + long sizeOfFile = nsSummary.getSizeOfFiles(); + int[] fileBucket = nsSummary.getFileSizeBucket(); + nsSummary.setNumOfFiles(numOfFile + 1); + long dataSize = keyInfo.getDataSize(); + nsSummary.setSizeOfFiles(sizeOfFile + dataSize); + int binIndex = ReconUtils.getBinIndex(dataSize); + + ++fileBucket[binIndex]; + nsSummary.setFileSizeBucket(fileBucket); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = directoryInfo.getParentObjectID(); + long objectId = directoryInfo.getObjectID(); + // write the dir name to the current directory + String dirName = directoryInfo.getName(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary curNSSummary = nsSummaryMap.get(objectId); + if (curNSSummary == null) { + // If we don't have it in this batch we try to get it from the DB + curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId); + } + if (curNSSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + curNSSummary = new NSSummary(); + } + curNSSummary.setDirName(dirName); + nsSummaryMap.put(objectId, curNSSummary); + + // Write the child dir list to the parent directory + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + if (nsSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + nsSummary = new NSSummary(); + } + nsSummary.addChildDir(objectId); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handleDeleteKeyEvent(OmKeyInfo keyInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = keyInfo.getParentObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + + // Just in case the OmKeyInfo isn't correctly written. + if (nsSummary == null) { + LOG.error("The namespace table is not correctly populated."); + return; + } + int numOfFile = nsSummary.getNumOfFiles(); + long sizeOfFile = nsSummary.getSizeOfFiles(); + int[] fileBucket = nsSummary.getFileSizeBucket(); + + long dataSize = keyInfo.getDataSize(); + int binIndex = ReconUtils.getBinIndex(dataSize); + + // decrement count, data size, and bucket count + // even if there's no direct key, we still keep the entry because + // we still need children dir IDs info + nsSummary.setNumOfFiles(numOfFile - 1); + nsSummary.setSizeOfFiles(sizeOfFile - dataSize); + --fileBucket[binIndex]; + nsSummary.setFileSizeBucket(fileBucket); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = directoryInfo.getParentObjectID(); + long objectId = directoryInfo.getObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + + // Just in case the OmDirectoryInfo isn't correctly written. + if (nsSummary == null) { + LOG.error("The namespace table is not correctly populated."); + return; + } + + nsSummary.removeChildDir(objectId); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected boolean flushAndCommitNSToDB(Map nsSummaryMap) { + try { + writeNSSummariesToDB(nsSummaryMap); + nsSummaryMap.clear(); + } catch (IOException e) { + LOG.error("Unable to write Namespace Summary data in Recon DB.", e); + return false; + } + return true; + } + + protected boolean checkAndCallFlushToDB( + Map nsSummaryMap) { + // if map contains more than entries, flush to DB and clear the map + if (null != nsSummaryMap && nsSummaryMap.size() >= + nsSummaryFlushToDBMaxThreshold) { + return flushAndCommitNSToDB(nsSummaryMap); + } + return true; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java new file mode 100644 index 000000000000..9806b198b255 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; + +/** + * Class for handling FSO specific tasks. + */ +public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler { + + private static final Logger LOG = + LoggerFactory.getLogger(NSSummaryTaskWithFSO.class); + + public NSSummaryTaskWithFSO(ReconNamespaceSummaryManager + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager, + OzoneConfiguration + ozoneConfiguration) { + super(reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); + } + + // We only listen to updates from FSO-enabled KeyTable(FileTable) and DirTable + public Collection getTaskTables() { + return Arrays.asList(FILE_TABLE, DIRECTORY_TABLE); + } + + public boolean processWithFSO(OMUpdateEventBatch events) { + Iterator eventIterator = events.getIterator(); + final Collection taskTables = getTaskTables(); + Map nsSummaryMap = new HashMap<>(); + + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); + + // we only process updates on OM's FileTable and Dirtable + String table = omdbUpdateEvent.getTable(); + boolean updateOnFileTable = table.equals(FILE_TABLE); + if (!taskTables.contains(table)) { + continue; + } + + String updatedKey = omdbUpdateEvent.getKey(); + + try { + if (updateOnFileTable) { + // key update on fileTable + OMDBUpdateEvent keyTableUpdateEvent = + (OMDBUpdateEvent) omdbUpdateEvent; + OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue(); + OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue(); + + switch (action) { + case PUT: + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldKeyInfo != null) { + // delete first, then put + handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old keyInfo for {}.", + updatedKey); + } + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + + } else { + // directory update on DirTable + OMDBUpdateEvent dirTableUpdateEvent = + (OMDBUpdateEvent) omdbUpdateEvent; + OmDirectoryInfo updatedDirectoryInfo = dirTableUpdateEvent.getValue(); + OmDirectoryInfo oldDirectoryInfo = dirTableUpdateEvent.getOldValue(); + + switch (action) { + case PUT: + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldDirectoryInfo != null) { + // delete first, then put + handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old dirInfo for {}.", + updatedKey); + } + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to process Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + if (!checkAndCallFlushToDB(nsSummaryMap)) { + return false; + } + } + + // flush and commit left out entries at end + if (!flushAndCommitNSToDB(nsSummaryMap)) { + return false; + } + + LOG.info("Completed a process run of NSSummaryTaskWithFSO"); + return true; + } + + public boolean reprocessWithFSO(OMMetadataManager omMetadataManager) { + Map nsSummaryMap = new HashMap<>(); + + try { + Table dirTable = + omMetadataManager.getDirectoryTable(); + try (TableIterator> + dirTableIter = dirTable.iterator()) { + while (dirTableIter.hasNext()) { + Table.KeyValue kv = dirTableIter.next(); + OmDirectoryInfo directoryInfo = kv.getValue(); + handlePutDirEvent(directoryInfo, nsSummaryMap); + if (!checkAndCallFlushToDB(nsSummaryMap)) { + return false; + } + } + } + + // Get fileTable used by FSO + Table keyTable = + omMetadataManager.getFileTable(); + + try (TableIterator> + keyTableIter = keyTable.iterator()) { + while (keyTableIter.hasNext()) { + Table.KeyValue kv = keyTableIter.next(); + OmKeyInfo keyInfo = kv.getValue(); + handlePutKeyEvent(keyInfo, nsSummaryMap); + if (!checkAndCallFlushToDB(nsSummaryMap)) { + return false; + } + } + } + + } catch (IOException ioEx) { + LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + // flush and commit left out keys at end + if (!flushAndCommitNSToDB(nsSummaryMap)) { + return false; + } + LOG.info("Completed a reprocess run of NSSummaryTaskWithFSO"); + return true; + } + + +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java new file mode 100644 index 000000000000..a2a9598261e5 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; + +/** + * Class for handling Legacy specific tasks. + */ +public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler { + + private static final BucketLayout BUCKET_LAYOUT = BucketLayout.LEGACY; + + private static final Logger LOG = + LoggerFactory.getLogger(NSSummaryTaskWithLegacy.class); + + private boolean enableFileSystemPaths; + + public NSSummaryTaskWithLegacy(ReconNamespaceSummaryManager + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager, + OzoneConfiguration + ozoneConfiguration) { + super(reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); + // true if FileSystemPaths enabled + enableFileSystemPaths = ozoneConfiguration + .getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); + } + + public boolean processWithLegacy(OMUpdateEventBatch events) { + Iterator eventIterator = events.getIterator(); + Map nsSummaryMap = new HashMap<>(); + + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); + + // we only process updates on OM's KeyTable + String table = omdbUpdateEvent.getTable(); + boolean updateOnKeyTable = table.equals(KEY_TABLE); + if (!updateOnKeyTable) { + continue; + } + + String updatedKey = omdbUpdateEvent.getKey(); + + try { + OMDBUpdateEvent keyTableUpdateEvent = + (OMDBUpdateEvent) omdbUpdateEvent; + OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue(); + OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue(); + + // KeyTable entries belong to both Legacy and OBS buckets. + // Check bucket layout and if it's OBS + // continue to the next iteration. + // Check just for the current KeyInfo. + String volumeName = updatedKeyInfo.getVolumeName(); + String bucketName = updatedKeyInfo.getBucketName(); + String bucketDBKey = getReconOMMetadataManager() + .getBucketKey(volumeName, bucketName); + // Get bucket info from bucket table + OmBucketInfo omBucketInfo = getReconOMMetadataManager() + .getBucketTable().getSkipCache(bucketDBKey); + + if (omBucketInfo.getBucketLayout() + .isObjectStore(enableFileSystemPaths)) { + continue; + } + + setKeyParentID(updatedKeyInfo); + + if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + switch (action) { + case PUT: + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldKeyInfo != null) { + // delete first, then put + setKeyParentID(oldKeyInfo); + handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old keyInfo for {}.", + updatedKey); + } + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } else { + OmDirectoryInfo updatedDirectoryInfo = + new OmDirectoryInfo.Builder() + .setName(updatedKeyInfo.getKeyName()) + .setObjectID(updatedKeyInfo.getObjectID()) + .setParentObjectID(updatedKeyInfo.getParentObjectID()) + .build(); + + OmDirectoryInfo oldDirectoryInfo = null; + + if (oldKeyInfo != null) { + oldDirectoryInfo = + new OmDirectoryInfo.Builder() + .setName(oldKeyInfo.getKeyName()) + .setObjectID(oldKeyInfo.getObjectID()) + .setParentObjectID(oldKeyInfo.getParentObjectID()) + .build(); + } + + switch (action) { + case PUT: + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldDirectoryInfo != null) { + // delete first, then put + handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old dirInfo for {}.", + updatedKey); + } + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to process Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + if (!checkAndCallFlushToDB(nsSummaryMap)) { + return false; + } + } + + // flush and commit left out entries at end + if (!flushAndCommitNSToDB(nsSummaryMap)) { + return false; + } + + LOG.info("Completed a process run of NSSummaryTaskWithLegacy"); + return true; + } + + public boolean reprocessWithLegacy(OMMetadataManager omMetadataManager) { + Map nsSummaryMap = new HashMap<>(); + + try { + Table keyTable = + omMetadataManager.getKeyTable(BUCKET_LAYOUT); + + try (TableIterator> + keyTableIter = keyTable.iterator()) { + + while (keyTableIter.hasNext()) { + Table.KeyValue kv = keyTableIter.next(); + OmKeyInfo keyInfo = kv.getValue(); + + // KeyTable entries belong to both Legacy and OBS buckets. + // Check bucket layout and if it's OBS + // continue to the next iteration. + String volumeName = keyInfo.getVolumeName(); + String bucketName = keyInfo.getBucketName(); + String bucketDBKey = omMetadataManager + .getBucketKey(volumeName, bucketName); + // Get bucket info from bucket table + OmBucketInfo omBucketInfo = omMetadataManager + .getBucketTable().getSkipCache(bucketDBKey); + + if (omBucketInfo.getBucketLayout() + .isObjectStore(enableFileSystemPaths)) { + continue; + } + + setKeyParentID(keyInfo); + + if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + OmDirectoryInfo directoryInfo = + new OmDirectoryInfo.Builder() + .setName(keyInfo.getKeyName()) + .setObjectID(keyInfo.getObjectID()) + .setParentObjectID(keyInfo.getParentObjectID()) + .build(); + handlePutDirEvent(directoryInfo, nsSummaryMap); + } else { + handlePutKeyEvent(keyInfo, nsSummaryMap); + } + if (!checkAndCallFlushToDB(nsSummaryMap)) { + return false; + } + } + } + } catch (IOException ioEx) { + LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + + // flush and commit left out entries at end + if (!flushAndCommitNSToDB(nsSummaryMap)) { + return false; + } + LOG.info("Completed a reprocess run of NSSummaryTaskWithLegacy"); + return true; + } + + /** + * KeyTable entries don't have the parentId set. + * In order to reuse the existing FSO methods that rely on + * the parentId, we have to set it explicitly. + * @param keyInfo + * @throws IOException + */ + private void setKeyParentID(OmKeyInfo keyInfo) throws IOException { + String[] keyPath = keyInfo.getKeyName().split(OM_KEY_PREFIX); + + // If the path contains only one key then keyPath.length + // will be 1 and the parent will be a bucket. + // If the keyPath.length is greater than 1 then + // there is at least one directory. + if (keyPath.length > 1) { + String[] dirs = Arrays.copyOf(keyPath, keyPath.length - 1); + String parentKeyName = String.join(OM_KEY_PREFIX, dirs); + parentKeyName += OM_KEY_PREFIX; + String fullParentKeyName = + getReconOMMetadataManager().getOzoneKey(keyInfo.getVolumeName(), + keyInfo.getBucketName(), parentKeyName); + OmKeyInfo parentKeyInfo = getReconOMMetadataManager() + .getKeyTable(BUCKET_LAYOUT) + .getSkipCache(fullParentKeyName); + + if (parentKeyInfo != null) { + keyInfo.setParentObjectID(parentKeyInfo.getObjectID()); + } else { + throw new IOException("ParentKeyInfo for " + + "NSSummaryTaskWithLegacy is null"); + } + } else { + String bucketKey = getReconOMMetadataManager() + .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName()); + OmBucketInfo parentBucketInfo = + getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey); + + if (parentBucketInfo != null) { + keyInfo.setParentObjectID(parentBucketInfo.getObjectID()); + } else { + throw new IOException("ParentKeyInfo for " + + "NSSummaryTaskWithLegacy is null"); + } + } + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java new file mode 100644 index 000000000000..d1ef0bac1960 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import com.google.inject.Inject; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.WithObjectID; +import org.hadoop.ozone.recon.schema.MetricsSchemaDefinition; +import org.hadoop.ozone.recon.schema.tables.daos.VolumeMetricsDao; +import org.hadoop.ozone.recon.schema.tables.pojos.VolumeMetrics; +import org.jooq.DSLContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.VOLUME_TABLE; +import static org.hadoop.ozone.recon.schema.tables.VolumeMetricsTable.VOLUME_METRICS; + +/** + * Class to iterate over the OM DB and populate the Recon metrics DB with + * the metrics at volume, bucket, key, file and directory level. + */ +public class EntityMetricsGeneratorTask implements ReconOmTask { + + private static final Logger LOG = + LoggerFactory.getLogger(EntityMetricsGeneratorTask.class); + + private VolumeMetricsDao volumeMetricsDao; + private DSLContext dslContext; + + @Inject + public EntityMetricsGeneratorTask(VolumeMetricsDao volumeMetricsDao, + MetricsSchemaDefinition + metricsSchemaDefinition) { + this.volumeMetricsDao = volumeMetricsDao; + this.dslContext = metricsSchemaDefinition.getDSLContext(); + } + + /** + * Read the Keys from OM snapshot DB and calculate the upper bound of + * File Size it belongs to. + * + * @param omMetadataManager OM Metadata instance. + * @return Pair + */ + @Override + public Pair reprocess(OMMetadataManager omMetadataManager) { + Map volumeMetricsMap = new HashMap<>(); + Table omVolumeArgsTable = + omMetadataManager.getVolumeTable(); + try (TableIterator> + keyIter = omVolumeArgsTable.iterator()) { + while (keyIter.hasNext()) { + VolumeMetrics volumeMetrics = new VolumeMetrics(); + Table.KeyValue kv = keyIter.next(); + OmVolumeArgs value = kv.getValue(); + volumeMetrics.setVolumeObjectid(value.getObjectID()); + List buckets = listBucketsUnderVolume(omMetadataManager, + value.getVolume()); + volumeMetrics.setBucketCount(buckets.size()); + volumeMetrics.setSize(getTotalSizesUnderVolume( + omMetadataManager, value.getVolume())); + volumeMetrics.setCreationTimestamp(new Timestamp( + value.getCreationTime())); + volumeMetrics.setLastUpdatedTimestamp(new Timestamp( + value.getModificationTime())); + volumeMetricsMap.put(value.getObjectID(), volumeMetrics); + } + } catch (IOException ioEx) { + LOG.error("Unable to populate File Size Count in Recon DB. ", ioEx); + return new ImmutablePair<>(getTaskName(), false); + } + // Truncate table before inserting new rows + int execute = dslContext.delete(VOLUME_METRICS).execute(); + LOG.info("Deleted {} records from {}", execute, VOLUME_METRICS); + + updateVolumeMetricsToDB(true, volumeMetricsMap); + + LOG.info("Completed a 'reprocess' run of FileSizeCountTask."); + return new ImmutablePair<>(getTaskName(), true); + } + + @Override + public String getTaskName() { + return "EntityMetricsGeneratorTask"; + } + + public Collection getTaskTables() { + return Arrays.asList(VOLUME_TABLE, BUCKET_TABLE, KEY_TABLE, + FILE_TABLE, DIRECTORY_TABLE); + } + + /** + * Read the Keys from update events and update the count of files + * pertaining to a certain upper bound. + * + * @param events Update events - PUT/DELETE. + * @return Pair + */ + @Override + public Pair process(OMUpdateEventBatch events) { + Iterator eventIterator = events.getIterator(); + Map volumeMetricsMap = new HashMap<>(); + final Collection taskTables = getTaskTables(); + + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); + // Filter event inside process method to avoid duping + String table = omdbUpdateEvent.getTable(); + if (!taskTables.contains(table)) { + continue; + } + String updatedKey = omdbUpdateEvent.getKey(); + + try { + if (table.equals(VOLUME_TABLE)) { + OMDBUpdateEvent volTableUpdateEvent = + (OMDBUpdateEvent) omdbUpdateEvent; + OmVolumeArgs updatedVolInfo = volTableUpdateEvent.getValue(); + OmVolumeArgs oldVolInfo = volTableUpdateEvent.getOldValue(); + + switch (action) { + case PUT: + handlePutVolEvent(updatedVolInfo, volumeMetricsMap); + break; + + case DELETE: + handleDeleteVolEvent(updatedVolInfo, volumeMetricsMap); + break; + + case UPDATE: + //handleDeleteKeyEvent(updatedKey, omdbUpdateEvent.getOldValue(), + //fileSizeCountMap); + //handlePutKeyEvent(omKeyInfo, fileSizeCountMap); + break; + + default: + LOG.trace("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } + } catch (Exception e) { + LOG.error("Unexpected exception while processing key {}.", + updatedKey, e); + return new ImmutablePair<>(getTaskName(), false); + } + } + updateVolumeMetricsToDB(false, volumeMetricsMap); + LOG.info("Completed a 'process' run of FileSizeCountTask."); + return new ImmutablePair<>(getTaskName(), true); + } + + private void handleDeleteVolEvent(OmVolumeArgs updatedVolInfo, + Map volumeMetricsMap) { + VolumeMetrics volumeMetrics = volumeMetricsMap.get( + updatedVolInfo.getObjectID()); + if (volumeMetrics == null) { + // If we don't have it in this batch we try to get it from the DB + volumeMetrics = volumeMetricsDao.findById(updatedVolInfo.getObjectID()); + } + if (volumeMetrics == null) { + return; + } + volumeMetricsMap.remove(updatedVolInfo.getObjectID()); + } + + + /** + * Populate DB with the counts of file sizes calculated + * using the dao. + * + */ + private void updateVolumeMetricsToDB(boolean isDbTruncated, + Map + volumeMetricsMap) { + List insertToDb = new ArrayList<>(); + List updateInDb = new ArrayList<>(); + + volumeMetricsMap.keySet().forEach((Long key) -> { + VolumeMetrics volumeMetrics = volumeMetricsMap.get(key); + VolumeMetrics newRecord = new VolumeMetrics(); + newRecord.setVolumeObjectid(volumeMetrics.getVolumeObjectid()); + newRecord.setBucketCount(volumeMetrics.getBucketCount()); + newRecord.setAccessCount(volumeMetrics.getAccessCount()); + newRecord.setSize(volumeMetrics.getSize()); + newRecord.setCreationTimestamp(volumeMetrics.getCreationTimestamp()); + newRecord.setLastUpdatedTimestamp( + volumeMetrics.getLastUpdatedTimestamp()); + if (!isDbTruncated) { + // Get the current count from database and update + VolumeMetrics volMetrics = volumeMetricsDao.findById( + volumeMetrics.getVolumeObjectid()); + if (volMetrics == null && + (newRecord.getBucketCount() > 0 || + newRecord.getAccessCount() > 0 || + newRecord.getSize() > 0)) { + // insert new row only for non-zero counts. + insertToDb.add(newRecord); + } else if (volMetrics != null) { + newRecord.setBucketCount(volMetrics.getBucketCount() + + volumeMetrics.getBucketCount()); + newRecord.setAccessCount(volMetrics.getAccessCount() + + volumeMetrics.getAccessCount()); + newRecord.setSize(volMetrics.getSize() + + volumeMetrics.getSize()); + newRecord.setLastUpdatedTimestamp( + volumeMetrics.getLastUpdatedTimestamp()); + updateInDb.add(newRecord); + } + } else if (newRecord.getBucketCount() > 0 || + newRecord.getAccessCount() > 0 || + newRecord.getSize() > 0) { + // insert new row only for non-zero counts. + insertToDb.add(newRecord); + } + }); + volumeMetricsDao.insert(insertToDb); + volumeMetricsDao.update(updateInDb); + } + + private BucketLayout getBucketLayout() { + return BucketLayout.DEFAULT; + } + + private void handlePutVolEvent(OmVolumeArgs updatedVolInfo, + Map volumeMetricsMap) { + VolumeMetrics volumeMetrics = volumeMetricsMap.get( + updatedVolInfo.getObjectID()); + if (volumeMetrics == null) { + // If we don't have it in this batch we try to get it from the DB + volumeMetrics = volumeMetricsDao.findById(updatedVolInfo.getObjectID()); + } + if (volumeMetrics == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + volumeMetrics = new VolumeMetrics(); + } + volumeMetrics.setVolumeObjectid(updatedVolInfo.getObjectID()); + volumeMetrics.setBucketCount(0); + volumeMetrics.setSize(0); + volumeMetrics.setCreationTimestamp(new Timestamp( + updatedVolInfo.getCreationTime())); + volumeMetrics.setLastUpdatedTimestamp(new Timestamp( + updatedVolInfo.getModificationTime())); + volumeMetricsMap.put(updatedVolInfo.getObjectID(), volumeMetrics); + } + + /** + * Calculate and update the count of files being tracked by + * fileSizeCountMap. + * Used by reprocess() and process(). + * + * @param omKeyInfo OmKey being updated for count + */ + /*private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, + Map + fileSizeCountMap) { + if (omKeyInfo == null) { + LOG.warn("Deleting a key not found while handling DELETE key event. Key" + + " not found in Recon OM DB : {}", key); + } else { + FileSizeCountTask.FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); + Long count = fileSizeCountMap.containsKey(countKey) ? + fileSizeCountMap.get(countKey) - 1L : -1L; + fileSizeCountMap.put(countKey, count); + } + }*/ + + /** + * List all buckets under a volume, if volume name is null, return all + * buckets under the system. + * + * @param omMetadataManager + * @param volumeName volume name + * @return a list of buckets + * @throws IOException IOE + */ + private List listBucketsUnderVolume( + OMMetadataManager omMetadataManager, final String volumeName) + throws IOException { + List result = new ArrayList<>(); + // if volume name is null, seek prefix is an empty string + String seekPrefix = ""; + + Table bucketTable = + omMetadataManager.getBucketTable(); + + try (TableIterator> + iterator = bucketTable.iterator()) { + + if (volumeName != null) { + if (!volumeExists(omMetadataManager, volumeName)) { + return result; + } + seekPrefix = omMetadataManager.getVolumeKey(volumeName + OM_KEY_PREFIX); + } + + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + + String key = kv.getKey(); + OmBucketInfo omBucketInfo = kv.getValue(); + + if (omBucketInfo != null) { + // We should return only the keys, whose keys match with + // the seek prefix + if (key.startsWith(seekPrefix)) { + result.add(omBucketInfo); + } + } + } + } + return result; + } + + private Integer getTotalSizesUnderVolume( + OMMetadataManager omMetadataManager, final String volumeName) + throws IOException { + int totalSize = 0; + // if volume name is null, seek prefix is an empty string + String seekPrefix = ""; + + Table bucketTable = + omMetadataManager.getBucketTable(); + + try (TableIterator> + iterator = bucketTable.iterator()) { + + if (volumeName != null) { + if (!volumeExists(omMetadataManager, volumeName)) { + return Integer.valueOf(totalSize); + } + seekPrefix = omMetadataManager.getVolumeKey(volumeName + OM_KEY_PREFIX); + } + + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + + String key = kv.getKey(); + OmBucketInfo omBucketInfo = kv.getValue(); + + if (omBucketInfo != null) { + // We should return only the keys, whose keys match with + // the seek prefix + if (key.startsWith(seekPrefix)) { + totalSize += omBucketInfo.getUsedBytes(); + } + } + } + } + return Integer.valueOf(totalSize); + } + + private boolean volumeExists(OMMetadataManager omMetadataManager, + String volName) throws IOException { + String volDBKey = omMetadataManager.getVolumeKey(volName); + return omMetadataManager.getVolumeTable().getSkipCache(volDBKey) != null; + } + +} From 50d7eb5d3896bfa1b67621a3dfe6d80150343c4c Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Thu, 22 Dec 2022 12:08:50 +0530 Subject: [PATCH 6/8] Revert "HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well." This reverts commit 4b07e59e3461af624bdcff118d6784931ed5eee4. --- .../recon/schema/MetricsSchemaDefinition.java | 156 ------- .../ozone/recon/api/types/CountStats.java | 2 +- .../ozone/recon/api/types/VolumeMetrics.java | 99 ----- .../EntityMetricsGenTaskDbEventHandler.java | 228 ---------- .../tasks/EntityMetricsGenTaskWithFSO.java | 218 ---------- .../tasks/EntityMetricsGenTaskWithLegacy.java | 310 -------------- .../tasks/EntityMetricsGeneratorTask.java | 399 ------------------ 7 files changed, 1 insertion(+), 1411 deletions(-) delete mode 100644 hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java deleted file mode 100644 index ea3e5cedd25a..000000000000 --- a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/MetricsSchemaDefinition.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.hadoop.ozone.recon.schema; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import org.jooq.DSLContext; -import org.jooq.impl.DSL; -import org.jooq.impl.SQLDataType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.transaction.annotation.Transactional; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.SQLException; - -import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK; - -/** - * Programmatic schema definition of Recon generated metrics - * at volume, bucket, key, file, directory level. - */ -@Singleton -public class MetricsSchemaDefinition implements ReconSchemaDefinition { - - private static final Logger LOG = - LoggerFactory.getLogger(MetricsSchemaDefinition.class); - - private final DataSource dataSource; - private DSLContext dslContext; - - public static final String VOLUME_METRICS_TABLE_NAME = - "VOLUME_METRICS"; - public static final String BUCKET_METRICS_TABLE_NAME = - "BUCKET_METRICS"; - public static final String KEY_METRICS_TABLE_NAME = - "KEY_METRICS"; - public static final String FILE_METRICS_TABLE_NAME = - "FILE_METRICS"; - public static final String DIR_METRICS_TABLE_NAME = - "DIR_METRICS"; - - @Inject - MetricsSchemaDefinition(DataSource dataSource) { - this.dataSource = dataSource; - } - - @Override - @Transactional - public void initializeSchema() throws SQLException { - Connection conn = dataSource.getConnection(); - dslContext = DSL.using(conn); - if (!TABLE_EXISTS_CHECK.test(conn, VOLUME_METRICS_TABLE_NAME)) { - createVolumeMetricsTable(); - } - if (!TABLE_EXISTS_CHECK.test(conn, BUCKET_METRICS_TABLE_NAME)) { - createBucketMetricsTable(); - } - if (!TABLE_EXISTS_CHECK.test(conn, KEY_METRICS_TABLE_NAME)) { - createKeyMetricsTable(); - } - if (!TABLE_EXISTS_CHECK.test(conn, FILE_METRICS_TABLE_NAME)) { - createFileMetricsTable(); - } - if (!TABLE_EXISTS_CHECK.test(conn, DIR_METRICS_TABLE_NAME)) { - createDirMetricsTable(); - } - } - - private void createVolumeMetricsTable() { - dslContext.createTableIfNotExists(VOLUME_METRICS_TABLE_NAME) - .column("volume_objectid", SQLDataType.BIGINT.nullable(false)) - .column("bucket_count", SQLDataType.INTEGER.nullable(false)) - .column("access_count", SQLDataType.INTEGER.nullable(false)) - .column("size", SQLDataType.INTEGER.nullable(false)) - .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .constraint(DSL.constraint("pk_volume_objectid") - .primaryKey("volume_objectid")) - .execute(); - } - - private void createBucketMetricsTable() { - dslContext.createTableIfNotExists(BUCKET_METRICS_TABLE_NAME) - .column("bucket_objectid", SQLDataType.BIGINT.nullable(false)) - .column("key_count", SQLDataType.INTEGER.nullable(false)) - .column("access_count", SQLDataType.INTEGER.nullable(false)) - .column("size", SQLDataType.INTEGER.nullable(false)) - .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .constraint(DSL.constraint("pk_bucket_objectid") - .primaryKey("bucket_objectid")) - .execute(); - } - - private void createKeyMetricsTable() { - dslContext.createTableIfNotExists(KEY_METRICS_TABLE_NAME) - .column("key_objectid", SQLDataType.BIGINT.nullable(false)) - .column("access_count", SQLDataType.INTEGER.nullable(false)) - .column("size", SQLDataType.INTEGER.nullable(false)) - .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .constraint(DSL.constraint("pk_key_objectid") - .primaryKey("key_objectid")) - .execute(); - } - - private void createFileMetricsTable() { - dslContext.createTableIfNotExists(FILE_METRICS_TABLE_NAME) - .column("file_objectid", SQLDataType.BIGINT.nullable(false)) - .column("access_count", SQLDataType.INTEGER.nullable(false)) - .column("size", SQLDataType.INTEGER.nullable(false)) - .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .constraint(DSL.constraint("pk_file_objectid") - .primaryKey("file_objectid")) - .execute(); - } - - private void createDirMetricsTable() { - dslContext.createTableIfNotExists(DIR_METRICS_TABLE_NAME) - .column("dir_objectid", SQLDataType.BIGINT.nullable(false)) - .column("access_count", SQLDataType.INTEGER.nullable(false)) - .column("size", SQLDataType.INTEGER.nullable(false)) - .column("creation_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .column("last_updated_timestamp", SQLDataType.TIMESTAMP.nullable(false)) - .constraint(DSL.constraint("pk_dir_objectid") - .primaryKey("dir_objectid")) - .execute(); - } - - /** - * Returns the DSL context. - * - * @return dslContext - */ - public DSLContext getDSLContext() { - return dslContext; - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java index 1d88831d807e..950ef5ff5886 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java @@ -24,7 +24,7 @@ * volumes/buckets/dir/files etc. * */ public class CountStats { - /** Total number of volumes under root, -1 for other types. */ + /** Total number of volumes under root, -1 for other types. */ @JsonProperty("numVolume") private int numVolume; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java deleted file mode 100644 index bc6e8da6a9c2..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeMetrics.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api.types; - -/** - * Class to encapsulate volume level metrics from OM. - */ - -public final class VolumeMetrics { - private final long objectID; - private final long bucketCount; - private final long accessCount; - private final long size; - private final long creationTimeStamp; - private final long lastUpdatedTimeStamp; - - private VolumeMetrics() { - this(0L, 0L, 0L, 0L, - 0L, 0L); - } - - private VolumeMetrics(long objectID, - long bucketCount, - long accessCount, - long size, - long creationTimeStamp, - long lastUpdatedTimeStamp) { - this.objectID = objectID; - this.bucketCount = bucketCount; - this.accessCount = accessCount; - this.size = size; - this.creationTimeStamp = creationTimeStamp; - this.lastUpdatedTimeStamp = lastUpdatedTimeStamp; - } - - /** - * Builder of VolumeMetrics. - */ - public static class Builder { - - private long objectID; - private long bucketCount; - private long accessCount; - private long size; - private long creationTimeStamp; - private long lastUpdatedTimeStamp; - - public VolumeMetrics build() { - return new VolumeMetrics(objectID, bucketCount, - accessCount, size, creationTimeStamp, lastUpdatedTimeStamp); - } - - public Builder setObjectID(long objectID) { - this.objectID = objectID; - return this; - } - - public Builder setBucketCount(long bucketCount) { - this.bucketCount = bucketCount; - return this; - } - - public Builder setAccessCount(long accessCount) { - this.accessCount = accessCount; - return this; - } - - public Builder setSize(long size) { - this.size = size; - return this; - } - - public Builder setCreationTimeStamp(long creationTimeStamp) { - this.creationTimeStamp = creationTimeStamp; - return this; - } - - public Builder setLastUpdatedTimeStamp(long lastUpdatedTimeStamp) { - this.lastUpdatedTimeStamp = lastUpdatedTimeStamp; - return this; - } - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java deleted file mode 100644 index 648f6811007c..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskDbEventHandler.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.tasks; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; - -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT; - -/** - * Class for holding all NSSummaryTask methods - * related to DB operations so that they can commonly be - * used in NSSummaryTaskWithFSO and NSSummaryTaskWithLegacy. - */ -public class NSSummaryTaskDbEventHandler { - - private static final Logger LOG = - LoggerFactory.getLogger(NSSummaryTaskDbEventHandler.class); - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private ReconOMMetadataManager reconOMMetadataManager; - - private final long nsSummaryFlushToDBMaxThreshold; - - public NSSummaryTaskDbEventHandler(ReconNamespaceSummaryManager - reconNamespaceSummaryManager, - ReconOMMetadataManager - reconOMMetadataManager, - OzoneConfiguration - ozoneConfiguration) { - this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; - this.reconOMMetadataManager = reconOMMetadataManager; - nsSummaryFlushToDBMaxThreshold = ozoneConfiguration.getLong( - OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, - OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT); - } - - public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { - return reconNamespaceSummaryManager; - } - - public ReconOMMetadataManager getReconOMMetadataManager() { - return reconOMMetadataManager; - } - - protected void writeNSSummariesToDB(Map nsSummaryMap) - throws IOException { - try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { - nsSummaryMap.keySet().forEach((Long key) -> { - try { - reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, - key, nsSummaryMap.get(key)); - } catch (IOException e) { - LOG.error("Unable to write Namespace Summary data in Recon DB.", - e); - } - }); - reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); - } - } - - protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map nsSummaryMap) throws IOException { - long parentObjectId = keyInfo.getParentObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - if (nsSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - nsSummary = new NSSummary(); - } - int numOfFile = nsSummary.getNumOfFiles(); - long sizeOfFile = nsSummary.getSizeOfFiles(); - int[] fileBucket = nsSummary.getFileSizeBucket(); - nsSummary.setNumOfFiles(numOfFile + 1); - long dataSize = keyInfo.getDataSize(); - nsSummary.setSizeOfFiles(sizeOfFile + dataSize); - int binIndex = ReconUtils.getBinIndex(dataSize); - - ++fileBucket[binIndex]; - nsSummary.setFileSizeBucket(fileBucket); - nsSummaryMap.put(parentObjectId, nsSummary); - } - - protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = directoryInfo.getParentObjectID(); - long objectId = directoryInfo.getObjectID(); - // write the dir name to the current directory - String dirName = directoryInfo.getName(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary curNSSummary = nsSummaryMap.get(objectId); - if (curNSSummary == null) { - // If we don't have it in this batch we try to get it from the DB - curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId); - } - if (curNSSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - curNSSummary = new NSSummary(); - } - curNSSummary.setDirName(dirName); - nsSummaryMap.put(objectId, curNSSummary); - - // Write the child dir list to the parent directory - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - if (nsSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - nsSummary = new NSSummary(); - } - nsSummary.addChildDir(objectId); - nsSummaryMap.put(parentObjectId, nsSummary); - } - - protected void handleDeleteKeyEvent(OmKeyInfo keyInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = keyInfo.getParentObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - - // Just in case the OmKeyInfo isn't correctly written. - if (nsSummary == null) { - LOG.error("The namespace table is not correctly populated."); - return; - } - int numOfFile = nsSummary.getNumOfFiles(); - long sizeOfFile = nsSummary.getSizeOfFiles(); - int[] fileBucket = nsSummary.getFileSizeBucket(); - - long dataSize = keyInfo.getDataSize(); - int binIndex = ReconUtils.getBinIndex(dataSize); - - // decrement count, data size, and bucket count - // even if there's no direct key, we still keep the entry because - // we still need children dir IDs info - nsSummary.setNumOfFiles(numOfFile - 1); - nsSummary.setSizeOfFiles(sizeOfFile - dataSize); - --fileBucket[binIndex]; - nsSummary.setFileSizeBucket(fileBucket); - nsSummaryMap.put(parentObjectId, nsSummary); - } - - protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = directoryInfo.getParentObjectID(); - long objectId = directoryInfo.getObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - - // Just in case the OmDirectoryInfo isn't correctly written. - if (nsSummary == null) { - LOG.error("The namespace table is not correctly populated."); - return; - } - - nsSummary.removeChildDir(objectId); - nsSummaryMap.put(parentObjectId, nsSummary); - } - - protected boolean flushAndCommitNSToDB(Map nsSummaryMap) { - try { - writeNSSummariesToDB(nsSummaryMap); - nsSummaryMap.clear(); - } catch (IOException e) { - LOG.error("Unable to write Namespace Summary data in Recon DB.", e); - return false; - } - return true; - } - - protected boolean checkAndCallFlushToDB( - Map nsSummaryMap) { - // if map contains more than entries, flush to DB and clear the map - if (null != nsSummaryMap && nsSummaryMap.size() >= - nsSummaryFlushToDBMaxThreshold) { - return flushAndCommitNSToDB(nsSummaryMap); - } - return true; - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java deleted file mode 100644 index 9806b198b255..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithFSO.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.tasks; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; - -/** - * Class for handling FSO specific tasks. - */ -public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler { - - private static final Logger LOG = - LoggerFactory.getLogger(NSSummaryTaskWithFSO.class); - - public NSSummaryTaskWithFSO(ReconNamespaceSummaryManager - reconNamespaceSummaryManager, - ReconOMMetadataManager - reconOMMetadataManager, - OzoneConfiguration - ozoneConfiguration) { - super(reconNamespaceSummaryManager, - reconOMMetadataManager, ozoneConfiguration); - } - - // We only listen to updates from FSO-enabled KeyTable(FileTable) and DirTable - public Collection getTaskTables() { - return Arrays.asList(FILE_TABLE, DIRECTORY_TABLE); - } - - public boolean processWithFSO(OMUpdateEventBatch events) { - Iterator eventIterator = events.getIterator(); - final Collection taskTables = getTaskTables(); - Map nsSummaryMap = new HashMap<>(); - - while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); - - // we only process updates on OM's FileTable and Dirtable - String table = omdbUpdateEvent.getTable(); - boolean updateOnFileTable = table.equals(FILE_TABLE); - if (!taskTables.contains(table)) { - continue; - } - - String updatedKey = omdbUpdateEvent.getKey(); - - try { - if (updateOnFileTable) { - // key update on fileTable - OMDBUpdateEvent keyTableUpdateEvent = - (OMDBUpdateEvent) omdbUpdateEvent; - OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue(); - OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue(); - - switch (action) { - case PUT: - handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - case DELETE: - handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - case UPDATE: - if (oldKeyInfo != null) { - // delete first, then put - handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap); - } else { - LOG.warn("Update event does not have the old keyInfo for {}.", - updatedKey); - } - handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - default: - LOG.debug("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - - } else { - // directory update on DirTable - OMDBUpdateEvent dirTableUpdateEvent = - (OMDBUpdateEvent) omdbUpdateEvent; - OmDirectoryInfo updatedDirectoryInfo = dirTableUpdateEvent.getValue(); - OmDirectoryInfo oldDirectoryInfo = dirTableUpdateEvent.getOldValue(); - - switch (action) { - case PUT: - handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - case DELETE: - handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - case UPDATE: - if (oldDirectoryInfo != null) { - // delete first, then put - handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap); - } else { - LOG.warn("Update event does not have the old dirInfo for {}.", - updatedKey); - } - handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - default: - LOG.debug("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } - } catch (IOException ioEx) { - LOG.error("Unable to process Namespace Summary data in Recon DB. ", - ioEx); - return false; - } - if (!checkAndCallFlushToDB(nsSummaryMap)) { - return false; - } - } - - // flush and commit left out entries at end - if (!flushAndCommitNSToDB(nsSummaryMap)) { - return false; - } - - LOG.info("Completed a process run of NSSummaryTaskWithFSO"); - return true; - } - - public boolean reprocessWithFSO(OMMetadataManager omMetadataManager) { - Map nsSummaryMap = new HashMap<>(); - - try { - Table dirTable = - omMetadataManager.getDirectoryTable(); - try (TableIterator> - dirTableIter = dirTable.iterator()) { - while (dirTableIter.hasNext()) { - Table.KeyValue kv = dirTableIter.next(); - OmDirectoryInfo directoryInfo = kv.getValue(); - handlePutDirEvent(directoryInfo, nsSummaryMap); - if (!checkAndCallFlushToDB(nsSummaryMap)) { - return false; - } - } - } - - // Get fileTable used by FSO - Table keyTable = - omMetadataManager.getFileTable(); - - try (TableIterator> - keyTableIter = keyTable.iterator()) { - while (keyTableIter.hasNext()) { - Table.KeyValue kv = keyTableIter.next(); - OmKeyInfo keyInfo = kv.getValue(); - handlePutKeyEvent(keyInfo, nsSummaryMap); - if (!checkAndCallFlushToDB(nsSummaryMap)) { - return false; - } - } - } - - } catch (IOException ioEx) { - LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", - ioEx); - return false; - } - // flush and commit left out keys at end - if (!flushAndCommitNSToDB(nsSummaryMap)) { - return false; - } - LOG.info("Completed a reprocess run of NSSummaryTaskWithFSO"); - return true; - } - - -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java deleted file mode 100644 index a2a9598261e5..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGenTaskWithLegacy.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.tasks; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; - -/** - * Class for handling Legacy specific tasks. - */ -public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler { - - private static final BucketLayout BUCKET_LAYOUT = BucketLayout.LEGACY; - - private static final Logger LOG = - LoggerFactory.getLogger(NSSummaryTaskWithLegacy.class); - - private boolean enableFileSystemPaths; - - public NSSummaryTaskWithLegacy(ReconNamespaceSummaryManager - reconNamespaceSummaryManager, - ReconOMMetadataManager - reconOMMetadataManager, - OzoneConfiguration - ozoneConfiguration) { - super(reconNamespaceSummaryManager, - reconOMMetadataManager, ozoneConfiguration); - // true if FileSystemPaths enabled - enableFileSystemPaths = ozoneConfiguration - .getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); - } - - public boolean processWithLegacy(OMUpdateEventBatch events) { - Iterator eventIterator = events.getIterator(); - Map nsSummaryMap = new HashMap<>(); - - while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); - - // we only process updates on OM's KeyTable - String table = omdbUpdateEvent.getTable(); - boolean updateOnKeyTable = table.equals(KEY_TABLE); - if (!updateOnKeyTable) { - continue; - } - - String updatedKey = omdbUpdateEvent.getKey(); - - try { - OMDBUpdateEvent keyTableUpdateEvent = - (OMDBUpdateEvent) omdbUpdateEvent; - OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue(); - OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue(); - - // KeyTable entries belong to both Legacy and OBS buckets. - // Check bucket layout and if it's OBS - // continue to the next iteration. - // Check just for the current KeyInfo. - String volumeName = updatedKeyInfo.getVolumeName(); - String bucketName = updatedKeyInfo.getBucketName(); - String bucketDBKey = getReconOMMetadataManager() - .getBucketKey(volumeName, bucketName); - // Get bucket info from bucket table - OmBucketInfo omBucketInfo = getReconOMMetadataManager() - .getBucketTable().getSkipCache(bucketDBKey); - - if (omBucketInfo.getBucketLayout() - .isObjectStore(enableFileSystemPaths)) { - continue; - } - - setKeyParentID(updatedKeyInfo); - - if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { - switch (action) { - case PUT: - handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - case DELETE: - handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - case UPDATE: - if (oldKeyInfo != null) { - // delete first, then put - setKeyParentID(oldKeyInfo); - handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap); - } else { - LOG.warn("Update event does not have the old keyInfo for {}.", - updatedKey); - } - handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); - break; - - default: - LOG.debug("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } else { - OmDirectoryInfo updatedDirectoryInfo = - new OmDirectoryInfo.Builder() - .setName(updatedKeyInfo.getKeyName()) - .setObjectID(updatedKeyInfo.getObjectID()) - .setParentObjectID(updatedKeyInfo.getParentObjectID()) - .build(); - - OmDirectoryInfo oldDirectoryInfo = null; - - if (oldKeyInfo != null) { - oldDirectoryInfo = - new OmDirectoryInfo.Builder() - .setName(oldKeyInfo.getKeyName()) - .setObjectID(oldKeyInfo.getObjectID()) - .setParentObjectID(oldKeyInfo.getParentObjectID()) - .build(); - } - - switch (action) { - case PUT: - handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - case DELETE: - handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - case UPDATE: - if (oldDirectoryInfo != null) { - // delete first, then put - handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap); - } else { - LOG.warn("Update event does not have the old dirInfo for {}.", - updatedKey); - } - handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); - break; - - default: - LOG.debug("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } - } catch (IOException ioEx) { - LOG.error("Unable to process Namespace Summary data in Recon DB. ", - ioEx); - return false; - } - if (!checkAndCallFlushToDB(nsSummaryMap)) { - return false; - } - } - - // flush and commit left out entries at end - if (!flushAndCommitNSToDB(nsSummaryMap)) { - return false; - } - - LOG.info("Completed a process run of NSSummaryTaskWithLegacy"); - return true; - } - - public boolean reprocessWithLegacy(OMMetadataManager omMetadataManager) { - Map nsSummaryMap = new HashMap<>(); - - try { - Table keyTable = - omMetadataManager.getKeyTable(BUCKET_LAYOUT); - - try (TableIterator> - keyTableIter = keyTable.iterator()) { - - while (keyTableIter.hasNext()) { - Table.KeyValue kv = keyTableIter.next(); - OmKeyInfo keyInfo = kv.getValue(); - - // KeyTable entries belong to both Legacy and OBS buckets. - // Check bucket layout and if it's OBS - // continue to the next iteration. - String volumeName = keyInfo.getVolumeName(); - String bucketName = keyInfo.getBucketName(); - String bucketDBKey = omMetadataManager - .getBucketKey(volumeName, bucketName); - // Get bucket info from bucket table - OmBucketInfo omBucketInfo = omMetadataManager - .getBucketTable().getSkipCache(bucketDBKey); - - if (omBucketInfo.getBucketLayout() - .isObjectStore(enableFileSystemPaths)) { - continue; - } - - setKeyParentID(keyInfo); - - if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { - OmDirectoryInfo directoryInfo = - new OmDirectoryInfo.Builder() - .setName(keyInfo.getKeyName()) - .setObjectID(keyInfo.getObjectID()) - .setParentObjectID(keyInfo.getParentObjectID()) - .build(); - handlePutDirEvent(directoryInfo, nsSummaryMap); - } else { - handlePutKeyEvent(keyInfo, nsSummaryMap); - } - if (!checkAndCallFlushToDB(nsSummaryMap)) { - return false; - } - } - } - } catch (IOException ioEx) { - LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", - ioEx); - return false; - } - - // flush and commit left out entries at end - if (!flushAndCommitNSToDB(nsSummaryMap)) { - return false; - } - LOG.info("Completed a reprocess run of NSSummaryTaskWithLegacy"); - return true; - } - - /** - * KeyTable entries don't have the parentId set. - * In order to reuse the existing FSO methods that rely on - * the parentId, we have to set it explicitly. - * @param keyInfo - * @throws IOException - */ - private void setKeyParentID(OmKeyInfo keyInfo) throws IOException { - String[] keyPath = keyInfo.getKeyName().split(OM_KEY_PREFIX); - - // If the path contains only one key then keyPath.length - // will be 1 and the parent will be a bucket. - // If the keyPath.length is greater than 1 then - // there is at least one directory. - if (keyPath.length > 1) { - String[] dirs = Arrays.copyOf(keyPath, keyPath.length - 1); - String parentKeyName = String.join(OM_KEY_PREFIX, dirs); - parentKeyName += OM_KEY_PREFIX; - String fullParentKeyName = - getReconOMMetadataManager().getOzoneKey(keyInfo.getVolumeName(), - keyInfo.getBucketName(), parentKeyName); - OmKeyInfo parentKeyInfo = getReconOMMetadataManager() - .getKeyTable(BUCKET_LAYOUT) - .getSkipCache(fullParentKeyName); - - if (parentKeyInfo != null) { - keyInfo.setParentObjectID(parentKeyInfo.getObjectID()); - } else { - throw new IOException("ParentKeyInfo for " + - "NSSummaryTaskWithLegacy is null"); - } - } else { - String bucketKey = getReconOMMetadataManager() - .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName()); - OmBucketInfo parentBucketInfo = - getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey); - - if (parentBucketInfo != null) { - keyInfo.setParentObjectID(parentBucketInfo.getObjectID()); - } else { - throw new IOException("ParentKeyInfo for " + - "NSSummaryTaskWithLegacy is null"); - } - } - } -} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java deleted file mode 100644 index d1ef0bac1960..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/EntityMetricsGeneratorTask.java +++ /dev/null @@ -1,399 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.tasks; - -import com.google.inject.Inject; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.WithObjectID; -import org.hadoop.ozone.recon.schema.MetricsSchemaDefinition; -import org.hadoop.ozone.recon.schema.tables.daos.VolumeMetricsDao; -import org.hadoop.ozone.recon.schema.tables.pojos.VolumeMetrics; -import org.jooq.DSLContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.VOLUME_TABLE; -import static org.hadoop.ozone.recon.schema.tables.VolumeMetricsTable.VOLUME_METRICS; - -/** - * Class to iterate over the OM DB and populate the Recon metrics DB with - * the metrics at volume, bucket, key, file and directory level. - */ -public class EntityMetricsGeneratorTask implements ReconOmTask { - - private static final Logger LOG = - LoggerFactory.getLogger(EntityMetricsGeneratorTask.class); - - private VolumeMetricsDao volumeMetricsDao; - private DSLContext dslContext; - - @Inject - public EntityMetricsGeneratorTask(VolumeMetricsDao volumeMetricsDao, - MetricsSchemaDefinition - metricsSchemaDefinition) { - this.volumeMetricsDao = volumeMetricsDao; - this.dslContext = metricsSchemaDefinition.getDSLContext(); - } - - /** - * Read the Keys from OM snapshot DB and calculate the upper bound of - * File Size it belongs to. - * - * @param omMetadataManager OM Metadata instance. - * @return Pair - */ - @Override - public Pair reprocess(OMMetadataManager omMetadataManager) { - Map volumeMetricsMap = new HashMap<>(); - Table omVolumeArgsTable = - omMetadataManager.getVolumeTable(); - try (TableIterator> - keyIter = omVolumeArgsTable.iterator()) { - while (keyIter.hasNext()) { - VolumeMetrics volumeMetrics = new VolumeMetrics(); - Table.KeyValue kv = keyIter.next(); - OmVolumeArgs value = kv.getValue(); - volumeMetrics.setVolumeObjectid(value.getObjectID()); - List buckets = listBucketsUnderVolume(omMetadataManager, - value.getVolume()); - volumeMetrics.setBucketCount(buckets.size()); - volumeMetrics.setSize(getTotalSizesUnderVolume( - omMetadataManager, value.getVolume())); - volumeMetrics.setCreationTimestamp(new Timestamp( - value.getCreationTime())); - volumeMetrics.setLastUpdatedTimestamp(new Timestamp( - value.getModificationTime())); - volumeMetricsMap.put(value.getObjectID(), volumeMetrics); - } - } catch (IOException ioEx) { - LOG.error("Unable to populate File Size Count in Recon DB. ", ioEx); - return new ImmutablePair<>(getTaskName(), false); - } - // Truncate table before inserting new rows - int execute = dslContext.delete(VOLUME_METRICS).execute(); - LOG.info("Deleted {} records from {}", execute, VOLUME_METRICS); - - updateVolumeMetricsToDB(true, volumeMetricsMap); - - LOG.info("Completed a 'reprocess' run of FileSizeCountTask."); - return new ImmutablePair<>(getTaskName(), true); - } - - @Override - public String getTaskName() { - return "EntityMetricsGeneratorTask"; - } - - public Collection getTaskTables() { - return Arrays.asList(VOLUME_TABLE, BUCKET_TABLE, KEY_TABLE, - FILE_TABLE, DIRECTORY_TABLE); - } - - /** - * Read the Keys from update events and update the count of files - * pertaining to a certain upper bound. - * - * @param events Update events - PUT/DELETE. - * @return Pair - */ - @Override - public Pair process(OMUpdateEventBatch events) { - Iterator eventIterator = events.getIterator(); - Map volumeMetricsMap = new HashMap<>(); - final Collection taskTables = getTaskTables(); - - while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); - // Filter event inside process method to avoid duping - String table = omdbUpdateEvent.getTable(); - if (!taskTables.contains(table)) { - continue; - } - String updatedKey = omdbUpdateEvent.getKey(); - - try { - if (table.equals(VOLUME_TABLE)) { - OMDBUpdateEvent volTableUpdateEvent = - (OMDBUpdateEvent) omdbUpdateEvent; - OmVolumeArgs updatedVolInfo = volTableUpdateEvent.getValue(); - OmVolumeArgs oldVolInfo = volTableUpdateEvent.getOldValue(); - - switch (action) { - case PUT: - handlePutVolEvent(updatedVolInfo, volumeMetricsMap); - break; - - case DELETE: - handleDeleteVolEvent(updatedVolInfo, volumeMetricsMap); - break; - - case UPDATE: - //handleDeleteKeyEvent(updatedKey, omdbUpdateEvent.getOldValue(), - //fileSizeCountMap); - //handlePutKeyEvent(omKeyInfo, fileSizeCountMap); - break; - - default: - LOG.trace("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } - } catch (Exception e) { - LOG.error("Unexpected exception while processing key {}.", - updatedKey, e); - return new ImmutablePair<>(getTaskName(), false); - } - } - updateVolumeMetricsToDB(false, volumeMetricsMap); - LOG.info("Completed a 'process' run of FileSizeCountTask."); - return new ImmutablePair<>(getTaskName(), true); - } - - private void handleDeleteVolEvent(OmVolumeArgs updatedVolInfo, - Map volumeMetricsMap) { - VolumeMetrics volumeMetrics = volumeMetricsMap.get( - updatedVolInfo.getObjectID()); - if (volumeMetrics == null) { - // If we don't have it in this batch we try to get it from the DB - volumeMetrics = volumeMetricsDao.findById(updatedVolInfo.getObjectID()); - } - if (volumeMetrics == null) { - return; - } - volumeMetricsMap.remove(updatedVolInfo.getObjectID()); - } - - - /** - * Populate DB with the counts of file sizes calculated - * using the dao. - * - */ - private void updateVolumeMetricsToDB(boolean isDbTruncated, - Map - volumeMetricsMap) { - List insertToDb = new ArrayList<>(); - List updateInDb = new ArrayList<>(); - - volumeMetricsMap.keySet().forEach((Long key) -> { - VolumeMetrics volumeMetrics = volumeMetricsMap.get(key); - VolumeMetrics newRecord = new VolumeMetrics(); - newRecord.setVolumeObjectid(volumeMetrics.getVolumeObjectid()); - newRecord.setBucketCount(volumeMetrics.getBucketCount()); - newRecord.setAccessCount(volumeMetrics.getAccessCount()); - newRecord.setSize(volumeMetrics.getSize()); - newRecord.setCreationTimestamp(volumeMetrics.getCreationTimestamp()); - newRecord.setLastUpdatedTimestamp( - volumeMetrics.getLastUpdatedTimestamp()); - if (!isDbTruncated) { - // Get the current count from database and update - VolumeMetrics volMetrics = volumeMetricsDao.findById( - volumeMetrics.getVolumeObjectid()); - if (volMetrics == null && - (newRecord.getBucketCount() > 0 || - newRecord.getAccessCount() > 0 || - newRecord.getSize() > 0)) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } else if (volMetrics != null) { - newRecord.setBucketCount(volMetrics.getBucketCount() + - volumeMetrics.getBucketCount()); - newRecord.setAccessCount(volMetrics.getAccessCount() + - volumeMetrics.getAccessCount()); - newRecord.setSize(volMetrics.getSize() + - volumeMetrics.getSize()); - newRecord.setLastUpdatedTimestamp( - volumeMetrics.getLastUpdatedTimestamp()); - updateInDb.add(newRecord); - } - } else if (newRecord.getBucketCount() > 0 || - newRecord.getAccessCount() > 0 || - newRecord.getSize() > 0) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } - }); - volumeMetricsDao.insert(insertToDb); - volumeMetricsDao.update(updateInDb); - } - - private BucketLayout getBucketLayout() { - return BucketLayout.DEFAULT; - } - - private void handlePutVolEvent(OmVolumeArgs updatedVolInfo, - Map volumeMetricsMap) { - VolumeMetrics volumeMetrics = volumeMetricsMap.get( - updatedVolInfo.getObjectID()); - if (volumeMetrics == null) { - // If we don't have it in this batch we try to get it from the DB - volumeMetrics = volumeMetricsDao.findById(updatedVolInfo.getObjectID()); - } - if (volumeMetrics == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - volumeMetrics = new VolumeMetrics(); - } - volumeMetrics.setVolumeObjectid(updatedVolInfo.getObjectID()); - volumeMetrics.setBucketCount(0); - volumeMetrics.setSize(0); - volumeMetrics.setCreationTimestamp(new Timestamp( - updatedVolInfo.getCreationTime())); - volumeMetrics.setLastUpdatedTimestamp(new Timestamp( - updatedVolInfo.getModificationTime())); - volumeMetricsMap.put(updatedVolInfo.getObjectID(), volumeMetrics); - } - - /** - * Calculate and update the count of files being tracked by - * fileSizeCountMap. - * Used by reprocess() and process(). - * - * @param omKeyInfo OmKey being updated for count - */ - /*private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, - Map - fileSizeCountMap) { - if (omKeyInfo == null) { - LOG.warn("Deleting a key not found while handling DELETE key event. Key" + - " not found in Recon OM DB : {}", key); - } else { - FileSizeCountTask.FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); - Long count = fileSizeCountMap.containsKey(countKey) ? - fileSizeCountMap.get(countKey) - 1L : -1L; - fileSizeCountMap.put(countKey, count); - } - }*/ - - /** - * List all buckets under a volume, if volume name is null, return all - * buckets under the system. - * - * @param omMetadataManager - * @param volumeName volume name - * @return a list of buckets - * @throws IOException IOE - */ - private List listBucketsUnderVolume( - OMMetadataManager omMetadataManager, final String volumeName) - throws IOException { - List result = new ArrayList<>(); - // if volume name is null, seek prefix is an empty string - String seekPrefix = ""; - - Table bucketTable = - omMetadataManager.getBucketTable(); - - try (TableIterator> - iterator = bucketTable.iterator()) { - - if (volumeName != null) { - if (!volumeExists(omMetadataManager, volumeName)) { - return result; - } - seekPrefix = omMetadataManager.getVolumeKey(volumeName + OM_KEY_PREFIX); - } - - while (iterator.hasNext()) { - Table.KeyValue kv = iterator.next(); - - String key = kv.getKey(); - OmBucketInfo omBucketInfo = kv.getValue(); - - if (omBucketInfo != null) { - // We should return only the keys, whose keys match with - // the seek prefix - if (key.startsWith(seekPrefix)) { - result.add(omBucketInfo); - } - } - } - } - return result; - } - - private Integer getTotalSizesUnderVolume( - OMMetadataManager omMetadataManager, final String volumeName) - throws IOException { - int totalSize = 0; - // if volume name is null, seek prefix is an empty string - String seekPrefix = ""; - - Table bucketTable = - omMetadataManager.getBucketTable(); - - try (TableIterator> - iterator = bucketTable.iterator()) { - - if (volumeName != null) { - if (!volumeExists(omMetadataManager, volumeName)) { - return Integer.valueOf(totalSize); - } - seekPrefix = omMetadataManager.getVolumeKey(volumeName + OM_KEY_PREFIX); - } - - while (iterator.hasNext()) { - Table.KeyValue kv = iterator.next(); - - String key = kv.getKey(); - OmBucketInfo omBucketInfo = kv.getValue(); - - if (omBucketInfo != null) { - // We should return only the keys, whose keys match with - // the seek prefix - if (key.startsWith(seekPrefix)) { - totalSize += omBucketInfo.getUsedBytes(); - } - } - } - } - return Integer.valueOf(totalSize); - } - - private boolean volumeExists(OMMetadataManager omMetadataManager, - String volName) throws IOException { - String volDBKey = omMetadataManager.getVolumeKey(volName); - return omMetadataManager.getVolumeTable().getSkipCache(volDBKey) != null; - } - -} From b7d5a955efc0e7486074bdee2b9e437f9e4770f5 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Thu, 22 Dec 2022 12:13:30 +0530 Subject: [PATCH 7/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../org/apache/hadoop/ozone/recon/api/types/CountStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java index 950ef5ff5886..1d88831d807e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/CountStats.java @@ -24,7 +24,7 @@ * volumes/buckets/dir/files etc. * */ public class CountStats { - /** Total number of volumes under root, -1 for other types. */ + /** Total number of volumes under root, -1 for other types. */ @JsonProperty("numVolume") private int numVolume; From 7d51838c0fcf50b06d2de54d7c44aae7d63fbac7 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Wed, 11 Jan 2023 11:20:44 +0530 Subject: [PATCH 8/8] HDDS-5541. Namespace summary endpoint can carry basic information on the entity as well. --- .../hadoop/ozone/recon/api/handlers/FSOBucketHandler.java | 6 ++++++ .../ozone/recon/api/handlers/LegacyBucketHandler.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java index cb6ff098ae88..7cf2b46c85a2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/FSOBucketHandler.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.recon.api.handlers; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -279,6 +280,11 @@ public OmKeyInfo getKeyInfo(String[] names) throws IOException { @Override public OmDirectoryInfo getDirInfo(String[] names) throws IOException { + String path = OM_KEY_PREFIX; + path += String.join(OM_KEY_PREFIX, names); + Preconditions.checkArgument( + names.length >= 3, + "Path should be a directory: %s", path); long parentObjectId = getDirObjectId(names, names.length - 1); String dirKey = getOmMetadataManager().getOzonePathKey( getVolumeObjectId(names), diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java index 9105eb147f41..b0e5b8336f77 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.recon.api.handlers; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -326,6 +327,11 @@ public Table getKeyTable() { @Override public OmDirectoryInfo getDirInfo(String[] names) throws IOException { + String path = OM_KEY_PREFIX; + path += String.join(OM_KEY_PREFIX, names); + Preconditions.checkArgument( + names.length >= 3, + "Path should be a directory: %s", path); return OmDirectoryInfo.newBuilder() .setName(names[2]) .build();