HDDS-7810. Support namespace summaries (du, dist & counts) for OBJECT_STORE buckets.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java

@@ -178,9 +178,7 @@ public static BucketHandler getBucketHandler(
             omMetadataManager, reconSCM, bucketInfo);
       } else if (bucketInfo.getBucketLayout()
           .equals(BucketLayout.OBJECT_STORE)) {
-        // TODO: HDDS-7810 Write a handler for object store bucket
-        // We can use LegacyBucketHandler for OBS bucket for now.
-        return new LegacyBucketHandler(reconNamespaceSummaryManager,
+        return new OBSBucketHandler(reconNamespaceSummaryManager,
             omMetadataManager, reconSCM, bucketInfo);
       } else {
         LOG.error("Unsupported bucket layout: " +

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java

@@ -32,6 +32,8 @@
 import org.apache.hadoop.ozone.recon.api.types.NSSummary;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -46,6 +48,10 @@
  */
 public abstract class EntityHandler {
 
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      EntityHandler.class);
+
   private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
 
   private final ReconOMMetadataManager omMetadataManager;
@@ -118,67 +124,67 @@ public String[] getNames() {
    * @return the entity handler of client's request
    */
   public static EntityHandler getEntityHandler(
-          ReconNamespaceSummaryManager reconNamespaceSummaryManager,
-          ReconOMMetadataManager omMetadataManager,
-          OzoneStorageContainerManager reconSCM,
-          String path) throws IOException {
+      ReconNamespaceSummaryManager reconNamespaceSummaryManager,
+      ReconOMMetadataManager omMetadataManager,
+      OzoneStorageContainerManager reconSCM,
+      String path) throws IOException {
     BucketHandler bucketHandler;
 
     String normalizedPath = normalizePath(path);
     String[] names = parseRequestPath(normalizedPath);
     if (path.equals(OM_KEY_PREFIX)) {
       return EntityType.ROOT.create(reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM, null, path);
+          omMetadataManager, reconSCM, null, path);
     }
 
     if (names.length == 0) {
       return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM, null, path);
+          omMetadataManager, reconSCM, null, path);
     } else if (names.length == 1) { // volume level check
       String volName = names[0];
       if (!volumeExists(omMetadataManager, volName)) {
         return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
-                omMetadataManager, reconSCM, null, path);
+            omMetadataManager, reconSCM, null, path);
       }
       return EntityType.VOLUME.create(reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM, null, path);
+          omMetadataManager, reconSCM, null, path);
     } else if (names.length == 2) { // bucket level check
       String volName = names[0];
       String bucketName = names[1];
 
       bucketHandler = BucketHandler.getBucketHandler(
-              reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM,
-              volName, bucketName);
+          reconNamespaceSummaryManager,
+          omMetadataManager, reconSCM,
+          volName, bucketName);
 
       if (bucketHandler == null
           || !bucketHandler.bucketExists(volName, bucketName)) {
         return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
-                omMetadataManager, reconSCM, null, path);
+            omMetadataManager, reconSCM, null, path);
       }
       return EntityType.BUCKET.create(reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM, bucketHandler, path);
+          omMetadataManager, reconSCM, bucketHandler, path);
     } else { // length > 3. check dir or key existence
       String volName = names[0];
       String bucketName = names[1];
 
       String keyName = BucketHandler.getKeyName(names);
 
       bucketHandler = BucketHandler.getBucketHandler(
-              reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM,
-              volName, bucketName);
+          reconNamespaceSummaryManager,
+          omMetadataManager, reconSCM,
+          volName, bucketName);
 
       // check if either volume or bucket doesn't exist
       if (bucketHandler == null
           || !volumeExists(omMetadataManager, volName)
           || !bucketHandler.bucketExists(volName, bucketName)) {
         return EntityType.UNKNOWN.create(reconNamespaceSummaryManager,
-                omMetadataManager, reconSCM, null, path);
+            omMetadataManager, reconSCM, null, path);
       }
       return bucketHandler.determineKeyPath(keyName)
           .create(reconNamespaceSummaryManager,
-          omMetadataManager, reconSCM, bucketHandler, path);
+              omMetadataManager, reconSCM, bucketHandler, path);
     }
   }
 

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/OBSBucketHandler.java

@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.handlers;
+
+
+import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.recon.api.types.DUResponse;
+import org.apache.hadoop.ozone.recon.api.types.EntityType;
+import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
+import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+
+/**
+ * Class for handling Legacy buckets.
+ */
+public class OBSBucketHandler extends BucketHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      OBSBucketHandler.class);
+
+  private final String vol;
+  private final String bucket;
+  private final OmBucketInfo omBucketInfo;
+
+  public OBSBucketHandler(
+      ReconNamespaceSummaryManager reconNamespaceSummaryManager,
+      ReconOMMetadataManager omMetadataManager,
+      OzoneStorageContainerManager reconSCM,
+      OmBucketInfo bucketInfo) {
+    super(reconNamespaceSummaryManager, omMetadataManager,
+        reconSCM);
+    this.omBucketInfo = bucketInfo;
+    this.vol = omBucketInfo.getVolumeName();
+    this.bucket = omBucketInfo.getBucketName();
+  }
+
+  /**
+   * Helper function to check if a path is a key, or invalid.
+   *
+   * @param keyName key name
+   * @return KEY, or UNKNOWN
+   * @throws IOException
+   */
+  public EntityType determineKeyPath(String keyName)
+      throws IOException {
+    String key = OM_KEY_PREFIX + vol +
+        OM_KEY_PREFIX + bucket +
+        OM_KEY_PREFIX + keyName;
+
+    Table<String, OmKeyInfo> keyTable = getKeyTable();
+
+    TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+        iterator = keyTable.iterator();
+
+    iterator.seek(key);
+    if (iterator.hasNext()) {
+      Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
+      String dbKey = kv.getKey();
+      if (dbKey.equals(key)) {
+        return EntityType.KEY;
+      }
+    }
+    return EntityType.UNKNOWN;
+  }
+
+  /**
+   * This method handles disk usage of direct keys.
+   *
+   * @param parentId       parent OBS bucket
+   * @param withReplica    if withReplica is enabled, set sizeWithReplica
+   *                       for each direct key's DU
+   * @param listFile       if listFile is enabled, append key DU as a children
+   *                       keys
+   * @param duData         the current DU data
+   * @param normalizedPath the normalized path request
+   * @return the total DU of all direct keys
+   * @throws IOException IOE
+   */
+  public long handleDirectKeys(long parentId, boolean withReplica,
+                               boolean listFile,
+                               List<DUResponse.DiskUsage> duData,
+                               String normalizedPath) throws IOException {
+
+    Table<String, OmKeyInfo> keyTable = getKeyTable();
+    long keyDataSizeWithReplica = 0L;
+
+    TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+        iterator = keyTable.iterator();
+
+    String seekPrefix = OM_KEY_PREFIX +
+        vol +
+        OM_KEY_PREFIX +
+        bucket +
+        OM_KEY_PREFIX;
+
+    NSSummary nsSummary = getReconNamespaceSummaryManager()
+        .getNSSummary(parentId);
+    // Handle the case of an empty bucket.
+    if (nsSummary == null) {
+      return 0;
+    }
+
+    iterator.seek(seekPrefix);
+
+    while (iterator.hasNext()) {
+      // KeyName : OmKeyInfo-Object
+      Table.KeyValue<String, OmKeyInfo> kv = iterator.next();
+      String dbKey = kv.getKey();
+
+      // Exit loop if the key doesn't match the seekPrefix.
+      if (!dbKey.startsWith(seekPrefix)) {
+        break;
+      }
+
+      OmKeyInfo keyInfo = kv.getValue();
+      if (keyInfo != null) {
+        DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage();
+        String objectName = keyInfo.getKeyName();
+        diskUsage.setSubpath(objectName);
+        diskUsage.setKey(true);
+        diskUsage.setSize(keyInfo.getDataSize());
+
+        if (withReplica) {
+          long keyDU = keyInfo.getReplicatedSize();
+          keyDataSizeWithReplica += keyDU;
+          diskUsage.setSizeWithReplica(keyDU);
+        }
+        // List all the keys for the OBS bucket if requested.
+        if (listFile) {
+          duData.add(diskUsage);
+        }
+      }
+    }
+
+    return keyDataSizeWithReplica;
+  }
+
+  /**
+   * Object stores do not support directories, hence return null.
+   *
+   * @return null
+   */
+  public long calculateDUUnderObject(long parentId)
+      throws IOException {
+    return Long.parseLong(null);
+  }
+
+  /**
+   * Object stores do not support directories, hence return null.
+   *
+   * @return null
+   */
+  public long getDirObjectId(String[] names) throws IOException {
+    return Long.parseLong(null);
+  }
+
+  /**
+   * Object stores do not support directories, hence return null.
+   *
+   * @return null
+   */
+  public long getDirObjectId(String[] names, int cutoff) throws IOException {
+    return Long.parseLong(null);
+  }
+
+
+  public OmKeyInfo getKeyInfo(String[] names) throws IOException {
+    String ozoneKey = OM_KEY_PREFIX;
+    ozoneKey += String.join(OM_KEY_PREFIX, names);
+
+    OmKeyInfo keyInfo = getKeyTable().getSkipCache(ozoneKey);
+    return keyInfo;
+  }
+
+  /**
+   * Object stores do not support directories, hence return null.
+   *
+   * @return null
+   */
+  @Override
+  public OmDirectoryInfo getDirInfo(String[] names) throws IOException {
+    return null;
+  }
+
+  public Table<String, OmKeyInfo> getKeyTable() {
+    Table keyTable =
+        getOmMetadataManager().getKeyTable(getBucketLayout());
+    return keyTable;
+  }
+
+  public BucketLayout getBucketLayout() {
+    return BucketLayout.OBJECT_STORE;
+  }
+
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java

@@ -67,6 +67,7 @@ public class NSSummaryTask implements ReconOmTask {
   private final ReconOMMetadataManager reconOMMetadataManager;
   private final NSSummaryTaskWithFSO nsSummaryTaskWithFSO;
   private final NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy;
+  private final NSSummaryTaskWithOBS nsSummaryTaskWithOBS;
   private final OzoneConfiguration ozoneConfiguration;
 
   @Inject
@@ -85,6 +86,9 @@ public NSSummaryTask(ReconNamespaceSummaryManager
     this.nsSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy(
         reconNamespaceSummaryManager,
         reconOMMetadataManager, ozoneConfiguration);
+    this.nsSummaryTaskWithOBS = new NSSummaryTaskWithOBS(
+        reconNamespaceSummaryManager,
+        reconOMMetadataManager, ozoneConfiguration);
   }
 
   @Override
@@ -101,6 +105,11 @@ public Pair<String, Boolean> process(OMUpdateEventBatch events) {
     } else {
       LOG.error("processWithFSO failed.");
     }
+    if (success) {
+      success = nsSummaryTaskWithOBS.processWithOBS(events);
+    } else {
+      LOG.error("processWithOBS failed.");
+    }
     return new ImmutablePair<>(getTaskName(), success);
   }
 
@@ -121,6 +130,8 @@ public Pair<String, Boolean> reprocess(OMMetadataManager omMetadataManager) {
         .reprocessWithFSO(omMetadataManager));
     tasks.add(() -> nsSummaryTaskWithLegacy
         .reprocessWithLegacy(reconOMMetadataManager));
+    tasks.add(() -> nsSummaryTaskWithOBS
+        .reprocessWithOBS(reconOMMetadataManager));
 
     List<Future<Boolean>> results;
     ExecutorService executorService = Executors