HDDS-7121. Support namespace summaries (du, dist & counts) for legacy FS buckets

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java

@@ -70,6 +70,19 @@ public boolean isLegacy() {
     return this.equals(LEGACY);
   }
 
+  public boolean isObjectStore(boolean enableFileSystemPaths) {
+    if (this.equals(OBJECT_STORE)) {
+      return true;
+    } else {
+      // If bucket layout is Legacy and FileSystemPaths
+      // are disabled, then the bucket operates as OBS.
+      if (this.equals(LEGACY) && !enableFileSystemPaths) {
+        return true;
+      }
+      return false;
+    }
+  }
+
   public boolean shouldNormalizePaths(boolean enableFileSystemPaths) {
     switch (this) {
     case OBJECT_STORE:

hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_RUNNER_IMAGE=apache/ozone-runner
+OZONE_OPTS=

hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md

@@ -0,0 +1,21 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# For Legacy Bucket Operations
+
+For Legacy buckets, set `ozone.om.enable.filesystem.paths` to `true` for them to behave like FSO buckets, 
+otherwise Legacy buckets act like OBS buckets.
+
+This is the same as `compose/ozone` but for testing operations that need `ozone.om.enable.filesystem.paths`
+flag enabled.

hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml

@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
+x-common-config:
+  &common-config
+  image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+  volumes:
+    - ../..:/opt/hadoop
+  env_file:
+    - docker-config
+
+x-replication:
+  &replication
+  OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
+
+services:
+  datanode:
+    <<: *common-config
+    ports:
+      - 9864
+      - 9882
+    environment:
+      <<: *replication
+      OZONE_OPTS:
+    command: ["ozone","datanode"]
+  om:
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      OZONE_OPTS:
+      <<: *replication
+    ports:
+      - 9874:9874
+      - 9862:9862
+    command: ["ozone","om"]
+  scm:
+    <<: *common-config
+    ports:
+      - 9876:9876
+      - 9860:9860
+    environment:
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+      OZONE_OPTS:
+      <<: *replication
+    command: ["ozone","scm"]
+  s3g:
+    <<: *common-config
+    environment:
+      OZONE_OPTS:
+      <<: *replication
+    ports:
+      - 9878:9878
+    command: ["ozone","s3g"]
+  recon:
+    <<: *common-config
+    ports:
+      - 9888:9888
+    environment:
+      OZONE_OPTS:
+      <<: *replication
+    command: ["ozone","recon"]

hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=ofs://om
+CORE-SITE.XML_fs.trash.interval=1
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.enable.filesystem.paths=true
+OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.block.size=1MB
+OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
+OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
+OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
+OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
+OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
+OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
+OZONE-SITE.XML_hdds.container.report.interval=60s
+OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
+OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
+OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
+OZONE-SITE.XML_hdds.heartbeat.interval=5s
+
+OZONE_CONF_DIR=/etc/hadoop
+OZONE_LOG_DIR=/var/log/hadoop
+
+no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1

hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:unsecure
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env 5
+
+execute_robot_test scm -v BUCKET_LAYOUT:LEGACY recon/recon-nssummary.robot
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot → hadoop-ozone/dist/src/main/smoketest/recon/recon-nssummary.robot

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 *** Settings ***
-Documentation       Smoke test for Recon Namespace Summary Endpoint for FSO buckets.
+Documentation       Smoke test for Recon Namespace Summary Endpoint for ${BUCKET_LAYOUT} buckets.
 Library             OperatingSystem
 Library             String
 Library             BuiltIn
@@ -29,6 +29,7 @@ ${SUMMARY_URL}              ${ADMIN_NAMESPACE_URL}/summary
 ${DISK_USAGE_URL}           ${ADMIN_NAMESPACE_URL}/du
 ${QUOTA_USAGE_URL}          ${ADMIN_NAMESPACE_URL}/quota
 ${FILE_SIZE_DIST_URL}       ${ADMIN_NAMESPACE_URL}/dist
+${BUCKET_LAYOUT}            FILE_SYSTEM_OPTIMIZED
 ${VOLUME}
 ${BUCKET}
 
@@ -42,7 +43,7 @@ Create volume
 Create bucket
     ${random} =     Generate Random String  5  [LOWER]
                     Set Suite Variable     ${BUCKET}    buc-${random}
-    ${result} =     Execute             ozone sh bucket create -l FILE_SYSTEM_OPTIMIZED /${VOLUME}/${BUCKET}
+    ${result} =     Execute             ozone sh bucket create -l ${BUCKET_LAYOUT} /${VOLUME}/${BUCKET}
                     Should not contain  ${result}       Failed
 
 Create keys
@@ -83,7 +84,7 @@ Check Access
     kinit as recon admin
     Check http return code      ${url}       200
 
-Test Summary                            
+Test Summary
     [Arguments]         ${url}        ${expected}
            ${result} =         Execute                              curl --negotiate -u : -LSs ${url}
                                Should contain      ${result}       \"status\":\"OK\"
@@ -131,7 +132,8 @@ Check Recon Namespace Summary Key
     Wait For Summary      ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/file1   KEY
 
 Check Recon Namespace Summary Directory
-    Wait For Summary      ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2   DIRECTORY
+    Run Keyword If    '${BUCKET_LAYOUT}' == 'LEGACY'                    Wait For Summary      ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2/   DIRECTORY
+    Run Keyword If    '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED'     Wait For Summary      ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2    DIRECTORY
 
 Check Recon Namespace Disk Usage
     Wait For Summary      ${DISK_USAGE_URL}?path=/${VOLUME}/${BUCKET}&files=true&replica=true     \"sizeWithReplica\"
@@ -143,4 +145,4 @@ Check Recon Namespace Bucket Quota Usage
     Wait For Summary      ${QUOTA_USAGE_URL}?path=/${VOLUME}/${BUCKET}   \"used\"
 
 Check Recon Namespace File Size Distribution Root
-    Wait For Summary      ${FILE_SIZE_DIST_URL}?path=/                   \"dist\"
+    Wait For Summary      ${FILE_SIZE_DIST_URL}?path=/                   \"dist\"

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java

@@ -110,7 +110,7 @@ public void testNSSummaryCLIRoot() throws UnsupportedEncodingException {
     // Should throw warning - only buckets can have bucket layout.
     Assert.assertTrue(
         getOutContentString().contains(
-            "[Warning] Namespace CLI is only designed for FSO mode."));
+            "[Warning] Namespace CLI is not designed for OBS bucket layout."));
     Assert.assertTrue(getOutContentString()
         .contains("Put more files into it to visualize DU"));
     Assert.assertTrue(getOutContentString().contains(
@@ -128,7 +128,7 @@ public void testNSSummaryCLIFSO() throws UnsupportedEncodingException {
     // Should not throw warning, since bucket is in FSO bucket layout.
     Assert.assertFalse(
         getOutContentString().contains(
-            "[Warning] Namespace CLI is only designed for FSO mode."));
+            "[Warning] Namespace CLI is not designed for OBS bucket layout."));
     Assert.assertTrue(getOutContentString()
         .contains("Put more files into it to visualize DU"));
     Assert.assertTrue(getOutContentString().contains(
@@ -146,7 +146,7 @@ public void testNSSummaryCLIOBS() throws UnsupportedEncodingException {
     // Should throw warning, since bucket is in OBS bucket layout.
     Assert.assertTrue(
         getOutContentString().contains(
-            "[Warning] Namespace CLI is only designed for FSO mode."));
+            "[Warning] Namespace CLI is not designed for OBS bucket layout."));
     Assert.assertTrue(getOutContentString()
         .contains("Put more files into it to visualize DU"));
     Assert.assertTrue(getOutContentString().contains(

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java

@@ -47,7 +47,7 @@
 import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider;
 import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl;
 import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
-import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO;
+import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask;
 import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
 import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask;
 import org.apache.hadoop.ozone.recon.tasks.TableCountTask;
@@ -126,7 +126,7 @@ protected void configure() {
       taskBinder.addBinding().to(ContainerKeyMapperTask.class);
       taskBinder.addBinding().to(FileSizeCountTask.class);
       taskBinder.addBinding().to(TableCountTask.class);
-      taskBinder.addBinding().to(NSSummaryTaskWithFSO.class);
+      taskBinder.addBinding().to(NSSummaryTask.class);
     }
   }
 

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java

@@ -38,6 +38,8 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
+
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
 import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded;
 
@@ -183,8 +185,24 @@ public static BucketHandler getBucketHandler(
                 OzoneStorageContainerManager reconSCM,
                 OmBucketInfo bucketInfo) throws IOException {
 
-    return new FSOBucketHandler(reconNamespaceSummaryManager,
-              omMetadataManager, reconSCM, bucketInfo);
+    // If bucketInfo is null then entity type is UNKNOWN
+    if (Objects.isNull(bucketInfo)) {
+      return null;
+    } else {
+      if (bucketInfo.getBucketLayout()
+          .equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+        return new FSOBucketHandler(reconNamespaceSummaryManager,
+            omMetadataManager, reconSCM, bucketInfo);
+      } else if (bucketInfo.getBucketLayout()
+          .equals(BucketLayout.LEGACY)) {
+        return new LegacyBucketHandler(reconNamespaceSummaryManager,
+            omMetadataManager, reconSCM, bucketInfo);
+      } else {
+        LOG.error("Unsupported bucket layout: " +
+            bucketInfo.getBucketLayout());
+        return null;
+      }
+    }
   }
 
   public static BucketHandler getBucketHandler(
@@ -197,11 +215,7 @@ public static BucketHandler getBucketHandler(
     OmBucketInfo bucketInfo = omMetadataManager
         .getBucketTable().getSkipCache(bucketKey);
 
-    if (bucketInfo == null) {
-      return null;
-    } else {
-      return getBucketHandler(reconNamespaceSummaryManager,
-          omMetadataManager, reconSCM, bucketInfo);
-    }
+    return getBucketHandler(reconNamespaceSummaryManager,
+        omMetadataManager, reconSCM, bucketInfo);
   }
 }