HDDS-12333. Move ozone.om.enable.filesystem.paths into OmConfig

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3558,21 +3558,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.om.enable.filesystem.paths</name>
-    <tag>OZONE, OM</tag>
-    <value>false</value>
-    <description>If true, key names will be interpreted as file system paths.
-      "/" will be treated as a special character and paths will be normalized
-      and must follow Unix filesystem path naming conventions. This flag will
-      be helpful when objects created by S3G need to be accessed using OFS/O3Fs.
-      If false, it will fallback to default behavior of Key/MPU create
-      requests where key paths are not normalized and any intermediate
-      directories will not be created or any file checks happens to check
-      filesystem semantics.
-    </description>
-  </property>
-
   <property>
     <name>ozone.om.key.path.lock.enabled</name>
     <tag>OZONE, OM</tag>

hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
 import org.apache.hadoop.ozone.shell.OzoneAddress;
@@ -40,8 +41,6 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
 import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT;
 
 /**
  * Executes Delete Key.
@@ -70,7 +69,7 @@ protected void execute(OzoneClient client, OzoneAddress address)
     }
 
     if (bucket.getBucketLayout().isLegacy() && keyName.endsWith(OZONE_URI_DELIMITER)
-        && (getConf().getBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT))) {
+        && (getConf().getBoolean(OmConfig.Keys.ENABLE_FILESYSTEM_PATHS, OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS))) {
       out().printf("Use FS(ofs/o3fs) interface to delete legacy bucket directory %n");
       return;
     }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -342,12 +342,12 @@ private OMConfigKeys() {
   public static final boolean OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT =
           false;
 
-  // This config needs to be enabled, when S3G created objects used via
-  // FileSystem API.
+  @Deprecated
   public static final String OZONE_OM_ENABLE_FILESYSTEM_PATHS =
-      "ozone.om.enable.filesystem.paths";
+      OmConfig.Keys.ENABLE_FILESYSTEM_PATHS;
+  @Deprecated
   public static final boolean OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT =
-      false;
+      OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS;
 
   public static final String OZONE_SERVER_DEFAULT_REPLICATION_KEY =
       "ozone.server.default.replication";

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java

@@ -28,6 +28,22 @@
 @ConfigGroup(prefix = "ozone.om")
 public class OmConfig {
 
+  /** This config needs to be enabled, when S3G created objects used via FileSystem API. */
+  @Config(
+      key = "enable.filesystem.paths",
+      defaultValue = "false",
+      description = "If true, key names will be interpreted as file system paths. " +
+          "'/' will be treated as a special character and paths will be normalized " +
+          "and must follow Unix filesystem path naming conventions. This flag will " +
+          "be helpful when objects created by S3G need to be accessed using OFS/O3Fs. " +
+          "If false, it will fallback to default behavior of Key/MPU create " +
+          "requests where key paths are not normalized and any intermediate " +
+          "directories will not be created or any file checks happens to check " +
+          "filesystem semantics.",
+      tags = { ConfigTag.OM, ConfigTag.OZONE }
+  )
+  private boolean fileSystemPathEnabled;
+
   @Config(
       key = "server.list.max.size",
       defaultValue = "1000",
@@ -36,6 +52,14 @@ public class OmConfig {
   )
   private long maxListSize;
 
+  public boolean isFileSystemPathEnabled() {
+    return fileSystemPathEnabled;
+  }
+
+  public void setFileSystemPathEnabled(boolean newValue) {
+    fileSystemPathEnabled = newValue;
+  }
+
   public long getMaxListSize() {
     return maxListSize;
   }
@@ -56,13 +80,15 @@ public void validate() {
    * String keys for tests and grep.
    */
   public static final class Keys {
+    public static final String ENABLE_FILESYSTEM_PATHS = "ozone.om.enable.filesystem.paths";
     public static final String SERVER_LIST_MAX_SIZE = "ozone.om.server.list.max.size";
   }
 
   /**
    * Default values for tests.
    */
-  static final class Defaults {
+  public static final class Defaults {
+    public static final boolean ENABLE_FILESYSTEM_PATHS = false;
     public static final long SERVER_LIST_MAX_SIZE = 1000;
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java

@@ -94,6 +94,7 @@ private void addPropertiesNotInXml() {
         OMConfigKeys.OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY,
         OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED,
         OMConfigKeys.OZONE_OM_FEATURES_DISABLED,
+        OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
         OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE,
         OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY,
         OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS,

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java

@@ -112,7 +112,6 @@
 import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OmFailoverProxyUtil;
 import org.apache.hadoop.ozone.om.OzoneManager;
@@ -1741,8 +1740,7 @@ public void testMissingParentBucketUsedNamespace(BucketLayout layout)
     OzoneBucket bucket = volume.getBucket(bucketName);
 
     if (layout.equals(BucketLayout.LEGACY)) {
-      OzoneConfiguration conf = cluster.getConf();
-      conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true);
+      cluster.getOzoneManager().getConfig().setFileSystemPathEnabled(true);
     }
 
     // the directory "/dir1", ""/dir1/dir2/", "/dir1/dir2/dir3/"

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java

@@ -23,12 +23,10 @@
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.BucketArgs;
@@ -44,10 +42,12 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 
 import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.NonHATests;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,11 +68,11 @@
 /**
  * Test verifies object store with OZONE_OM_ENABLE_FILESYSTEM_PATHS enabled.
  */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
 @Timeout(200)
-public class TestObjectStoreWithLegacyFS {
+public abstract class TestObjectStoreWithLegacyFS implements NonHATests.TestCase {
 
-  private static MiniOzoneCluster cluster = null;
-  private static OzoneClient client;
+  private OzoneClient client;
 
   private String volumeName;
 
@@ -82,30 +82,22 @@ public class TestObjectStoreWithLegacyFS {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(TestObjectStoreWithLegacyFS.class);
+  private boolean originalFileSystemPathEnabled;
 
   @BeforeAll
-  public static void initClass() throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
-
-    conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true);
-    conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
-        BucketLayout.LEGACY.name());
-    cluster = MiniOzoneCluster.newBuilder(conf)
-        .setNumDatanodes(3)
-        .build();
-    cluster.waitForClusterToBeReady();
-    client = cluster.newClient();
+  void initClass() throws Exception {
+    OmConfig omConfig = cluster().getOzoneManager().getConfig();
+    originalFileSystemPathEnabled = omConfig.isFileSystemPathEnabled();
+    omConfig.setFileSystemPathEnabled(true);
+
+    client = cluster().newClient();
   }
 
-  /**
-   * Shutdown MiniOzoneCluster.
-   */
   @AfterAll
-  public static void shutdown() {
+  void cleanup() {
     IOUtils.closeQuietly(client);
-    if (cluster != null) {
-      cluster.shutdown();
-    }
+    OmConfig omConfig = cluster().getOzoneManager().getConfig();
+    omConfig.setFileSystemPathEnabled(originalFileSystemPathEnabled);
   }
 
   @BeforeEach
@@ -132,11 +124,11 @@ public void testFlatKeyStructureWithOBS() throws Exception {
         .createKey("dir1/dir2/dir3/key-1", 0);
     stream.close();
     Table<String, OmKeyInfo> keyTable =
-        cluster.getOzoneManager().getMetadataManager()
+        cluster().getOzoneManager().getMetadataManager()
             .getKeyTable(BucketLayout.OBJECT_STORE);
 
     String seekKey = "dir";
-    String dbKey = cluster.getOzoneManager().getMetadataManager()
+    String dbKey = cluster().getOzoneManager().getMetadataManager()
         .getOzoneKey(volumeName, bucketName, seekKey);
 
     GenericTestUtils

hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java

@@ -170,6 +170,14 @@ public MiniOzoneCluster cluster() {
     }
   }
 
+  @Nested
+  class ObjectStoreWithLegacyFS extends org.apache.hadoop.ozone.om.TestObjectStoreWithLegacyFS {
+    @Override
+    public MiniOzoneCluster cluster() {
+      return getCluster();
+    }
+  }
+
   @Nested
   class OmBlockVersioning extends org.apache.hadoop.ozone.om.TestOmBlockVersioning {
     @Override

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -192,7 +192,6 @@ public class KeyManagerImpl implements KeyManager {
   private SnapshotDeletingService snapshotDeletingService;
 
   private final KeyProviderCryptoExtension kmsProvider;
-  private final boolean enableFileSystemPaths;
   private DirectoryDeletingService dirDeletingService;
   private final OMPerformanceMetrics metrics;
 
@@ -218,9 +217,6 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient,
     this.grpcBlockTokenEnabled = conf.getBoolean(
         HDDS_BLOCK_TOKEN_ENABLED,
         HDDS_BLOCK_TOKEN_ENABLED_DEFAULT);
-    this.enableFileSystemPaths =
-        conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
-            OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
 
     this.ozoneManager = om;
     this.scmClient = scmClient;
@@ -473,7 +469,7 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args, BucketLayout bucketLayout)
         bucketName);
     try {
       keyName = OMClientRequest
-          .validateAndNormalizeKey(enableFileSystemPaths, keyName,
+          .validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(), keyName,
               bucketLayout);
 
       if (bucketLayout.isFileSystemOptimized()) {
@@ -665,7 +661,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName,
     // underlying table using an iterator. That automatically creates a
     // snapshot of the data, so we don't need these locks at a higher level
     // when we iterate.
-    if (bucketLayout.shouldNormalizePaths(enableFileSystemPaths)) {
+    if (bucketLayout.shouldNormalizePaths(ozoneManager.getEnableFileSystemPaths())) {
       startKey = OmUtils.normalizeKey(startKey, true);
       keyPrefix = OmUtils.normalizeKey(keyPrefix, true);
     }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -278,8 +278,6 @@
 import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KEY_PATH_LOCK_ENABLED;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
@@ -4467,8 +4465,7 @@ public void setExitManagerForTesting(ExitManager exitManagerForTesting) {
   }
 
   public boolean getEnableFileSystemPaths() {
-    return configuration.getBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS,
-        OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
+    return config.isFileSystemPathEnabled();
   }
 
   public boolean getKeyPathLockEnabled() {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java

@@ -21,7 +21,7 @@
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -211,8 +211,8 @@ private static boolean isEnableFileSystemPaths(ReconOMMetadataManager omMetadata
     if (configuration == null) {
       configuration = new OzoneConfiguration();
     }
-    return configuration.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
-        OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
+    return configuration.getBoolean(OmConfig.Keys.ENABLE_FILESYSTEM_PATHS,
+        OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS);
   }
 
   public static BucketHandler getBucketHandler(

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java

@@ -20,8 +20,8 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -64,8 +64,8 @@ public NSSummaryTaskWithLegacy(ReconNamespaceSummaryManager
         reconOMMetadataManager, ozoneConfiguration);
     // true if FileSystemPaths enabled
     enableFileSystemPaths = ozoneConfiguration
-        .getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
-            OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
+        .getBoolean(OmConfig.Keys.ENABLE_FILESYSTEM_PATHS,
+            OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS);
   }
 
   public boolean processWithLegacy(OMUpdateEventBatch events) {

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -60,6 +60,7 @@
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OmConfig;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -113,7 +114,6 @@
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT;
 import static org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
@@ -360,7 +360,7 @@ public Response put(
         OS3Exception os3Exception = newError(INVALID_REQUEST, keyPath, ex);
         os3Exception.setErrorMessage("An error occurred (InvalidRequest) " +
             "when calling the PutObject/MPU PartUpload operation: " +
-            OZONE_OM_ENABLE_FILESYSTEM_PATHS + " is enabled Keys are" +
+            OmConfig.Keys.ENABLE_FILESYSTEM_PATHS + " is enabled Keys are" +
             " considered as Unix Paths. Path has Violated FS Semantics " +
             "which caused put operation to fail.");
         throw os3Exception;
@@ -932,7 +932,7 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket,
         OS3Exception os3Exception = newError(INVALID_REQUEST, key, ex);
         os3Exception.setErrorMessage("An error occurred (InvalidRequest) " +
             "when calling the CompleteMultipartUpload operation: " +
-            OZONE_OM_ENABLE_FILESYSTEM_PATHS + " is enabled Keys are " +
+            OmConfig.Keys.ENABLE_FILESYSTEM_PATHS + " is enabled Keys are " +
             "considered as Unix Paths. A directory already exists with a " +
             "given KeyName caused failure for MPU");
         throw os3Exception;