HDDS-4026. Dir rename failed when sets 'ozone.om.enable.filesystem.paths' to true

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java

@@ -35,10 +35,15 @@
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import static org.junit.Assert.fail;
 
 /**
  * Class tests create with object store and getFileStatus.
@@ -120,6 +125,100 @@ public void test() throws Exception {
 
   }
 
+
+  @Test
+  public void testObjectStoreCreateWithO3fs() throws Exception {
+    OzoneVolume ozoneVolume =
+        cluster.getRpcClient().getObjectStore().getVolume(volumeName);
+
+    OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
+
+
+    // Use ObjectStore API to create keys. This similates how s3 create keys.
+    String parentDir = "/dir1/dir2/dir3/dir4/";
+
+
+    List<String> keys = new ArrayList<>();
+    keys.add("/dir1");
+    keys.add("/dir1/dir2");
+    keys.add("/dir1/dir2/dir3");
+    keys.add("/dir1/dir2/dir3/dir4/");
+    for (int i=1; i <= 3; i++) {
+      int length = 10;
+      String fileName = parentDir.concat("/file" + i + "/");
+      keys.add(fileName);
+      OzoneOutputStream ozoneOutputStream =
+          ozoneBucket.createKey(fileName, length);
+      byte[] b = new byte[10];
+      Arrays.fill(b, (byte)96);
+      ozoneOutputStream.write(b);
+      ozoneOutputStream.close();
+    }
+
+    // check
+    for (int i=1; i <= 3; i++) {
+      String fileName = parentDir.concat("/file" + i + "/");
+      Path p = new Path(fileName);
+      Assert.assertTrue(o3fs.getFileStatus(p).isFile());
+      checkAncestors(p);
+    }
+
+    // Delete keys with object store api delete
+    for (int i = 1; i <= 3; i++) {
+      String fileName = parentDir.concat("/file" + i + "/");
+      ozoneBucket.deleteKey(fileName);
+    }
+
+
+    // Delete parent dir via o3fs.
+    boolean result = o3fs.delete(new Path("/dir1"), true);
+    Assert.assertTrue(result);
+
+    // No Key should exist.
+    for(String key : keys) {
+      checkPath(new Path(key));
+    }
+
+
+    for (int i=1; i <= 3; i++) {
+      int length = 10;
+      String fileName = parentDir.concat("/file" + i + "/");
+      OzoneOutputStream ozoneOutputStream =
+          ozoneBucket.createKey(fileName, length);
+      byte[] b = new byte[10];
+      Arrays.fill(b, (byte)96);
+      ozoneOutputStream.write(b);
+      ozoneOutputStream.close();
+    }
+
+    o3fs.mkdirs(new Path("/dest"));
+    o3fs.rename(new Path("/dir1"), new Path("/dest"));
+
+    // No source Key should exist.
+    for(String key : keys) {
+      checkPath(new Path(key));
+    }
+
+    // check dest path.
+    for (int i=1; i <= 3; i++) {
+      String fileName = "/dest/".concat(parentDir.concat("/file" + i + "/"));
+      Path p = new Path(fileName);
+      Assert.assertTrue(o3fs.getFileStatus(p).isFile());
+      checkAncestors(p);
+    }
+
+  }
+
+  private void checkPath(Path path) {
+    try {
+      o3fs.getFileStatus(path);
+      fail("testObjectStoreCreateWithO3fs failed for Path" + path);
+    } catch (IOException ex) {
+      Assert.assertTrue(ex instanceof FileNotFoundException);
+      Assert.assertTrue(ex.getMessage().contains("No such file or directory"));
+    }
+  }
+
   private void checkAncestors(Path p) throws Exception {
     p = p.getParent();
     while(p.getParent() != null) {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OMMetrics;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -97,7 +98,8 @@ public class TestOzoneFileInterfaces {
    */
   @Parameters
   public static Collection<Object[]> data() {
-    return Arrays.asList(new Object[][] {{false, true}, {true, false}});
+    return Arrays.asList(new Object[][] {{false, true, true},
+        {true, false, false}});
   }
 
   private boolean setDefaultFs;
@@ -118,10 +120,13 @@ public static Collection<Object[]> data() {
 
   private OMMetrics omMetrics;
 
+  private boolean enableFileSystemPaths;
+
   public TestOzoneFileInterfaces(boolean setDefaultFs,
-      boolean useAbsolutePath) {
+      boolean useAbsolutePath, boolean enabledFileSystemPaths) {
     this.setDefaultFs = setDefaultFs;
     this.useAbsolutePath = useAbsolutePath;
+    this.enableFileSystemPaths = enabledFileSystemPaths;
     GlobalStorageStatistics.INSTANCE.reset();
   }
 
@@ -131,6 +136,8 @@ public void init() throws Exception {
     bucketName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
 
     OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
+        enableFileSystemPaths);
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
         .build();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java

@@ -18,8 +18,10 @@
 
 package org.apache.hadoop.fs.ozone;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Set;
@@ -42,6 +44,7 @@
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -66,14 +69,25 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Ozone file system tests that are not covered by contract tests.
  */
+@RunWith(Parameterized.class)
 public class TestOzoneFileSystem {
 
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[]{true}, new Object[]{false});
+  }
+
+  public TestOzoneFileSystem(boolean setDefaultFs) {
+    this.enabledFileSystemPaths = setDefaultFs;
+  }
   /**
    * Set a timeout for each test.
    */
@@ -83,6 +97,8 @@ public class TestOzoneFileSystem {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestOzoneFileSystem.class);
 
+  private boolean enabledFileSystemPaths;
+
   private MiniOzoneCluster cluster;
   private FileSystem fs;
   private OzoneFileSystem o3fs;
@@ -215,6 +231,8 @@ public void testFileSystem() throws Exception {
     testRenameDir();
     testSeekOnFileLength();
     testDeleteRoot();
+
+    testRecursiveDelete();
   }
 
   @After
@@ -229,6 +247,8 @@ private void setupOzoneFileSystem()
       throws IOException, TimeoutException, InterruptedException {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(FS_TRASH_INTERVAL_KEY, 1);
+    conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
+        enabledFileSystemPaths);
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
         .build();
@@ -296,7 +316,7 @@ private void testDeleteCreatesFakeParentDir() throws Exception {
     }
 
     // Delete the child key
-    fs.delete(child, false);
+    fs.delete(child, true);
 
     // Deleting the only child should create the parent dir key if it does
     // not exist
@@ -305,6 +325,78 @@ private void testDeleteCreatesFakeParentDir() throws Exception {
     assertEquals(parentKey, parentKeyInfo.getName());
   }
 
+
+  private void testRecursiveDelete() throws Exception {
+    Path grandparent = new Path("/gdir1");
+
+    for (int i = 1; i <= 10; i++) {
+      Path parent = new Path(grandparent, "pdir" +i);
+      Path child = new Path(parent, "child");
+      ContractTestUtils.touch(fs, child);
+    }
+
+    // Delete the grandparent, which should delete all keys.
+    fs.delete(grandparent, true);
+
+    checkPath(grandparent);
+
+    for (int i = 1; i <= 10; i++) {
+      Path parent = new Path(grandparent, "dir" +i);
+      Path child = new Path(parent, "child");
+      checkPath(parent);
+      checkPath(child);
+    }
+
+
+    Path level0 = new Path("/level0");
+
+    for (int i = 1; i <= 3; i++) {
+      Path level1 = new Path(level0, "level" +i);
+      Path level2 = new Path(level1, "level" +i);
+      Path level1File = new Path(level1, "file1");
+      Path level2File = new Path(level2, "file1");
+      ContractTestUtils.touch(fs, level1File);
+      ContractTestUtils.touch(fs, level2File);
+    }
+
+    // Delete at sub directory level.
+    for (int i = 1; i <= 3; i++) {
+      Path level1 = new Path(level0, "level" +i);
+      Path level2 = new Path(level1, "level" +i);
+      fs.delete(level2, true);
+      fs.delete(level1, true);
+    }
+
+
+    // Delete level0 finally.
+    fs.delete(grandparent, true);
+
+    // Check if it exists or not.
+    checkPath(grandparent);
+
+    for (int i = 1; i <= 3; i++) {
+      Path level1 = new Path(level0, "level" +i);
+      Path level2 = new Path(level1, "level" +i);
+      Path level1File = new Path(level1, "file1");
+      Path level2File = new Path(level2, "file1");
+      checkPath(level1);
+      checkPath(level2);
+      checkPath(level1File);
+      checkPath(level2File);
+    }
+
+  }
+
+  private void checkPath(Path path) {
+    try {
+      fs.getFileStatus(path);
+      fail("testRecursiveDelete failed");
+    } catch (IOException ex) {
+      Assert.assertTrue(ex instanceof FileNotFoundException);
+      Assert.assertTrue(ex.getMessage().contains("No such file or directory"));
+    }
+  }
+
   private void testFileDelete() throws Exception {
     Path grandparent = new Path("/testBatchDelete");
     Path parent = new Path(grandparent, "parent");

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java

@@ -87,19 +87,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
 
     KeyArgs renameKeyArgs = renameKeyRequest.getKeyArgs();
 
-    // Set modification time and normalize key if needed.
+    // Set modification time.
     KeyArgs.Builder newKeyArgs = renameKeyArgs.toBuilder()
-            .setModificationTime(Time.now())
-        .setKeyName(validateAndNormalizeKey(
-            ozoneManager.getEnableFileSystemPaths(),
-            renameKeyArgs.getKeyName()));
+            .setModificationTime(Time.now());
 
     return getOmRequest().toBuilder()
         .setRenameKeyRequest(renameKeyRequest.toBuilder()
-            .setKeyArgs(newKeyArgs)
-            .setToKeyName(validateAndNormalizeKey(
-                ozoneManager.getEnableFileSystemPaths(),
-                renameKeyRequest.getToKeyName())))
+            .setKeyArgs(newKeyArgs))
         .setUserInfo(getUserInfo()).build();
 
   }