From e52ba2155a86b1df02ab54448b7a71d64ef0a19f Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Fri, 30 Oct 2020 18:37:58 +0530 Subject: [PATCH 1/9] HDDS-4346.Ozone specific Trash Policy --- .../hadoop/ozone/om/TrashPolicyOzone.java | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java new file mode 100644 index 000000000000..2082fc2ee831 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Collection; +import java.util.Date; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.TrashPolicy; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** TrashPolicy for Ozone Specific Trash Operations.Through this implementation + * of TrashPolicy ozone-specific trash optimizations are/will be made such as + * having a multithreaded TrashEmptier. + */ +public class TrashPolicyOzone extends TrashPolicy { + + private static final Logger LOG = + LoggerFactory.getLogger(TrashPolicyOzone.class); + + private static final Path CURRENT = new Path("Current"); + + private static final FsPermission PERMISSION = + new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE); + + private static final DateFormat CHECKPOINT = new SimpleDateFormat( + "yyMMddHHmmss"); + /** Format of checkpoint directories used prior to Hadoop 0.23. */ + private static final DateFormat OLD_CHECKPOINT = + new SimpleDateFormat("yyMMddHHmm"); + private static final int MSECS_PER_MINUTE = 60*1000; + + private long emptierInterval; + + public TrashPolicyOzone(){ + } + + private TrashPolicyOzone(FileSystem fs, Configuration conf){ + initialize(conf, fs); + } + + @Override + public void initialize(Configuration conf, FileSystem fileSystem, Path path) { + this.fs = fs; + this.deletionInterval = (long)(conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + this.emptierInterval = (long)(conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + } + + @Override + public void initialize(Configuration conf, FileSystem fs) { + this.fs = fs; + this.deletionInterval = (long)(conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + this.emptierInterval = (long)(conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + if (deletionInterval < 0) { + LOG.warn("Invalid value {} for deletion interval," + + " deletion interaval can not be negative." + + "Changing to default value 0", deletionInterval); + this.deletionInterval = 0; + } + } + + private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { + return Path.mergePaths(basePath, rmFilePath); + } + + @Override + public boolean isEnabled() { + return deletionInterval > 0; + } + + @Override + public boolean moveToTrash(Path path) throws IOException { + if (!isEnabled()) { + return false; + } + + if (!path.isAbsolute()) { // make path absolute + path = new Path(fs.getWorkingDirectory(), path); + } + + // check that path exists + fs.getFileStatus(path); + String qpath = fs.makeQualified(path).toString(); + + Path trashRoot = fs.getTrashRoot(path); + Path trashCurrent = new Path(trashRoot, CURRENT); + if (qpath.startsWith(trashRoot.toString())) { + return false; // already in trash + } + + if (trashRoot.getParent().toString().startsWith(qpath)) { + throw new IOException("Cannot move \"" + path + + "\" to the trash, as it contains the trash"); + } + + Path trashPath = makeTrashRelativePath(trashCurrent, path); + Path baseTrashPath = makeTrashRelativePath(trashCurrent, path.getParent()); + + IOException cause = null; + + // try twice, in case checkpoint between the mkdirs() & rename() + for (int i = 0; i < 2; i++) { + try { + if (!fs.mkdirs(baseTrashPath, PERMISSION)) { // create current + LOG.warn("Can't create(mkdir) trash directory: " + baseTrashPath); + return false; + } + } catch (FileAlreadyExistsException e) { + // find the path which is not a directory, and modify baseTrashPath + // & trashPath, then mkdirs + Path existsFilePath = baseTrashPath; + while (!fs.exists(existsFilePath)) { + existsFilePath = existsFilePath.getParent(); + } + baseTrashPath = new Path(baseTrashPath.toString().replace( + existsFilePath.toString(), existsFilePath.toString() + Time.now()) + ); + trashPath = new Path(baseTrashPath, trashPath.getName()); + // retry, ignore current failure + --i; + continue; + } catch (IOException e) { + LOG.warn("Can't create trash directory: " + baseTrashPath, e); + cause = e; + break; + } + try { + // if the target path in Trash already exists, then append with + // a current time in millisecs. + String orig = trashPath.toString(); + + while(fs.exists(trashPath)) { + trashPath = new Path(orig + Time.now()); + } + + // move to current trash + fs.rename(path, trashPath); + LOG.info("Moved: '" + path + "' to trash at: " + trashPath); + return true; + } catch (IOException e) { + cause = e; + } + } + throw (IOException) + new IOException("Failed to move to trash: " + path).initCause(cause); + } + + + @Override + public void createCheckpoint() throws IOException { + createCheckpoint(new Date()); + } + + public void createCheckpoint(Date date) throws IOException { + Collection trashRoots = fs.getTrashRoots(false); + for (FileStatus trashRoot: trashRoots) { + LOG.info("TrashPolicyOzone#createCheckpoint for trashRoot: " + + trashRoot.getPath()); + createCheckpoint(trashRoot.getPath(), date); + } + } + + @Override + public void deleteCheckpoint() throws IOException { + deleteCheckpoint(false); + } + + @Override + public void deleteCheckpointsImmediately() throws IOException { + deleteCheckpoint(true); + } + + private void deleteCheckpoint(boolean deleteImmediately) throws IOException { + Collection trashRoots = fs.getTrashRoots(false); + for (FileStatus trashRoot : trashRoots) { + LOG.info("TrashPolicyOzone#deleteCheckpoint for trashRoot: " + + trashRoot.getPath()); + deleteCheckpoint(trashRoot.getPath(), deleteImmediately); + } + } + + @Override + public Path getCurrentTrashDir() { + return new Path(fs.getTrashRoot(null), CURRENT); + } + + @Override + public Path getCurrentTrashDir(Path path) throws IOException { + return new Path(fs.getTrashRoot(path), CURRENT); + } + + @Override + public Runnable getEmptier() throws IOException { + return new TrashPolicyOzone.Emptier(getConf(), emptierInterval); + } + + protected class Emptier implements Runnable { + + private Configuration conf; + // same as checkpoint interval + private long emptierInterval; + + Emptier(Configuration conf, long emptierInterval) throws IOException { + this.conf = conf; + this.emptierInterval = emptierInterval; + if (emptierInterval > deletionInterval || emptierInterval <= 0) { + LOG.info("The configured checkpoint interval is " + + (emptierInterval / MSECS_PER_MINUTE) + " minutes." + + " Using an interval of " + + (deletionInterval / MSECS_PER_MINUTE) + + " minutes that is used for deletion instead"); + this.emptierInterval = deletionInterval; + } + LOG.info("Ozone Manager trash configuration: Deletion interval = " + + (deletionInterval / MSECS_PER_MINUTE) + + " minutes, Emptier interval = " + + (this.emptierInterval / MSECS_PER_MINUTE) + " minutes."); + } + + @Override + public void run() { + if (emptierInterval == 0) { + return; // trash disabled + } + long now, end; + while (true) { + now = Time.now(); + end = ceiling(now, emptierInterval); + try { // sleep for interval + Thread.sleep(end - now); + } catch (InterruptedException e) { + break; // exit on interrupt + } + + try { + now = Time.now(); + if (now >= end) { + Collection trashRoots; + trashRoots = fs.getTrashRoots(true); // list all trash dirs + + for (FileStatus trashRoot : trashRoots) { // dump each trash + if (!trashRoot.isDirectory()) { + continue; + } + try { + TrashPolicyOzone trash = new TrashPolicyOzone(fs, conf); + trash.deleteCheckpoint(trashRoot.getPath(), false); + trash.createCheckpoint(trashRoot.getPath(), new Date(now)); + } catch (IOException e) { + LOG.warn("Trash caught: "+e+". Skipping " + + trashRoot.getPath() + "."); + } + } + } + } catch (Exception e) { + LOG.warn("RuntimeException during Trash.Emptier.run(): ", e); + } + } + try { + fs.close(); + } catch(IOException e) { + LOG.warn("Trash cannot close FileSystem: ", e); + } + } + + private long ceiling(long time, long interval) { + return floor(time, interval) + interval; + } + private long floor(long time, long interval) { + return (time / interval) * interval; + } + + } + + private void createCheckpoint(Path trashRoot, Date date) throws IOException { + if (!fs.exists(new Path(trashRoot, CURRENT))) { + return; + } + Path checkpointBase; + synchronized (CHECKPOINT) { + checkpointBase = new Path(trashRoot, CHECKPOINT.format(date)); + } + Path checkpoint = checkpointBase; + Path current = new Path(trashRoot, CURRENT); + + int attempt = 0; + while (true) { + try { + fs.rename(current, checkpoint); + LOG.info("Created trash checkpoint: " + checkpoint.toUri().getPath()); + break; + } catch (FileAlreadyExistsException e) { + if (++attempt > 1000) { + throw new IOException("Failed to checkpoint trash: " + checkpoint); + } + checkpoint = checkpointBase.suffix("-" + attempt); + } + } + } + + private void deleteCheckpoint(Path trashRoot, boolean deleteImmediately) + throws IOException { + LOG.info("TrashPolicyOzone#deleteCheckpoint for trashRoot: " + trashRoot); + + FileStatus[] dirs = null; + try { + dirs = fs.listStatus(trashRoot); // scan trash sub-directories + } catch (FileNotFoundException fnfe) { + return; + } + + long now = Time.now(); + for (int i = 0; i < dirs.length; i++) { + Path path = dirs[i].getPath(); + String dir = path.toUri().getPath(); + String name = path.getName(); + if (name.equals(CURRENT.getName())) { // skip current + continue; + } + + long time; + try { + time = getTimeFromCheckpoint(name); + } catch (ParseException e) { + LOG.warn("Unexpected item in trash: "+dir+". Ignoring."); + continue; + } + + if (((now - deletionInterval) > time) || deleteImmediately) { + if (fs.delete(path, true)) { + LOG.info("Deleted trash checkpoint: "+dir); + } else { + LOG.warn("Couldn't delete checkpoint: " + dir + " Ignoring."); + } + } + } + } + + private long getTimeFromCheckpoint(String name) throws ParseException { + long time; + + try { + synchronized (CHECKPOINT) { + time = CHECKPOINT.parse(name).getTime(); + } + } catch (ParseException pe) { + // Check for old-style checkpoint directories left over + // after an upgrade from Hadoop 1.x + synchronized (OLD_CHECKPOINT) { + time = OLD_CHECKPOINT.parse(name).getTime(); + } + } + + return time; + } +} From fbad42e5de5504a687081e61ab2b7cf0cfbe6b39 Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Fri, 30 Oct 2020 20:55:14 +0530 Subject: [PATCH 2/9] fix failing test --- .../main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java index 2082fc2ee831..b72dc99f094b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -73,7 +73,7 @@ private TrashPolicyOzone(FileSystem fs, Configuration conf){ } @Override - public void initialize(Configuration conf, FileSystem fileSystem, Path path) { + public void initialize(Configuration conf, FileSystem fs, Path path) { this.fs = fs; this.deletionInterval = (long)(conf.getFloat( FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) From 89bea7623f5ad95dc139a31d2229ea7ca99c3bef Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Tue, 3 Nov 2020 21:32:19 +0530 Subject: [PATCH 3/9] Addressed review comment --- .../hadoop/ozone/om/TrashPolicyOzone.java | 138 +----------------- 1 file changed, 4 insertions(+), 134 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java index b72dc99f094b..50e07ca70e12 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -31,13 +31,13 @@ import java.util.Date; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.TrashPolicyDefault; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +46,7 @@ * of TrashPolicy ozone-specific trash optimizations are/will be made such as * having a multithreaded TrashEmptier. */ -public class TrashPolicyOzone extends TrashPolicy { +public class TrashPolicyOzone extends TrashPolicyDefault { private static final Logger LOG = LoggerFactory.getLogger(TrashPolicyOzone.class); @@ -100,136 +100,6 @@ public void initialize(Configuration conf, FileSystem fs) { } } - private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { - return Path.mergePaths(basePath, rmFilePath); - } - - @Override - public boolean isEnabled() { - return deletionInterval > 0; - } - - @Override - public boolean moveToTrash(Path path) throws IOException { - if (!isEnabled()) { - return false; - } - - if (!path.isAbsolute()) { // make path absolute - path = new Path(fs.getWorkingDirectory(), path); - } - - // check that path exists - fs.getFileStatus(path); - String qpath = fs.makeQualified(path).toString(); - - Path trashRoot = fs.getTrashRoot(path); - Path trashCurrent = new Path(trashRoot, CURRENT); - if (qpath.startsWith(trashRoot.toString())) { - return false; // already in trash - } - - if (trashRoot.getParent().toString().startsWith(qpath)) { - throw new IOException("Cannot move \"" + path + - "\" to the trash, as it contains the trash"); - } - - Path trashPath = makeTrashRelativePath(trashCurrent, path); - Path baseTrashPath = makeTrashRelativePath(trashCurrent, path.getParent()); - - IOException cause = null; - - // try twice, in case checkpoint between the mkdirs() & rename() - for (int i = 0; i < 2; i++) { - try { - if (!fs.mkdirs(baseTrashPath, PERMISSION)) { // create current - LOG.warn("Can't create(mkdir) trash directory: " + baseTrashPath); - return false; - } - } catch (FileAlreadyExistsException e) { - // find the path which is not a directory, and modify baseTrashPath - // & trashPath, then mkdirs - Path existsFilePath = baseTrashPath; - while (!fs.exists(existsFilePath)) { - existsFilePath = existsFilePath.getParent(); - } - baseTrashPath = new Path(baseTrashPath.toString().replace( - existsFilePath.toString(), existsFilePath.toString() + Time.now()) - ); - trashPath = new Path(baseTrashPath, trashPath.getName()); - // retry, ignore current failure - --i; - continue; - } catch (IOException e) { - LOG.warn("Can't create trash directory: " + baseTrashPath, e); - cause = e; - break; - } - try { - // if the target path in Trash already exists, then append with - // a current time in millisecs. - String orig = trashPath.toString(); - - while(fs.exists(trashPath)) { - trashPath = new Path(orig + Time.now()); - } - - // move to current trash - fs.rename(path, trashPath); - LOG.info("Moved: '" + path + "' to trash at: " + trashPath); - return true; - } catch (IOException e) { - cause = e; - } - } - throw (IOException) - new IOException("Failed to move to trash: " + path).initCause(cause); - } - - - @Override - public void createCheckpoint() throws IOException { - createCheckpoint(new Date()); - } - - public void createCheckpoint(Date date) throws IOException { - Collection trashRoots = fs.getTrashRoots(false); - for (FileStatus trashRoot: trashRoots) { - LOG.info("TrashPolicyOzone#createCheckpoint for trashRoot: " + - trashRoot.getPath()); - createCheckpoint(trashRoot.getPath(), date); - } - } - - @Override - public void deleteCheckpoint() throws IOException { - deleteCheckpoint(false); - } - - @Override - public void deleteCheckpointsImmediately() throws IOException { - deleteCheckpoint(true); - } - - private void deleteCheckpoint(boolean deleteImmediately) throws IOException { - Collection trashRoots = fs.getTrashRoots(false); - for (FileStatus trashRoot : trashRoots) { - LOG.info("TrashPolicyOzone#deleteCheckpoint for trashRoot: " + - trashRoot.getPath()); - deleteCheckpoint(trashRoot.getPath(), deleteImmediately); - } - } - - @Override - public Path getCurrentTrashDir() { - return new Path(fs.getTrashRoot(null), CURRENT); - } - - @Override - public Path getCurrentTrashDir(Path path) throws IOException { - return new Path(fs.getTrashRoot(path), CURRENT); - } - @Override public Runnable getEmptier() throws IOException { return new TrashPolicyOzone.Emptier(getConf(), emptierInterval); From b27a06322e820a8767cb04ff098a0c2e3a28eead Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Wed, 11 Nov 2020 16:34:11 +0530 Subject: [PATCH 4/9] remove initialise method as it uses the parent implementation --- .../hadoop/ozone/om/TrashPolicyOzone.java | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java index 50e07ca70e12..6d6f5d1c68aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -17,11 +17,6 @@ */ package org.apache.hadoop.ozone.om; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; - import java.io.FileNotFoundException; import java.io.IOException; import java.text.DateFormat; @@ -69,35 +64,7 @@ public TrashPolicyOzone(){ } private TrashPolicyOzone(FileSystem fs, Configuration conf){ - initialize(conf, fs); - } - - @Override - public void initialize(Configuration conf, FileSystem fs, Path path) { - this.fs = fs; - this.deletionInterval = (long)(conf.getFloat( - FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) - * MSECS_PER_MINUTE); - this.emptierInterval = (long)(conf.getFloat( - FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) - * MSECS_PER_MINUTE); - } - - @Override - public void initialize(Configuration conf, FileSystem fs) { - this.fs = fs; - this.deletionInterval = (long)(conf.getFloat( - FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) - * MSECS_PER_MINUTE); - this.emptierInterval = (long)(conf.getFloat( - FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) - * MSECS_PER_MINUTE); - if (deletionInterval < 0) { - LOG.warn("Invalid value {} for deletion interval," - + " deletion interaval can not be negative." - + "Changing to default value 0", deletionInterval); - this.deletionInterval = 0; - } + super.initialize(conf, fs); } @Override From a39e1a6667388a1d0650c46159fb1bcfaf907c63 Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Fri, 13 Nov 2020 22:01:23 +0530 Subject: [PATCH 5/9] call ozone trash emptier from OM --- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 9 +++++++++ .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index f8b07ab219dd..1ee683231f2e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -27,6 +27,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.TrashPolicy; +import org.apache.hadoop.fs.TrashPolicyDefault; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -41,6 +43,7 @@ import org.apache.hadoop.ozone.client.VolumeArgs; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; @@ -146,6 +149,8 @@ public static void init() throws Exception { conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); // fs.ofs.impl would be loaded from META-INF, no need to manually set it fs = FileSystem.get(conf); + conf.setClass("fs.trash.classname", TrashPolicyOzone.class, + TrashPolicy.class); trash = new Trash(conf); ofs = (RootedOzoneFileSystem) fs; adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter(); @@ -1184,6 +1189,10 @@ public void testTrash() throws Exception { try (FSDataOutputStream stream = fs.create(path)) { stream.write(1); } + Assert.assertTrue(trash.getConf().getClass( + "fs.trash.classname", TrashPolicyDefault.class). + isAssignableFrom(TrashPolicyOzone.class)); + // Call moveToTrash. We can't call protected fs.rename() directly trash.moveToTrash(path); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f8e9fb146db0..2d2c9f468a25 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -57,6 +57,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@ -1286,7 +1287,8 @@ public FileSystem run() throws IOException { return FileSystem.get(fsconf); } }); - + conf.setClass("fs.trash.classname", TrashPolicyOzone.class, + TrashPolicy.class); this.emptier = new Thread(new Trash(fs, conf). getEmptier(), "Trash Emptier"); this.emptier.setDaemon(true); From d7c30ee900bd0b0a7ed78effd4f494d4a607f8bc Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Mon, 16 Nov 2020 16:27:11 +0530 Subject: [PATCH 6/9] trigger new CI check From 080eed0407ccea949e1e683bcc0c4cbf1b733d4d Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Wed, 18 Nov 2020 10:30:05 +0530 Subject: [PATCH 7/9] addressed review comments --- .../org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 3 +-- .../main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 1ee683231f2e..642294269bda 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.TrashPolicy; -import org.apache.hadoop.fs.TrashPolicyDefault; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -1190,7 +1189,7 @@ public void testTrash() throws Exception { stream.write(1); } Assert.assertTrue(trash.getConf().getClass( - "fs.trash.classname", TrashPolicyDefault.class). + "fs.trash.classname", TrashPolicy.class). isAssignableFrom(TrashPolicyOzone.class)); // Call moveToTrash. We can't call protected fs.rename() directly diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java index 6d6f5d1c68aa..c0278bc291d8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -37,7 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** TrashPolicy for Ozone Specific Trash Operations.Through this implementation +/** + * TrashPolicy for Ozone Specific Trash Operations.Through this implementation * of TrashPolicy ozone-specific trash optimizations are/will be made such as * having a multithreaded TrashEmptier. */ From b2297c20cfef9a094bb82ec17f006ad3204bedbf Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Wed, 18 Nov 2020 12:10:49 +0530 Subject: [PATCH 8/9] trigger new CI check From 560b64487973e450a55c0b7f5a2e2efedc9a5588 Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Wed, 18 Nov 2020 13:30:32 +0530 Subject: [PATCH 9/9] trigger new CI check