HDFS-15982. Deleted data using HTTP API should be saved to the trash

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/**
+ * SkipTrash param to be used by DELETE query.
+ */
+public class DeleteSkipTrashParam extends BooleanParam {
+
+  public static final String NAME = "skiptrash";
+  public static final String DEFAULT = FALSE;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public DeleteSkipTrashParam(final Boolean value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public DeleteSkipTrashParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java

@@ -121,6 +121,7 @@ public class HttpFSFileSystem extends FileSystem
   public static final String ACLSPEC_PARAM = "aclspec";
   public static final String DESTINATION_PARAM = "destination";
   public static final String RECURSIVE_PARAM = "recursive";
+  public static final String SKIP_TRASH_PARAM = "skiptrash";
   public static final String SOURCES_PARAM = "sources";
   public static final String OWNER_PARAM = "owner";
   public static final String GROUP_PARAM = "group";

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.fs.XAttrCodec;
 import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
@@ -52,6 +53,8 @@
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -74,6 +77,8 @@
 @InterfaceAudience.Private
 public final class FSOperations {
 
+  private static final Logger LOG = LoggerFactory.getLogger(FSOperations.class);
+
   private static int bufferSize = 4096;
 
   private FSOperations() {
@@ -716,18 +721,22 @@ public static long copyBytes(InputStream in, OutputStream out, long count)
    */
   @InterfaceAudience.Private
   public static class FSDelete implements FileSystemAccess.FileSystemExecutor<JSONObject> {
-    private Path path;
-    private boolean recursive;
+    private final Path path;
+    private final boolean recursive;
+    private final boolean skipTrash;
 
     /**
      * Creates a Delete executor.
      *
      * @param path path to delete.
      * @param recursive if the delete should be recursive or not.
+     * @param skipTrash if the file must be deleted and not kept in trash
+     *     regardless of fs.trash.interval config value.
      */
-    public FSDelete(String path, boolean recursive) {
+    public FSDelete(String path, boolean recursive, boolean skipTrash) {
       this.path = new Path(path);
       this.recursive = recursive;
+      this.skipTrash = skipTrash;
     }
 
     /**
@@ -742,6 +751,19 @@ public FSDelete(String path, boolean recursive) {
      */
     @Override
     public JSONObject execute(FileSystem fs) throws IOException {
+      if (!skipTrash) {
+        boolean movedToTrash = Trash.moveToAppropriateTrash(fs, path,
+            fs.getConf());
+        if (movedToTrash) {
+          HttpFSServerWebApp.getMetrics().incrOpsDelete();
+          return toJSON(
+              StringUtils.toLowerCase(HttpFSFileSystem.DELETE_JSON), true);
+        }
+        // Same is the behavior with Delete shell command.
+        // If moveToAppropriateTrash() returns false, file deletion
+        // is attempted rather than throwing Error.
+        LOG.debug("Could not move {} to Trash, attempting removal", path);
+      }
       boolean deleted = fs.delete(path, recursive);
       HttpFSServerWebApp.get().getMetrics().incrOpsDelete();
       return toJSON(

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java

@@ -81,7 +81,8 @@ public class HttpFSParametersProvider extends ParametersProvider {
         new Class[]{ReplicationParam.class});
     PARAMS_DEF.put(Operation.SETTIMES,
         new Class[]{ModifiedTimeParam.class, AccessTimeParam.class});
-    PARAMS_DEF.put(Operation.DELETE, new Class[]{RecursiveParam.class});
+    PARAMS_DEF.put(Operation.DELETE, new Class[]{RecursiveParam.class,
+        DeleteSkipTrashParam.class});
     PARAMS_DEF.put(Operation.SETACL, new Class[]{AclPermissionParam.class});
     PARAMS_DEF.put(Operation.REMOVEACL, new Class[]{});
     PARAMS_DEF.put(Operation.MODIFYACLENTRIES,
@@ -241,6 +242,25 @@ public RecursiveParam() {
     }
   }
 
+  /**
+   * Class for delete's skipTrash parameter.
+   */
+  @InterfaceAudience.Private
+  public static class DeleteSkipTrashParam extends BooleanParam {
+
+    /**
+     * Parameter name.
+     */
+    public static final String NAME = HttpFSFileSystem.SKIP_TRASH_PARAM;
+
+    /**
+     * Constructor.
+     */
+    public DeleteSkipTrashParam() {
+      super(NAME, false);
+    }
+  }
+
   /**
    * Class for filter parameter.
    */

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AclPermissionParam;
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam;
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam;
+import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DeleteSkipTrashParam;
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam;
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ECPolicyParam;
 import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam;
@@ -539,9 +540,13 @@ public Response delete(@PathParam("path") String path,
       case DELETE: {
         Boolean recursive =
           params.get(RecursiveParam.NAME,  RecursiveParam.class);
-        AUDIT_LOG.info("[{}] recursive [{}]", path, recursive);
+        Boolean skipTrashParam = params.get(DeleteSkipTrashParam.NAME,
+            DeleteSkipTrashParam.class);
+        boolean skipTrash = skipTrashParam != null && skipTrashParam;
+        AUDIT_LOG.info("[{}] recursive [{}] skipTrash [{}]", path, recursive,
+            skipTrash);
         FSOperations.FSDelete command =
-          new FSOperations.FSDelete(path, recursive);
+          new FSOperations.FSDelete(path, recursive, skipTrash);
         JSONObject json = fsExecute(user, command);
         response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
         break;

hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs.http.server;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -529,6 +530,36 @@ private void createWithHttp(String filename, String perms,
     Assert.assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
   }
 
+  private void deleteWithHttp(String filename, String perms,
+      String unmaskedPerms, Boolean skipTrash) throws Exception {
+    String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
+    // Remove leading / from filename
+    if (filename.charAt(0) == '/') {
+      filename = filename.substring(1);
+    }
+    String pathOps;
+    if (perms == null) {
+      pathOps = MessageFormat.format("/webhdfs/v1/{0}?user.name={1}&op=DELETE",
+          filename, user);
+    } else {
+      pathOps = MessageFormat.format(
+          "/webhdfs/v1/{0}?user.name={1}&permission={2}&op=DELETE",
+          filename, user, perms);
+    }
+    if (unmaskedPerms != null) {
+      pathOps = pathOps + "&unmaskedpermission=" + unmaskedPerms;
+    }
+    if (skipTrash != null) {
+      pathOps = pathOps + "&skiptrash=" + skipTrash;
+    }
+    URL url = new URL(TestJettyHelper.getJettyURL(), pathOps);
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.addRequestProperty("Content-Type", "application/octet-stream");
+    conn.setRequestMethod("DELETE");
+    conn.connect();
+    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+  }
+
   /**
    * Talks to the http interface to create a directory.
    *
@@ -774,6 +805,37 @@ public void testPerms() throws Exception {
     Assert.assertTrue("321".equals(getPerms(statusJson)));
   }
 
+  /**
+   * Validate create and delete calls.
+   */
+  @Test
+  @TestDir
+  @TestJetty
+  @TestHdfs
+  public void testCreateDelete() throws Exception {
+    final String dir1 = "/testCreateDelete1";
+    final String path1 = dir1 + "/file1";
+    final String dir2 = "/testCreateDelete2";
+    final String path2 = dir2 + "/file2";
+
+    createHttpFSServer(false, false);
+    final Configuration conf = HttpFSServerWebApp.get()
+        .get(FileSystemAccess.class).getFileSystemConfiguration();
+    conf.setLong(FS_TRASH_INTERVAL_KEY, 5);
+    writeConf(conf, "hdfs-site.xml");
+
+    FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
+    fs.mkdirs(new Path(dir1));
+
+    createWithHttp(path1, null);
+    deleteWithHttp(path1, null, null, null);
+
+    fs.mkdirs(new Path(dir2));
+
+    createWithHttp(path2, null);
+    deleteWithHttp(path2, null, null, true);
+  }
+
   /**
    * Validate XAttr get/set/remove calls.
    */

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html

@@ -165,11 +165,32 @@ <h4 class="modal-title" id="delete-modal-title">Delete</h4>
         <div class="modal-footer">
           <button type="button" class="btn" data-dismiss="modal">Cancel</button>
           <button type="button" class="btn btn-success" id="delete-button"
-            data-complete-text="Deleting...">Delete</button>
+                  data-complete-text="Deleting..." data-toggle="modal" data-target="#delete-trash-modal">Delete
+          </button>
         </div>
       </div>
     </div>
   </div>
+      <div class="modal" id="delete-trash-modal" tabindex="-1" role="dialog" aria-hidden="true">
+        <div class="modal-dialog">
+          <div class="modal-content">
+            <div class="modal-header">
+              <button type="button" class="close" data-dismiss="modal"
+                      aria-hidden="true">&times;</button>
+              <h4 class="modal-title" id="delete-trash-modal-title">Delete Trash</h4>
+            </div>
+            <div class="modal-body">
+              <div class="panel-body">
+                <div id="delete-trash-prompt"></div>
+              </div>
+            </div>
+            <div class="modal-footer">
+              <button type="button" class="btn" id="skip-trash-button">Yes (Skip Trash)</button>
+              <button type="button" class="btn btn-success" id="trash-button">No</button>
+            </div>
+          </div>
+        </div>
+      </div>
 
       <div class="row">
       <div class="col-xs-9 col-md-9">

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js

@@ -82,23 +82,47 @@
   function delete_path(inode_name, absolute_file_path) {
     $('#delete-modal-title').text("Delete - " + inode_name);
     $('#delete-prompt').text("Are you sure you want to delete " + inode_name
-      + " ?");
-
-    $('#delete-button').click(function() {
+        + " ?");
+    $('#delete-trash-modal-title').text("Skip Trash - " + inode_name);
+    $('#delete-trash-prompt').text("Skipping Trash might delete file forever."
+        + " Do you want to skip-trash " + inode_name
+        + " ? (default behaviour - No)");
+
+    $('#skip-trash-button').click(function () {
+      // DELETE /webhdfs/v1/<path>?op=DELETE&recursive=<true|false>&skiptrash=true
+      var url = '/webhdfs/v1' + encode_path(absolute_file_path) +
+          '?op=DELETE' + '&recursive=true&skiptrash=true';
+      $.ajax(url,
+          {
+            type: 'DELETE'
+          }).done(function (data) {
+        browse_directory(current_directory);
+      }).fail(network_error_handler(url)
+      ).always(function () {
+        $('#delete-modal').modal('hide');
+        $('#delete-button').button('reset');
+        $('#delete-trash-modal').modal('hide');
+        $('#skip-trash-button').button('reset');
+      });
+    })
+    $('#trash-button').click(function () {
       // DELETE /webhdfs/v1/<path>?op=DELETE&recursive=<true|false>
       var url = '/webhdfs/v1' + encode_path(absolute_file_path) +
-        '?op=DELETE' + '&recursive=true';
-
+          '?op=DELETE' + '&recursive=true';
       $.ajax(url,
-        { type: 'DELETE'
-        }).done(function(data) {
-          browse_directory(current_directory);
-        }).fail(network_error_handler(url)
-         ).always(function() {
-           $('#delete-modal').modal('hide');
-           $('#delete-button').button('reset');
-        });
+          {
+            type: 'DELETE'
+          }).done(function (data) {
+        browse_directory(current_directory);
+      }).fail(network_error_handler(url)
+      ).always(function () {
+        $('#delete-modal').modal('hide');
+        $('#delete-button').button('reset');
+        $('#delete-trash-modal').modal('hide');
+        $('#trash-button').button('reset');
+      });
     })
+
     $('#delete-modal').modal();
   }