From e3c9d07d872e647e79c0148a49ad1aa6a53224a7 Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 7 Jun 2023 17:07:36 +0530 Subject: [PATCH 01/11] HDDS-8778. Support recursive volume delete using Ozone sh command. --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 90 +++++++++ .../apache/hadoop/ozone/shell/Handler.java | 10 + .../shell/volume/DeleteVolumeHandler.java | 188 +++++++++++++++++- 3 files changed, 286 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index fee69a9847c8..4ca73c2de209 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.shell.s3.S3Shell; import org.apache.hadoop.security.UserGroupInformation; @@ -1403,4 +1404,93 @@ public void testClientBucketLayoutValidation() { e); } } + + @Test + public void testRecursiveVolumeDelete() + throws Exception { + String volume1 = "volume1"; + String volume2 = "volume2"; + + // Create volume volume1 + // Create bucket bucket1 with layout FILE_SYSTEM_OPTIMIZED + // Insert some keys into it + generateKeys(OZONE_URI_DELIMITER + volume1, + "/bucketfso", + BucketLayout.FILE_SYSTEM_OPTIMIZED.toString()); + + // Create another volume volume2 with bucket and some keys into it. + generateKeys(OZONE_URI_DELIMITER + volume2, + "/bucket2", + BucketLayout.FILE_SYSTEM_OPTIMIZED.toString()); + + // Create OBS bucket in volume1 + String[] args = new String[] {"bucket", "create", volume1 + "/bucketobs"}; + execute(ozoneShell, args); + out.reset(); + + // Insert few keys into OBS bucket + String keyName = OZONE_URI_DELIMITER + volume1 + "/bucketobs" + + OZONE_URI_DELIMITER + "key"; + for (int i = 0; i < 5; i++) { + args = new String[] { + "key", "put", "o3://" + omServiceId + keyName + i, + testFile.getPath()}; + execute(ozoneShell, args); + } + out.reset(); + + // Create Legacy bucket in volume1 + args = new String[] {"bucket", "create", volume1 + "/bucketlegacy"}; + execute(ozoneShell, args); + out.reset(); + + // Insert few keys into legacy bucket + keyName = OZONE_URI_DELIMITER + volume1 + "/bucketlegacy" + + OZONE_URI_DELIMITER + "key"; + for (int i = 0; i < 5; i++) { + args = new String[] { + "key", "put", "o3://" + omServiceId + keyName + i, + testFile.getPath()}; + execute(ozoneShell, args); + } + out.reset(); + + // Try volume delete without recursive + // It should fail as volume is not empty + final String[] args1 = new String[] {"volume", "delete", volume1}; + LambdaTestUtils.intercept(ExecutionException.class, + "VOLUME_NOT_EMPTY", () -> execute(ozoneShell, args1)); + out.reset(); + + // volume1 should still exist + Assert.assertEquals(client.getObjectStore().getVolume(volume1) + .getName(), volume1); + + args = + new String[] {"volume", + "delete", volume1, "-r", "-id", omServiceId}; + // Try recursive delete without skipTrash, volume should not be deleted. + execute(ozoneShell, args); + out.reset(); + + // volume1 should still exist + Assert.assertEquals(client.getObjectStore().getVolume(volume1) + .getName(), volume1); + + // Delete volume1(containing OBS, FSO and Legacy buckets) + // recursively with skipTrash + args = + new String[] {"volume", + "delete", volume1, "-r", "-skipTrash", "-id", omServiceId}; + + execute(ozoneShell, args); + out.reset(); + // volume2 should still exist + Assert.assertEquals(client.getObjectStore().getVolume(volume2) + .getName(), volume2); + + // volume1 should not exist + LambdaTestUtils.intercept(OMException.class, + "VOLUME_NOT_FOUND", () -> client.getObjectStore().getVolume(volume1)); + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java index ae5edf5b1f77..bcbc0e0f024e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java @@ -56,6 +56,12 @@ public abstract class Handler implements Callable { @CommandLine.Spec private CommandLine.Model.CommandSpec spec; + @CommandLine.Option( + names = {"-id", "--service-id"}, + description = "Ozone Manager Service ID" + ) + private String omServiceId; + public boolean isVerbose() { return parent.isVerbose(); } @@ -143,6 +149,10 @@ protected OzoneConfiguration getConf() { return conf; } + protected String getOmServiceId() { + return omServiceId; + } + protected PrintStream out() { return System.out; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index 29f9e39132e5..8a0acb2139e3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -18,27 +18,211 @@ package org.apache.hadoop.ozone.shell.volume; +import com.google.common.base.Strings; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.shell.OzoneAddress; +import picocli.CommandLine; import picocli.CommandLine.Command; import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; /** * Executes deleteVolume call for the shell. */ @Command(name = "delete", - description = "deletes a volume if it is empty") + description = "deletes a volume") public class DeleteVolumeHandler extends VolumeHandler { + @CommandLine.Option( + names = {"-skipTrash"}, + description = "Delete volume without trash" + ) + private boolean bSkipTrash = false; + @CommandLine.Option( + names = {"-r"}, + description = "Delete volume recursively" + ) + private boolean bRecursive = false; + private ExecutorService executor; + private List bucketIdList = new ArrayList<>(); + private AtomicInteger cleanedBucketCounter = + new AtomicInteger(); + private int totalBucketCount; + private OzoneVolume vol; + private AtomicInteger numberOfBucketsCleaned = new AtomicInteger(0); + private volatile Throwable exception; + private static final int MAX_KEY_DELETE_BATCH_SIZE = 1000; @Override protected void execute(OzoneClient client, OzoneAddress address) throws IOException { String volumeName = address.getVolumeName(); - + try { + if (bRecursive) { + if (!bSkipTrash) { + out().printf("Use -skipTrash for recursive volume delete%n"); + return; + } + if (Strings.isNullOrEmpty(getOmServiceId())) { + out().printf("OmServiceID not provided, provide using " + + "-id %n"); + return; + } + vol = client.getObjectStore().getVolume(volumeName); + deleteVolumeRecursive(); + } + } catch (InterruptedException e) { + out().printf("Exception while deleting volume recursively%n"); + return; + } client.getObjectStore().deleteVolume(volumeName); out().printf("Volume %s is deleted%n", volumeName); } + + private void deleteVolumeRecursive() + throws InterruptedException { + // Get all the buckets for given volume + Iterator bucketIterator = + vol.listBuckets(null); + + while (bucketIterator.hasNext()) { + OzoneBucket bucket = bucketIterator.next(); + bucketIdList.add(bucket.getName()); + totalBucketCount++; + } + doCleanBuckets(); + } + + /** + * Clean OBS bucket recursively. + * + * @param bucket OzoneBucket + * @return boolean + */ + private boolean cleanOBSBucket(OzoneBucket bucket) { + ArrayList keys = new ArrayList<>(); + try { + if (!bucket.isLink()) { + Iterator iterator = bucket.listKeys(null); + while (iterator.hasNext()) { + keys.add(iterator.next().getName()); + if (MAX_KEY_DELETE_BATCH_SIZE == keys.size()) { + bucket.deleteKeys(keys); + keys.clear(); + } + } + // delete if any remaining keys left + if (keys.size() > 0) { + bucket.deleteKeys(keys); + } + } + vol.deleteBucket(bucket.getName()); + numberOfBucketsCleaned.getAndIncrement(); + return true; + } catch (Exception e) { + LOG.error("Could not clean bucket ", e); + return false; + } + } + + /** + * Clean Legacy/FSO bucket recursively. + * + * @param bucket OzoneBucket + * @return boolean + */ + private boolean cleanFSBucket(OzoneBucket bucket) { + try { + final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + + getOmServiceId() + PATH_SEPARATOR_STR; + String ofsPrefix = hostPrefix + vol.getName() + PATH_SEPARATOR_STR + + bucket.getName(); + final Path path = new Path(ofsPrefix); + OzoneConfiguration clientConf = new OzoneConfiguration(getConf()); + clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix); + clientConf.setInt(FS_TRASH_INTERVAL_KEY, 0); + FileSystem fs = FileSystem.get(clientConf); + if (!fs.delete(path, true)) { + throw new IOException("Failed to delete bucket"); + } + numberOfBucketsCleaned.getAndIncrement(); + return true; + } catch (Exception e) { + exception = e; + LOG.error("Could not clean bucket ", e); + return false; + } + } + + private class BucketCleaner implements Runnable { + @Override + public void run() { + int i; + while ((i = cleanedBucketCounter.getAndIncrement()) < totalBucketCount) { + try { + OzoneBucket bucket = vol.getBucket(bucketIdList.get(i)); + switch (bucket.getBucketLayout()) { + case FILE_SYSTEM_OPTIMIZED: + case LEGACY: + if (!cleanFSBucket(bucket)) { + throw new RuntimeException("Failed to clean bucket"); + } + break; + case OBJECT_STORE: + if (!cleanOBSBucket(bucket)) { + throw new RuntimeException("Failed to clean bucket"); + } + default: + throw new RuntimeException("Invalid bucket layout"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + + private void doCleanBuckets() throws InterruptedException { + int threadPoolSize = 10; + executor = Executors.newFixedThreadPool(threadPoolSize); + for (int i = 0; i < threadPoolSize; i++) { + executor.execute(new BucketCleaner()); + } + + try { + // wait until all Buckets are cleaned or exception occurred. + while (numberOfBucketsCleaned.get() != totalBucketCount + && exception == null) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw e; + } + } + } catch (InterruptedException e) { + LOG.error("Failed to wait until all Buckets are cleaned", e); + Thread.currentThread().interrupt(); + } + executor.shutdown(); + executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS); + } } From 25fde6393ba0be6aeb73564aabe65af76f7c21e6 Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 7 Jun 2023 18:11:22 +0530 Subject: [PATCH 02/11] HDDS-8778. Fix findbug issue. --- .../java/org/apache/hadoop/ozone/shell/Handler.java | 10 ---------- .../hadoop/ozone/shell/volume/DeleteVolumeHandler.java | 9 +++++++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java index bcbc0e0f024e..ae5edf5b1f77 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java @@ -56,12 +56,6 @@ public abstract class Handler implements Callable { @CommandLine.Spec private CommandLine.Model.CommandSpec spec; - @CommandLine.Option( - names = {"-id", "--service-id"}, - description = "Ozone Manager Service ID" - ) - private String omServiceId; - public boolean isVerbose() { return parent.isVerbose(); } @@ -149,10 +143,6 @@ protected OzoneConfiguration getConf() { return conf; } - protected String getOmServiceId() { - return omServiceId; - } - protected PrintStream out() { return System.out; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index 8a0acb2139e3..abbbc430d38b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -61,6 +61,11 @@ public class DeleteVolumeHandler extends VolumeHandler { description = "Delete volume recursively" ) private boolean bRecursive = false; + @CommandLine.Option( + names = {"-id", "--om-service-id"}, + description = "Ozone Manager Service ID" + ) + private String omServiceId; private ExecutorService executor; private List bucketIdList = new ArrayList<>(); private AtomicInteger cleanedBucketCounter = @@ -82,7 +87,7 @@ protected void execute(OzoneClient client, OzoneAddress address) out().printf("Use -skipTrash for recursive volume delete%n"); return; } - if (Strings.isNullOrEmpty(getOmServiceId())) { + if (Strings.isNullOrEmpty(omServiceId)) { out().printf("OmServiceID not provided, provide using " + "-id %n"); return; @@ -153,7 +158,7 @@ private boolean cleanOBSBucket(OzoneBucket bucket) { private boolean cleanFSBucket(OzoneBucket bucket) { try { final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + - getOmServiceId() + PATH_SEPARATOR_STR; + omServiceId + PATH_SEPARATOR_STR; String ofsPrefix = hostPrefix + vol.getName() + PATH_SEPARATOR_STR + bucket.getName(); final Path path = new Path(ofsPrefix); From 2609247064f3a61b89bb78195e62dbc8c6bfc901 Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 7 Jun 2023 18:48:48 +0530 Subject: [PATCH 03/11] HDDS-8778. Fix ServiceID issue. --- .../hadoop/ozone/shell/volume/DeleteVolumeHandler.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index abbbc430d38b..cb0b2d0359f8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; @@ -87,7 +88,8 @@ protected void execute(OzoneClient client, OzoneAddress address) out().printf("Use -skipTrash for recursive volume delete%n"); return; } - if (Strings.isNullOrEmpty(omServiceId)) { + if (OmUtils.isServiceIdsDefined(getConf()) && + Strings.isNullOrEmpty(omServiceId)) { out().printf("OmServiceID not provided, provide using " + "-id %n"); return; @@ -157,8 +159,10 @@ private boolean cleanOBSBucket(OzoneBucket bucket) { */ private boolean cleanFSBucket(OzoneBucket bucket) { try { - final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + - omServiceId + PATH_SEPARATOR_STR; + String hostPrefix = OZONE_OFS_URI_SCHEME + "://"; + if (!Strings.isNullOrEmpty(omServiceId)) { + hostPrefix += omServiceId + PATH_SEPARATOR_STR; + } String ofsPrefix = hostPrefix + vol.getName() + PATH_SEPARATOR_STR + bucket.getName(); final Path path = new Path(ofsPrefix); From 143f819cd4cc286be06d60418899c2e615a76f71 Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 7 Jun 2023 21:03:18 +0530 Subject: [PATCH 04/11] HDDS-8778. Fix Test case. --- .../java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 4ca73c2de209..02ffc57fae6d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1408,8 +1408,8 @@ public void testClientBucketLayoutValidation() { @Test public void testRecursiveVolumeDelete() throws Exception { - String volume1 = "volume1"; - String volume2 = "volume2"; + String volume1 = "volume10"; + String volume2 = "volume20"; // Create volume volume1 // Create bucket bucket1 with layout FILE_SYSTEM_OPTIMIZED From 5d9b760cf1c5b941d7e2fc8c7b8fdd6acc2f425d Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 7 Jun 2023 21:34:08 +0530 Subject: [PATCH 05/11] HDDS-8778. Fix address issue. --- .../apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index cb0b2d0359f8..e296a6df40f4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -45,6 +45,7 @@ import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; /** * Executes deleteVolume call for the shell. @@ -162,6 +163,9 @@ private boolean cleanFSBucket(OzoneBucket bucket) { String hostPrefix = OZONE_OFS_URI_SCHEME + "://"; if (!Strings.isNullOrEmpty(omServiceId)) { hostPrefix += omServiceId + PATH_SEPARATOR_STR; + } else { + hostPrefix += getConf().get(OZONE_OM_ADDRESS_KEY) + + PATH_SEPARATOR_STR; } String ofsPrefix = hostPrefix + vol.getName() + PATH_SEPARATOR_STR + bucket.getName(); From 6c06e1b4d0a8ef2c6d83fd8c2d8ed341a208b0d4 Mon Sep 17 00:00:00 2001 From: ashishk Date: Thu, 8 Jun 2023 16:43:03 +0530 Subject: [PATCH 06/11] HDDS-8778. Fix review comments. --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 8 ++--- .../shell/volume/DeleteVolumeHandler.java | 32 ++++++++++++++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 02ffc57fae6d..95894c842866 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1467,8 +1467,8 @@ public void testRecursiveVolumeDelete() .getName(), volume1); args = - new String[] {"volume", - "delete", volume1, "-r", "-id", omServiceId}; + new String[] {"volume", "delete", volume1, "-r", + "-yes", "-id", omServiceId}; // Try recursive delete without skipTrash, volume should not be deleted. execute(ozoneShell, args); out.reset(); @@ -1480,8 +1480,8 @@ public void testRecursiveVolumeDelete() // Delete volume1(containing OBS, FSO and Legacy buckets) // recursively with skipTrash args = - new String[] {"volume", - "delete", volume1, "-r", "-skipTrash", "-id", omServiceId}; + new String[] {"volume", "delete", volume1, "-r", + "-skipTrash", "-yes", "-id", omServiceId}; execute(ozoneShell, args); out.reset(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index e296a6df40f4..c02cf844ce5c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -33,15 +33,17 @@ import picocli.CommandLine.Command; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Scanner; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; @@ -68,6 +70,15 @@ public class DeleteVolumeHandler extends VolumeHandler { description = "Ozone Manager Service ID" ) private String omServiceId; + + @CommandLine.Option(names = {"-t", "--threads", "--thread"}, + description = "Number of threads used to execute") + private int threadNo = 10; + + @CommandLine.Option(names = "-yes", + description = "Continue without interactive user confirmation") + private boolean yes; + private ExecutorService executor; private List bucketIdList = new ArrayList<>(); private AtomicInteger cleanedBucketCounter = @@ -96,6 +107,19 @@ protected void execute(OzoneClient client, OzoneAddress address) return; } vol = client.getObjectStore().getVolume(volumeName); + if (!yes) { + // Ask for user confirmation + out().print("Enter 'yes' to confirm recursive volume delete '" + + volumeName + "': "); + out().flush(); + Scanner scanner = new Scanner(new InputStreamReader( + System.in, StandardCharsets.UTF_8)); + String confirmation = scanner.next().trim().toLowerCase(); + if (!confirmation.equals("yes")) { + out().println("Operation cancelled."); + return; + } + } deleteVolumeRecursive(); } } catch (InterruptedException e) { @@ -172,7 +196,6 @@ private boolean cleanFSBucket(OzoneBucket bucket) { final Path path = new Path(ofsPrefix); OzoneConfiguration clientConf = new OzoneConfiguration(getConf()); clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix); - clientConf.setInt(FS_TRASH_INTERVAL_KEY, 0); FileSystem fs = FileSystem.get(clientConf); if (!fs.delete(path, true)) { throw new IOException("Failed to delete bucket"); @@ -215,9 +238,8 @@ public void run() { } private void doCleanBuckets() throws InterruptedException { - int threadPoolSize = 10; - executor = Executors.newFixedThreadPool(threadPoolSize); - for (int i = 0; i < threadPoolSize; i++) { + executor = Executors.newFixedThreadPool(threadNo); + for (int i = 0; i < threadNo; i++) { executor.execute(new BucketCleaner()); } From 21098fe86b07689779842c34d2fc4b8f8ebcd1bd Mon Sep 17 00:00:00 2001 From: ashishk Date: Mon, 19 Jun 2023 09:27:57 +0530 Subject: [PATCH 07/11] HDDS-8778. Fix review comments. --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 18 ++------ .../shell/volume/DeleteVolumeHandler.java | 42 ++++--------------- 2 files changed, 11 insertions(+), 49 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 9b0b94623e4d..ca63aadbe476 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1524,22 +1524,10 @@ public void testRecursiveVolumeDelete() Assert.assertEquals(client.getObjectStore().getVolume(volume1) .getName(), volume1); + // Delete volume1(containing OBS, FSO and Legacy buckets) recursively args = - new String[] {"volume", "delete", volume1, "-r", - "-yes", "-id", omServiceId}; - // Try recursive delete without skipTrash, volume should not be deleted. - execute(ozoneShell, args); - out.reset(); - - // volume1 should still exist - Assert.assertEquals(client.getObjectStore().getVolume(volume1) - .getName(), volume1); - - // Delete volume1(containing OBS, FSO and Legacy buckets) - // recursively with skipTrash - args = - new String[] {"volume", "delete", volume1, "-r", - "-skipTrash", "-yes", "-id", omServiceId}; + new String[] {"volume", "delete", volume1, "-r=yes", + "-id", omServiceId}; execute(ozoneShell, args); out.reset(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index c02cf844ce5c..780a616ffb5b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -33,12 +33,9 @@ import picocli.CommandLine.Command; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Scanner; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -55,30 +52,24 @@ @Command(name = "delete", description = "deletes a volume") public class DeleteVolumeHandler extends VolumeHandler { - @CommandLine.Option( - names = {"-skipTrash"}, - description = "Delete volume without trash" - ) - private boolean bSkipTrash = false; @CommandLine.Option( names = {"-r"}, - description = "Delete volume recursively" + description = "This command will delete volume recursively." + + "\nThere is no trash recovery for FSO buckets using this command." + + "\nDelay is expected running this command." + + "\nEnter 'yes' to proceed", + interactive = true ) - private boolean bRecursive = false; + private String isRecursive; @CommandLine.Option( names = {"-id", "--om-service-id"}, description = "Ozone Manager Service ID" ) private String omServiceId; - + @CommandLine.Option(names = {"-t", "--threads", "--thread"}, description = "Number of threads used to execute") private int threadNo = 10; - - @CommandLine.Option(names = "-yes", - description = "Continue without interactive user confirmation") - private boolean yes; - private ExecutorService executor; private List bucketIdList = new ArrayList<>(); private AtomicInteger cleanedBucketCounter = @@ -95,11 +86,7 @@ protected void execute(OzoneClient client, OzoneAddress address) String volumeName = address.getVolumeName(); try { - if (bRecursive) { - if (!bSkipTrash) { - out().printf("Use -skipTrash for recursive volume delete%n"); - return; - } + if (!Strings.isNullOrEmpty(isRecursive) && isRecursive.equals("yes")) { if (OmUtils.isServiceIdsDefined(getConf()) && Strings.isNullOrEmpty(omServiceId)) { out().printf("OmServiceID not provided, provide using " + @@ -107,19 +94,6 @@ protected void execute(OzoneClient client, OzoneAddress address) return; } vol = client.getObjectStore().getVolume(volumeName); - if (!yes) { - // Ask for user confirmation - out().print("Enter 'yes' to confirm recursive volume delete '" + - volumeName + "': "); - out().flush(); - Scanner scanner = new Scanner(new InputStreamReader( - System.in, StandardCharsets.UTF_8)); - String confirmation = scanner.next().trim().toLowerCase(); - if (!confirmation.equals("yes")) { - out().println("Operation cancelled."); - return; - } - } deleteVolumeRecursive(); } } catch (InterruptedException e) { From 800033bb5ea24d7ad970e14331200bde7788c5f5 Mon Sep 17 00:00:00 2001 From: ashishk Date: Mon, 19 Jun 2023 15:52:59 +0530 Subject: [PATCH 08/11] HDDS-8778. Update bucket layout in test case. --- .../apache/hadoop/ozone/shell/TestOzoneShellHA.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index ca63aadbe476..aec1d17cb36c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1417,7 +1417,8 @@ public void testVolumeListKeys() BucketLayout.FILE_SYSTEM_OPTIMIZED.toString()); // Create OBS bucket in volx - String[] args = new String[]{"bucket", "create", volume1 + "/bucketobs"}; + String[] args = new String[]{"bucket", "create", "--layout", + BucketLayout.OBJECT_STORE.toString(), volume1 + "/bucketobs"}; execute(ozoneShell, args); out.reset(); @@ -1433,7 +1434,8 @@ public void testVolumeListKeys() out.reset(); // Create Legacy bucket in volx - args = new String[]{"bucket", "create", volume1 + "/bucketlegacy"}; + args = new String[]{"bucket", "create", "--layout", + BucketLayout.LEGACY.toString(), volume1 + "/bucketlegacy"}; execute(ozoneShell, args); out.reset(); @@ -1482,7 +1484,8 @@ public void testRecursiveVolumeDelete() BucketLayout.FILE_SYSTEM_OPTIMIZED.toString()); // Create OBS bucket in volume1 - String[] args = new String[] {"bucket", "create", volume1 + "/bucketobs"}; + String[] args = new String[] {"bucket", "create", "--layout", + BucketLayout.OBJECT_STORE.toString(), volume1 + "/bucketobs"}; execute(ozoneShell, args); out.reset(); @@ -1498,7 +1501,8 @@ public void testRecursiveVolumeDelete() out.reset(); // Create Legacy bucket in volume1 - args = new String[] {"bucket", "create", volume1 + "/bucketlegacy"}; + args = new String[] {"bucket", "create", "--layout", + BucketLayout.LEGACY.toString(), volume1 + "/bucketlegacy"}; execute(ozoneShell, args); out.reset(); From 87a65873b2a894c7ec1b391960b9f60800e5700d Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 21 Jun 2023 10:06:51 +0530 Subject: [PATCH 09/11] HDDS-8778. Update user interactive option and message. --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 2 +- .../shell/volume/DeleteVolumeHandler.java | 34 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index aec1d17cb36c..b093584a2118 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1530,7 +1530,7 @@ public void testRecursiveVolumeDelete() // Delete volume1(containing OBS, FSO and Legacy buckets) recursively args = - new String[] {"volume", "delete", volume1, "-r=yes", + new String[] {"volume", "delete", volume1, "-r", "--yes", "-id", omServiceId}; execute(ozoneShell, args); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index 780a616ffb5b..4d19a0863df0 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -33,9 +33,12 @@ import picocli.CommandLine.Command; import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Scanner; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -54,13 +57,10 @@ public class DeleteVolumeHandler extends VolumeHandler { @CommandLine.Option( names = {"-r"}, - description = "This command will delete volume recursively." + - "\nThere is no trash recovery for FSO buckets using this command." + - "\nDelay is expected running this command." + - "\nEnter 'yes' to proceed", - interactive = true + description = "Delete volume recursively" ) - private String isRecursive; + private boolean bRecursive; + @CommandLine.Option( names = {"-id", "--om-service-id"}, description = "Ozone Manager Service ID" @@ -70,6 +70,10 @@ public class DeleteVolumeHandler extends VolumeHandler { @CommandLine.Option(names = {"-t", "--threads", "--thread"}, description = "Number of threads used to execute") private int threadNo = 10; + + @CommandLine.Option(names = {"-y", "--yes"}, + description = "Continue without interactive user confirmation") + private boolean yes; private ExecutorService executor; private List bucketIdList = new ArrayList<>(); private AtomicInteger cleanedBucketCounter = @@ -86,13 +90,29 @@ protected void execute(OzoneClient client, OzoneAddress address) String volumeName = address.getVolumeName(); try { - if (!Strings.isNullOrEmpty(isRecursive) && isRecursive.equals("yes")) { + if (bRecursive) { if (OmUtils.isServiceIdsDefined(getConf()) && Strings.isNullOrEmpty(omServiceId)) { out().printf("OmServiceID not provided, provide using " + "-id %n"); return; } + if (!yes) { + // Ask for user confirmation + out().print("This command will delete volume recursively." + + "\nThere is no recovery option after using this command, " + + "and no trash for FSO buckets." + + "\nDelay is expected running this command." + + "\nEnter 'yes' to proceed': "); + out().flush(); + Scanner scanner = new Scanner(new InputStreamReader( + System.in, StandardCharsets.UTF_8)); + String confirmation = scanner.next().trim().toLowerCase(); + if (!confirmation.equals("yes")) { + out().println("Operation cancelled."); + return; + } + } vol = client.getObjectStore().getVolume(volumeName); deleteVolumeRecursive(); } From ec50b581861d53f215aa6b2bf1e64adaf17db79e Mon Sep 17 00:00:00 2001 From: ashishk Date: Wed, 21 Jun 2023 12:27:45 +0530 Subject: [PATCH 10/11] HDDS-8778. Update thread description. --- .../apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index 4d19a0863df0..1cab23c82bb0 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -68,7 +68,7 @@ public class DeleteVolumeHandler extends VolumeHandler { private String omServiceId; @CommandLine.Option(names = {"-t", "--threads", "--thread"}, - description = "Number of threads used to execute") + description = "Number of threads used to execute recursive delete") private int threadNo = 10; @CommandLine.Option(names = {"-y", "--yes"}, From 41cd4580a955a2057ad2de0a45f79ae26c405645 Mon Sep 17 00:00:00 2001 From: ashishk Date: Fri, 23 Jun 2023 14:54:26 +0530 Subject: [PATCH 11/11] HDDS-8778. Handling for single serviceID. --- .../shell/volume/DeleteVolumeHandler.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java index 1cab23c82bb0..691aaede7a5a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java @@ -22,7 +22,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; @@ -36,6 +35,7 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Scanner; @@ -48,6 +48,7 @@ import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; /** * Executes deleteVolume call for the shell. @@ -91,11 +92,17 @@ protected void execute(OzoneClient client, OzoneAddress address) String volumeName = address.getVolumeName(); try { if (bRecursive) { - if (OmUtils.isServiceIdsDefined(getConf()) && - Strings.isNullOrEmpty(omServiceId)) { - out().printf("OmServiceID not provided, provide using " + - "-id %n"); - return; + Collection serviceIds = getConf().getTrimmedStringCollection( + OZONE_OM_SERVICE_IDS_KEY); + if (Strings.isNullOrEmpty(omServiceId)) { + if (serviceIds.size() > 1) { + out().printf("OmServiceID not provided, provide using " + + "-id %n"); + return; + } else if (serviceIds.size() == 1) { + // Only one OM service ID configured, we can use that + omServiceId = serviceIds.iterator().next(); + } } if (!yes) { // Ask for user confirmation