From 0b04c9694da86941e04a480d55913701f9fe514d Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Fri, 23 Apr 2021 19:52:41 +0900 Subject: [PATCH 01/48] HADOOP-17655. Upgrade Jetty to 9.4.40. (#2947) Reviewed-by: Wei-Chiu Chuang --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index f09bf533beaab..d91131e6b980b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.39.v20210325 + 9.4.40.v20210413 _ _ From f76a2a7606760eaf9967dae7ab769d22b11b4447 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 23 Apr 2021 16:07:58 +0200 Subject: [PATCH 02/48] YARN-10654. Dots '.' in CSMappingRule path variables should be replaced. Contributed by Peter Bacsko --- .../placement/CSMappingPlacementRule.java | 22 ++++-- .../TestCSMappingPlacementRule.java | 67 +++++++++++++++++-- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java index 894bc8262221a..9648769231c96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java @@ -42,8 +42,6 @@ import java.util.List; import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; - /** * This class is responsible for making application submissions to queue * assignments, based on the configured ruleset. This class supports all @@ -55,6 +53,8 @@ public class CSMappingPlacementRule extends PlacementRule { private static final Logger LOG = LoggerFactory .getLogger(CSMappingPlacementRule.class); + private static final String DOT = "."; + private static final String DOT_REPLACEMENT = "_dot_"; private CapacitySchedulerQueueManager queueManager; private List mappingRules; @@ -194,12 +194,13 @@ private void setupGroupsForVariableContext(VariableContext vctx, String user) return; } Iterator it = groupsSet.iterator(); - String primaryGroup = it.next(); + String primaryGroup = cleanName(it.next()); ArrayList secondaryGroupList = new ArrayList<>(); while (it.hasNext()) { - secondaryGroupList.add(it.next()); + String groupName = cleanName(it.next()); + secondaryGroupList.add(groupName); } if (secondaryGroupList.size() == 0) { @@ -226,7 +227,7 @@ private VariableContext createVariableContext( ApplicationSubmissionContext asc, String user) { VariableContext vctx = new VariableContext(); - vctx.put("%user", user); + vctx.put("%user", cleanName(user)); //If the specified matches the default it means NO queue have been specified //as per ClientRMService#submitApplication which sets the queue to default //when no queue is provided. @@ -522,4 +523,15 @@ private ApplicationPlacementContext placeToDefault( " mapping rule. Please see the logs for details"); } } + + private String cleanName(String name) { + if (name.contains(DOT)) { + String converted = name.replaceAll("\\.", DOT_REPLACEMENT); + LOG.warn("Name {} is converted to {} when it is used as a queue name.", + name, converted); + return converted; + } else { + return name; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java index f6bb2a1587836..0cf10595ee9dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java @@ -63,13 +63,15 @@ public class TestCSMappingPlacementRule { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private Map> userGroups = ImmutableMap.of( - "alice", ImmutableSet.of("p_alice", "unique", "user"), - "bob", ImmutableSet.of("p_bob", "user", "developer"), - "charlie", ImmutableSet.of("p_charlie", "user", "tester"), - "dave", ImmutableSet.of("user"), - "emily", ImmutableSet.of("user", "tester", "developer") - ); + private Map> userGroups = + ImmutableMap.>builder() + .put("alice", ImmutableSet.of("p_alice", "unique", "user")) + .put("bob", ImmutableSet.of("p_bob", "user", "developer")) + .put("charlie", ImmutableSet.of("p_charlie", "user", "tester")) + .put("dave", ImmutableSet.of("user")) + .put("emily", ImmutableSet.of("user", "tester", "developer")) + .put("test.user", ImmutableSet.of("main.grp", "sec.test.grp")) + .build(); private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) { MockQueueHierarchyBuilder.create() @@ -79,6 +81,9 @@ private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) { .withManagedParentQueue("root.man") .withQueue("root.user.alice") .withQueue("root.user.bob") + .withQueue("root.user.test_dot_user") + .withQueue("root.groups.main_dot_grp") + .withQueue("root.groups.sec_dot_test_dot_grp") .withQueue("root.secondaryTests.unique") .withQueue("root.secondaryTests.user") .withQueue("root.ambiguous.user.charlie") @@ -718,4 +723,52 @@ public void testJSONFileConfiguration() throws IOException { assertConfigTestResult(rules); } + + @Test + public void testUserNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%user", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "test.user should be placed to root.users.test_dot_user", + engine, app, "test.user", "root.user.test_dot_user"); + } + + @Test + public void testPrimaryGroupNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%primary_group", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "Application should have been placed to root.groups.main_dot_grp", + engine, app, "test.user", "root.groups.main_dot_grp"); + } + + @Test + public void testSecondaryGroupNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%secondary_group", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "Application should have been placed to root.groups.sec_dot_test_dot_grp", + engine, app, "test.user", "root.groups.sec_dot_test_dot_grp"); + } } \ No newline at end of file From 919daec36b23457c107e819855791e7e6d144594 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 23 Apr 2021 16:12:12 +0200 Subject: [PATCH 03/48] YARN-10746. RmWebApp add default-node-label-expression to the queue info. Contributed by Gergely Pollak --- .../webapp/dao/CapacitySchedulerInfo.java | 3 ++ .../dao/CapacitySchedulerLeafQueueInfo.java | 12 ++--- .../dao/CapacitySchedulerQueueInfo.java | 8 ++- .../TestRMWebServicesCapacitySched.java | 52 +++++++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index fc18d4d6e4428..78e2133a90b2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,6 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected String queueType; protected String creationMethod; protected String autoCreationEligibility; + protected String defaultNodeLabelExpression; @XmlTransient static final float EPSILON = 1e-8f; @@ -114,6 +115,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent); autoCreationEligibility = CapacitySchedulerInfoHelper .getAutoCreationEligibility(parent); + + defaultNodeLabelExpression = parent.getDefaultNodeLabelExpression(); } public float getCapacity() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 6aef01bb3dd1c..16abb991df90d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -52,7 +52,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected ResourceInfo userAMResourceLimit; protected boolean preemptionDisabled; protected boolean intraQueuePreemptionDisabled; - protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; protected long maxApplicationLifetime; @@ -81,7 +80,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { intraQueuePreemptionDisabled = q.getIntraQueuePreemptionDisabled(); orderingPolicyDisplayName = q.getOrderingPolicy().getInfo(); orderingPolicyInfo = q.getOrderingPolicy().getConfigName(); - defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); defaultPriority = q.getDefaultApplicationPriority().getPriority(); ArrayList usersList = users.getUsersList(); if (usersList.isEmpty()) { @@ -148,17 +146,17 @@ public float getUserLimitFactor() { public float getConfiguredMaxAMResourceLimit() { return configuredMaxAMResourceLimit; } - + public ResourceInfo getAMResourceLimit() { return AMResourceLimit; } - + public ResourceInfo getUsedAMResource() { return usedAMResource; } public ResourceInfo getUserAMResourceLimit() { - return userAMResourceLimit; + return userAMResourceLimit; } public boolean getPreemptionDisabled() { @@ -172,10 +170,6 @@ public boolean getIntraQueuePreemptionDisabled() { public String getOrderingPolicyDisplayName() { return orderingPolicyDisplayName; } - - public String getDefaultNodeLabelExpression() { - return defaultNodeLabelExpression; - } public int getDefaultApplicationPriority() { return defaultPriority; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 3865cce909f5b..31857bb94b6b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -91,6 +91,7 @@ public class CapacitySchedulerQueueInfo { protected String queueType; protected String creationMethod; protected String autoCreationEligibility; + protected String defaultNodeLabelExpression; CapacitySchedulerQueueInfo() { }; @@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo { reservedContainers = q.getMetrics().getReservedContainers(); queueName = q.getQueueName(); state = q.getState(); + defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); resourcesUsed = new ResourceInfo(q.getUsedResources()); if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { hideReservationQueues = true; @@ -271,7 +273,7 @@ public ResourceInfo getResourcesUsed() { static float cap(float val, float low, float hi) { return Math.min(Math.max(val, low), hi); } - + public ArrayList getNodeLabels() { return this.nodeLabels; } @@ -335,4 +337,8 @@ public float getWeight() { public float getNormalizedWeight() { return normalizedWeight; } + + public String getDefaultNodeLabelExpression() { + return defaultNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index a0a088497b717..ab3294191bf88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -661,6 +661,58 @@ public void testPerUserResourcesXML() throws Exception { } } + @Test + public void testNodeLabelDefaultAPI() throws Exception { + CapacitySchedulerConfiguration config = + ((CapacityScheduler)rm.getResourceScheduler()).getConfiguration(); + + config.setDefaultNodeLabelExpression("root", "ROOT-INHERITED"); + config.setDefaultNodeLabelExpression("root.a", "root-a-default-label"); + rm.getResourceScheduler().reinitialize(config, rm.getRMContext()); + + //Start RM so that it accepts app submissions + rm.start(); + try { + //Get the XML from ws/v1/cluster/scheduler + WebResource r = resource(); + ClientResponse response = r.path("ws/v1/cluster/scheduler") + .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + String xml = response.getEntity(String.class); + DocumentBuilder db = DocumentBuilderFactory.newInstance() + .newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + //Parse the XML we got + Document dom = db.parse(is); + + NodeList allQueues = dom.getElementsByTagName("queue"); + for (int i = 0; i < allQueues.getLength(); ++i) { + Node queueNode = allQueues.item(i); + Node queuePathNode = getChildNodeByName(queueNode, "queuePath"); + if (queuePathNode == null) { + continue; + } + + String queuePath = queuePathNode.getTextContent(); + if (queuePath != null) { + if (queuePath.startsWith("root.a")) { + assertEquals("root-a-default-label", + getChildNodeByName(queueNode, "defaultNodeLabelExpression") + .getTextContent()); + } else { + assertEquals("ROOT-INHERITED", + getChildNodeByName(queueNode, "defaultNodeLabelExpression") + .getTextContent()); + } + } + } + } finally { + rm.stop(); + } + } + private void checkResourcesUsed(JSONObject queue) throws JSONException { queue.getJSONObject("resourcesUsed").getInt("memory"); queue.getJSONObject("resourcesUsed").getInt("vCores"); From 3b6b21e459c6d91c93f6aaa32bde53047a0f536f Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 23 Apr 2021 16:17:29 +0200 Subject: [PATCH 04/48] YARN-10750. TestMetricsInvariantChecker.testManyRuns is broken since HADOOP-17524. Contributed by Gergely Pollak --- .../src/test/resources/invariants.txt | 4 ---- .../src/test/resources/invariants_jdk9.txt | 4 ---- 2 files changed, 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants.txt index 363ed0d996e51..a77fdf086b118 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants.txt @@ -48,7 +48,3 @@ ThreadsBlocked >= 0 ThreadsWaiting >= 0 ThreadsTimedWaiting >= 0 ThreadsTerminated >= 0 -LogFatal >= 0 -LogError >= 0 -LogWarn >= 0 -LogInfo >= 0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants_jdk9.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants_jdk9.txt index e081334739710..b40a7aa7116f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants_jdk9.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/invariants_jdk9.txt @@ -48,7 +48,3 @@ ThreadsBlocked >= 0 ThreadsWaiting >= 0 ThreadsTimedWaiting >= 0 ThreadsTerminated >= 0 -LogFatal >= 0 -LogError >= 0 -LogWarn >= 0 -LogInfo >= 0 From 14a84c47b0efe287fb848856767b115cf5af4175 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Fri, 23 Apr 2021 17:38:34 +0200 Subject: [PATCH 05/48] YARN-10705. Misleading DEBUG log for container assignment needs to be removed when the container is actually reserved, not assigned in FairScheduler. Contributed by Siddharth Ahuja. --- .../scheduler/fair/FSLeafQueue.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 5eafa522f72ad..af366ec55dc0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -352,9 +352,19 @@ public Resource assignContainer(FSSchedulerNode node) { continue; } assigned = sched.assignContainer(node); - if (!assigned.equals(none())) { - LOG.debug("Assigned container in queue:{} container:{}", - getName(), assigned); + + boolean isContainerAssignedOrReserved = !assigned.equals(none()); + boolean isContainerReserved = + assigned.equals(FairScheduler.CONTAINER_RESERVED); + + // check if an assignment or a reservation was made. + if (isContainerAssignedOrReserved) { + // only log container assignment if there was an actual allocation, + // not a reservation. + if (!isContainerReserved && LOG.isDebugEnabled()) { + LOG.debug("Assigned container in queue:{} container:{}", + getName(), assigned); + } break; } } From 6800b21e3b07168fd5820133d20858c6ca4bdf59 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Fri, 23 Apr 2021 22:48:15 +0530 Subject: [PATCH 06/48] HADOOP-17620. DistCp: Use Iterator for listing target directory as well. (#2861). Contributed by Ayush Saxena. Signed-off-by: Vinayakumar B --- .../org/apache/hadoop/tools/SimpleCopyListing.java | 5 +++-- .../apache/hadoop/tools/mapred/CopyCommitter.java | 3 +++ .../tools/contract/AbstractContractDistCpTest.java | 13 ++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index 900ce6296db08..fb7ace5f1ead5 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -65,7 +65,8 @@ * Note: The SimpleCopyListing doesn't handle wild-cards in the input-paths. */ public class SimpleCopyListing extends CopyListing { - private static final Logger LOG = LoggerFactory.getLogger(SimpleCopyListing.class); + public static final Logger LOG = + LoggerFactory.getLogger(SimpleCopyListing.class); public static final int DEFAULT_FILE_STATUS_SIZE = 1000; public static final boolean DEFAULT_RANDOMIZE_FILE_LISTING = true; @@ -601,7 +602,7 @@ public WorkReport processItem( } private void printStats() { - LOG.info("Paths (files+dirs) cnt = {}; dirCnt = ", totalPaths, totalDirs); + LOG.info("Paths (files+dirs) cnt = {}; dirCnt = {}", totalPaths, totalDirs); } private void maybePrintStats() { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index e346d0b938c93..33ab3ee4eba81 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -564,12 +564,15 @@ private Path listTargetFiles(final Configuration conf, // thread count is picked up from the job int threads = conf.getInt(DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS, DistCpConstants.DEFAULT_LISTSTATUS_THREADS); + boolean useIterator = + conf.getBoolean(DistCpConstants.CONF_LABEL_USE_ITERATOR, false); LOG.info("Scanning destination directory {} with thread count: {}", targetFinalPath, threads); DistCpOptions options = new DistCpOptions.Builder(targets, resultNonePath) .withOverwrite(overwrite) .withSyncFolder(syncFolder) .withNumListstatusThreads(threads) + .withUseIterator(useIterator) .build(); DistCpContext distCpContext = new DistCpContext(options); distCpContext.setTargetPathExists(targetPathExists); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java index 202ead69a17bb..fab14d138b551 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java @@ -42,6 +42,7 @@ import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpConstants; import org.apache.hadoop.tools.DistCpOptions; +import org.apache.hadoop.tools.SimpleCopyListing; import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.util.DistCpTestUtils; import org.apache.hadoop.util.functional.RemoteIterators; @@ -628,11 +629,17 @@ public void testDistCpWithIterator() throws Exception { GenericTestUtils .createFiles(remoteFS, source, getDepth(), getWidth(), getWidth()); + GenericTestUtils.LogCapturer log = + GenericTestUtils.LogCapturer.captureLogs(SimpleCopyListing.LOG); + DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, source.toString(), - dest.toString(), "-useiterator", conf); + dest.toString(), "-useiterator -update -delete", conf); + + // Check the target listing was also done using iterator. + Assertions.assertThat(log.getOutput()).contains( + "Building listing using iterator mode for " + dest.toString()); - Assertions - .assertThat(RemoteIterators.toList(localFS.listFiles(dest, true))) + Assertions.assertThat(RemoteIterators.toList(localFS.listFiles(dest, true))) .describedAs("files").hasSize(getTotalFiles()); } From 027c8fb257eb5144a4cee42341bf6b774c0fd8d1 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 23 Apr 2021 18:44:41 +0100 Subject: [PATCH 07/48] HADOOP-17597. Optionally downgrade on S3A Syncable calls (#2801) Followup to HADOOP-13327, which changed S3A output stream hsync/hflush calls to raise an exception. Adds a new option fs.s3a.downgrade.syncable.exceptions When true, calls to Syncable hsync/hflush on S3A output streams will log once at warn (for entire process life, not just the stream), then increment IOStats with the relevant operation counter With the downgrade option false (default) * IOStats are incremented * The UnsupportedOperationException current raised includes a link to the JIRA. Contributed by Steve Loughran. --- .../fs/statistics/StoreStatisticNames.java | 6 + .../org/apache/hadoop/fs/s3a/Constants.java | 22 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 258 ++++++++++++++---- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 27 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 14 +- .../org/apache/hadoop/fs/s3a/Statistic.java | 8 + .../hadoop/fs/s3a/WriteOperationHelper.java | 5 + .../apache/hadoop/fs/s3a/WriteOperations.java | 6 + .../BlockOutputStreamStatistics.java | 10 + .../impl/EmptyS3AStatisticsContext.java | 8 + .../tools/hadoop-aws/troubleshooting_s3a.md | 37 ++- .../hadoop/fs/s3a/ITestDowngradeSyncable.java | 114 ++++++++ .../fs/s3a/TestS3ABlockOutputStream.java | 55 +++- 13 files changed, 499 insertions(+), 71 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index b6b08fe009e15..95144393585a1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -84,6 +84,12 @@ public final class StoreStatisticNames { /** {@value}. */ public static final String OP_IS_FILE = "op_is_file"; + /** {@value}. */ + public static final String OP_HFLUSH = "op_hflush"; + + /** {@value}. */ + public static final String OP_HSYNC = "op_hsync"; + /** {@value}. */ public static final String OP_IS_DIRECTORY = "op_is_directory"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index c4b8f6e3c4683..f6900cb1c8e19 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -329,7 +329,6 @@ private Constants() { * Default is {@link #FAST_UPLOAD_BUFFER_DISK} * Value: {@value} */ - @InterfaceStability.Unstable public static final String FAST_UPLOAD_BUFFER = "fs.s3a.fast.upload.buffer"; @@ -338,26 +337,22 @@ private Constants() { * Capacity is limited to available disk space. */ - @InterfaceStability.Unstable public static final String FAST_UPLOAD_BUFFER_DISK = "disk"; /** * Use an in-memory array. Fast but will run of heap rapidly: {@value}. */ - @InterfaceStability.Unstable public static final String FAST_UPLOAD_BUFFER_ARRAY = "array"; /** * Use a byte buffer. May be more memory efficient than the * {@link #FAST_UPLOAD_BUFFER_ARRAY}: {@value}. */ - @InterfaceStability.Unstable public static final String FAST_UPLOAD_BYTEBUFFER = "bytebuffer"; /** * Default buffer option: {@value}. */ - @InterfaceStability.Unstable public static final String DEFAULT_FAST_UPLOAD_BUFFER = FAST_UPLOAD_BUFFER_DISK; @@ -370,7 +365,6 @@ private Constants() { *

* Default is {@link #DEFAULT_FAST_UPLOAD_ACTIVE_BLOCKS} */ - @InterfaceStability.Unstable public static final String FAST_UPLOAD_ACTIVE_BLOCKS = "fs.s3a.fast.upload.active.blocks"; @@ -378,9 +372,23 @@ private Constants() { * Limit of queued block upload operations before writes * block. Value: {@value} */ - @InterfaceStability.Unstable public static final int DEFAULT_FAST_UPLOAD_ACTIVE_BLOCKS = 4; + /** + * Rather than raise an exception when an attempt is + * made to call the Syncable APIs, warn and downgrade. + * Value: {@value}. + */ + public static final String DOWNGRADE_SYNCABLE_EXCEPTIONS = + "fs.s3a.downgrade.syncable.exceptions"; + + /** + * Default value for syncable invocation. + * Value: {@value}. + */ + public static final boolean DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT = + false; + /** * The capacity of executor queues for operations other than block * upload, where {@link #FAST_UPLOAD_ACTIVE_BLOCKS} is used instead. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 4f06981bc2d45..65b9535ba65ea 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.PutTracker; +import org.apache.hadoop.fs.s3a.impl.LogExactlyOnce; import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.IOStatistics; @@ -62,10 +63,10 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.util.Progressable; +import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; -import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatistics; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; @@ -89,10 +90,8 @@ class S3ABlockOutputStream extends OutputStream implements private static final Logger LOG = LoggerFactory.getLogger(S3ABlockOutputStream.class); - private static final String E_NOT_SYNCABLE = "S3A streams are not Syncable"; - - /** Owner FileSystem. */ - private final S3AFileSystem fs; + private static final String E_NOT_SYNCABLE = + "S3A streams are not Syncable. See HADOOP-17597."; /** Object being uploaded. */ private final String key; @@ -136,62 +135,48 @@ class S3ABlockOutputStream extends OutputStream implements /** * Write operation helper; encapsulation of the filesystem operations. */ - private final WriteOperationHelper writeOperationHelper; + private final WriteOperations writeOperationHelper; /** * Track multipart put operation. */ private final PutTracker putTracker; + /** Should Syncable calls be downgraded? */ + private final boolean downgradeSyncableExceptions; + + /** + * Downagraded syncable API calls are only logged at warn + * once across the entire process. + */ + private static final LogExactlyOnce WARN_ON_SYNCABLE = + new LogExactlyOnce(LOG); + /** * An S3A output stream which uploads partitions in a separate pool of * threads; different {@link S3ADataBlocks.BlockFactory} * instances can control where data is buffered. - * - * @param fs S3AFilesystem - * @param key S3 object to work on. - * @param executorService the executor service to use to schedule work - * @param progress report progress in order to prevent timeouts. If - * this object implements {@code ProgressListener} then it will be - * directly wired up to the AWS client, so receive detailed progress - * information. - * @param blockSize size of a single block. - * @param blockFactory factory for creating stream destinations - * @param statistics stats for this stream - * @param writeOperationHelper state of the write operation. - * @param putTracker put tracking for commit support * @throws IOException on any problem */ - S3ABlockOutputStream(S3AFileSystem fs, - String key, - ExecutorService executorService, - Progressable progress, - long blockSize, - S3ADataBlocks.BlockFactory blockFactory, - BlockOutputStreamStatistics statistics, - WriteOperationHelper writeOperationHelper, - PutTracker putTracker) + S3ABlockOutputStream(BlockOutputStreamBuilder builder) throws IOException { - this.fs = fs; - this.key = key; - this.blockFactory = blockFactory; - this.blockSize = (int) blockSize; - this.statistics = statistics != null - ? statistics - : EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; + builder.validate(); + this.key = builder.key; + this.blockFactory = builder.blockFactory; + this.blockSize = (int) builder.blockSize; + this.statistics = builder.statistics; // test instantiations may not provide statistics; - this.iostatistics = statistics != null - ? statistics.getIOStatistics() - : emptyStatistics(); - this.writeOperationHelper = writeOperationHelper; - this.putTracker = putTracker; - Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE, - "Block size is too small: %d", blockSize); - this.executorService = MoreExecutors.listeningDecorator(executorService); + this.iostatistics = statistics.getIOStatistics(); + this.writeOperationHelper = builder.writeOperations; + this.putTracker = builder.putTracker; + this.executorService = MoreExecutors.listeningDecorator( + builder.executorService); this.multiPartUpload = null; + final Progressable progress = builder.progress; this.progressListener = (progress instanceof ProgressListener) ? (ProgressListener) progress : new ProgressableListener(progress); + downgradeSyncableExceptions = builder.downgradeSyncableExceptions; // create that first block. This guarantees that an open + close sequence // writes a 0-byte entry. createBlockIfNeeded(); @@ -597,7 +582,7 @@ public String toString() { } private void incrementWriteOperations() { - fs.incrementWriteOperations(); + writeOperationHelper.incrementWriteOperations(); } /** @@ -654,12 +639,31 @@ public boolean hasCapability(String capability) { @Override public void hflush() throws IOException { - throw new UnsupportedOperationException(E_NOT_SYNCABLE); + statistics.hflushInvoked(); + handleSyncableInvocation(); } @Override public void hsync() throws IOException { - throw new UnsupportedOperationException(E_NOT_SYNCABLE); + statistics.hsyncInvoked(); + handleSyncableInvocation(); + } + + /** + * Shared processing of Syncable operation reporting/downgrade. + */ + private void handleSyncableInvocation() { + final UnsupportedOperationException ex + = new UnsupportedOperationException(E_NOT_SYNCABLE); + if (!downgradeSyncableExceptions) { + throw ex; + } + // downgrading. + WARN_ON_SYNCABLE.warn("Application invoked the Syncable API against" + + " stream writing to {}. This is unsupported", + key); + // and log at debug + LOG.debug("Downgrading Syncable call", ex); } @Override @@ -982,4 +986,166 @@ public void progressChanged(ProgressEvent progressEvent) { } } + /** + * Create a builder. + * @return + */ + public static BlockOutputStreamBuilder builder() { + return new BlockOutputStreamBuilder(); + } + + /** + * Builder class for constructing an output stream. + */ + public static final class BlockOutputStreamBuilder { + + /** S3 object to work on. */ + private String key; + + /** The executor service to use to schedule work. */ + private ExecutorService executorService; + + /** + * Report progress in order to prevent timeouts. + * this object implements {@code ProgressListener} then it will be + * directly wired up to the AWS client, so receive detailed progress + * information. + */ + private Progressable progress; + + /** The size of a single block. */ + private long blockSize; + + /** The factory for creating stream destinations. */ + private S3ADataBlocks.BlockFactory blockFactory; + + /** The output statistics for the stream. */ + private BlockOutputStreamStatistics statistics = + EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; + + /** Operations to write data. */ + private WriteOperations writeOperations; + + /** put tracking for commit support. */ + private PutTracker putTracker; + + /** Should Syncable calls be downgraded? */ + private boolean downgradeSyncableExceptions; + + private BlockOutputStreamBuilder() { + } + + /** + * Validate the arguments. + */ + public void validate() { + requireNonNull(key, "null key"); + requireNonNull(executorService, "null executorService"); + requireNonNull(blockFactory, "null blockFactory"); + requireNonNull(statistics, "null statistics"); + requireNonNull(writeOperations, "null writeOperationHelper"); + requireNonNull(putTracker, "null putTracker"); + Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE, + "Block size is too small: %s", blockSize); + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withKey( + final String value) { + key = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withExecutorService( + final ExecutorService value) { + executorService = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withProgress( + final Progressable value) { + progress = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withBlockSize( + final long value) { + blockSize = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withBlockFactory( + final S3ADataBlocks.BlockFactory value) { + blockFactory = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withStatistics( + final BlockOutputStreamStatistics value) { + statistics = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withWriteOperations( + final WriteOperationHelper value) { + writeOperations = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withPutTracker( + final PutTracker value) { + putTracker = value; + return this; + } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withDowngradeSyncableExceptions( + final boolean value) { + downgradeSyncableExceptions = value; + return this; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 8db5d51def84e..7f0160a5070f6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1348,20 +1348,27 @@ public FSDataOutputStream create(Path f, FsPermission permission, String destKey = putTracker.getDestKey(); final BlockOutputStreamStatistics outputStreamStatistics = statisticsContext.newOutputStreamStatistics(); - return new FSDataOutputStream( - new S3ABlockOutputStream(this, - destKey, + final S3ABlockOutputStream.BlockOutputStreamBuilder builder = + S3ABlockOutputStream.builder() + .withKey(destKey) + .withBlockFactory(blockFactory) + .withBlockSize(partSize) + .withStatistics(outputStreamStatistics) + .withProgress(progress) + .withPutTracker(putTracker) + .withWriteOperations(getWriteOperationHelper()) + .withExecutorService( new SemaphoredDelegatingExecutor( boundedThreadPool, blockOutputActiveBlocks, true, - outputStreamStatistics), - progress, - partSize, - blockFactory, - outputStreamStatistics, - getWriteOperationHelper(), - putTracker), + outputStreamStatistics)) + .withDowngradeSyncableExceptions( + getConf().getBoolean( + DOWNGRADE_SYNCABLE_EXCEPTIONS, + DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT)); + return new FSDataOutputStream( + new S3ABlockOutputStream(builder), null); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java index dd28f3e59e037..169a74abef2d7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java @@ -1346,7 +1346,9 @@ private OutputStreamStatistics( STREAM_WRITE_EXCEPTIONS_COMPLETING_UPLOADS.getSymbol(), STREAM_WRITE_QUEUE_DURATION.getSymbol(), STREAM_WRITE_TOTAL_DATA.getSymbol(), - STREAM_WRITE_TOTAL_TIME.getSymbol()) + STREAM_WRITE_TOTAL_TIME.getSymbol(), + INVOCATION_HFLUSH.getSymbol(), + INVOCATION_HSYNC.getSymbol()) .withGauges( STREAM_WRITE_BLOCK_UPLOADS_PENDING.getSymbol(), STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING.getSymbol()) @@ -1489,6 +1491,16 @@ public void commitUploaded(long size) { incrementCounter(COMMITTER_BYTES_UPLOADED, size); } + @Override + public void hflushInvoked() { + incCounter(INVOCATION_HFLUSH.getSymbol(), 1); + } + + @Override + public void hsyncInvoked() { + incCounter(INVOCATION_HSYNC.getSymbol(), 1); + } + @Override public void close() { if (getBytesPendingUpload() > 0) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index 1a53f0d1f8797..c613c06c9bb4e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -137,6 +137,14 @@ public enum Statistic { StoreStatisticNames.OP_IS_FILE, "Calls of isFile()", TYPE_COUNTER), + INVOCATION_HFLUSH( + StoreStatisticNames.OP_HFLUSH, + "Calls of hflush()", + TYPE_COUNTER), + INVOCATION_HSYNC( + StoreStatisticNames.OP_HSYNC, + "Calls of hsync()", + TYPE_COUNTER), INVOCATION_LIST_FILES( StoreStatisticNames.OP_LIST_FILES, "Calls of listFiles()", diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 9bdf61c22a1ff..8b71fc3277142 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -693,4 +693,9 @@ public SelectObjectContentResult select( } }); } + + @Override + public void incrementWriteOperations() { + owner.incrementWriteOperations(); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java index 09b9cc924c6a4..0a8150ce34dbb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java @@ -338,4 +338,10 @@ SelectObjectContentResult select( SelectObjectContentRequest request, String action) throws IOException; + + /** + * Increment the write operation counter + * of the filesystem. + */ + void incrementWriteOperations(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java index b1cee718c206a..772b965d4f4a3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java @@ -134,4 +134,14 @@ public interface BlockOutputStreamStatistics extends Closeable, * @return the value or null if no matching gauge was found. */ Long lookupGaugeValue(String name); + + /** + * Syncable.hflush() has been invoked. + */ + void hflushInvoked(); + + /** + * Syncable.hsync() has been invoked. + */ + void hsyncInvoked(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java index c8cd80592088b..3a651026a0ba6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java @@ -482,6 +482,14 @@ public Long lookupGaugeValue(final String name) { return 0L; } + @Override + public void hflushInvoked() { + } + + @Override + public void hsyncInvoked() { + } + @Override public void close() throws IOException { } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 416793b8ed957..661dd2f36ad2b 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -22,7 +22,7 @@ Common problems working with S3 are 1. Classpath setup 1. Authentication -1. S3 Inconsistency side-effects +1. Incorrect configuration Troubleshooting IAM Assumed Roles is covered in its @@ -1027,7 +1027,7 @@ at the end of a write operation. If a process terminated unexpectedly, or failed to call the `close()` method on an output stream, the pending data will have been lost. -### File `flush()`, `hsync` and `hflush()` calls do not save data to S3 +### File `flush()` calls do not save data to S3 Again, this is due to the fact that the data is cached locally until the `close()` operation. The S3A filesystem cannot be used as a store of data @@ -1036,6 +1036,39 @@ if it is required that the data is persisted durably after every This includes resilient logging, HBase-style journaling and the like. The standard strategy here is to save to HDFS and then copy to S3. +### `UnsupportedOperationException` "S3A streams are not Syncable. See HADOOP-17597." + +The application has tried to call either the `Syncable.hsync()` or `Syncable.hflush()` +methods on an S3A output stream. This has been rejected because the +connector isn't saving any data at all. The `Syncable` API, especially the +`hsync()` call, are critical for applications such as HBase to safely +persist data. + +The S3A connector throws an `UnsupportedOperationException` when these API calls +are made, because the guarantees absolutely cannot be met: nothing is being flushed +or saved. + +* Applications which intend to invoke the Syncable APIs call `hasCapability("hsync")` on + the stream to see if they are supported. +* Or catch and downgrade `UnsupportedOperationException`. + +These recommendations _apply to all filesystems_. + +To downgrade the S3A connector to simply warning of the use of +`hsync()` or `hflush()` calls, set the option +`fs.s3a.downgrade.syncable.exceptions` to true. + +```xml + + fs.s3a.downgrade.syncable.exceptions + true + +``` + +The count of invocations of the two APIs are collected +in the S3A filesystem Statistics/IOStatistics and so +their use can be monitored. + ### `RemoteFileChangedException` and read-during-overwrite ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java new file mode 100644 index 0000000000000..0bcb11a823d3d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; +import org.apache.hadoop.fs.statistics.IOStatistics; + +import static org.apache.hadoop.fs.s3a.Constants.DOWNGRADE_SYNCABLE_EXCEPTIONS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter; +import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToString; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_HFLUSH; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_HSYNC; + + +public class ITestDowngradeSyncable extends AbstractS3ACostTest { + + protected static final Logger LOG = + LoggerFactory.getLogger(ITestDowngradeSyncable.class); + + + public ITestDowngradeSyncable() { + super(false, true, false); + } + + @Override + public Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + String bucketName = getTestBucketName(conf); + removeBucketOverrides(bucketName, conf, + DOWNGRADE_SYNCABLE_EXCEPTIONS); + conf.setBoolean(DOWNGRADE_SYNCABLE_EXCEPTIONS, true); + return conf; + } + + @Test + public void testHFlushDowngrade() throws Throwable { + describe("Verify that hflush() calls can be downgraded from fail" + + " to ignore; the relevant counter is updated"); + Path path = methodPath(); + S3AFileSystem fs = getFileSystem(); + final IOStatistics fsIoStats = fs.getIOStatistics(); + assertThatStatisticCounter(fsIoStats, OP_HFLUSH) + .isEqualTo(0); + + try (FSDataOutputStream out = fs.create(path, true)) { + out.write('1'); + // must succeed + out.hflush(); + // stats counter records the downgrade + IOStatistics iostats = out.getIOStatistics(); + LOG.info("IOStats {}", ioStatisticsToString(iostats)); + assertThatStatisticCounter(iostats, OP_HFLUSH) + .isEqualTo(1); + assertThatStatisticCounter(iostats, OP_HSYNC) + .isEqualTo(0); + } + // once closed. the FS will have its stats merged. + assertThatStatisticCounter(fsIoStats, OP_HFLUSH) + .isEqualTo(1); + } + + @Test + public void testHSyncDowngrade() throws Throwable { + describe("Verify that hsync() calls can be downgraded from fail" + + " to ignore; the relevant counter is updated"); + Path path = methodPath(); + S3AFileSystem fs = getFileSystem(); + final IOStatistics fsIoStats = fs.getIOStatistics(); + assertThatStatisticCounter(fsIoStats, OP_HSYNC) + .isEqualTo(0); + + try (FSDataOutputStream out = fs.create(path, true)) { + out.write('1'); + // must succeed + out.hsync(); + // stats counter records the downgrade + IOStatistics iostats = out.getIOStatistics(); + LOG.info("IOStats {}", ioStatisticsToString(iostats)); + assertThatStatisticCounter(iostats, OP_HFLUSH) + .isEqualTo(0); + assertThatStatisticCounter(iostats, OP_HSYNC) + .isEqualTo(1); + } + // once closed. the FS will have its stats merged. + assertThatStatisticCounter(fsIoStats, OP_HSYNC) + .isEqualTo(1); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index baa4a542c855a..de27411a41ab8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -43,8 +43,11 @@ public class TestS3ABlockOutputStream extends AbstractS3AMockTest { private S3ABlockOutputStream stream; - @Before - public void setUp() throws Exception { + /** + * Create an S3A Builder all mocked up from component pieces. + * @return stream builder. + */ + private S3ABlockOutputStream.BlockOutputStreamBuilder mockS3ABuilder() { ExecutorService executorService = mock(ExecutorService.class); Progressable progressable = mock(Progressable.class); S3ADataBlocks.BlockFactory blockFactory = @@ -52,11 +55,26 @@ public void setUp() throws Exception { long blockSize = Constants.DEFAULT_MULTIPART_SIZE; WriteOperationHelper oHelper = mock(WriteOperationHelper.class); PutTracker putTracker = mock(PutTracker.class); - stream = spy(new S3ABlockOutputStream(fs, "", executorService, - progressable, blockSize, blockFactory, null, oHelper, - putTracker)); + final S3ABlockOutputStream.BlockOutputStreamBuilder builder = + S3ABlockOutputStream.builder() + .withBlockFactory(blockFactory) + .withBlockSize(blockSize) + .withExecutorService(executorService) + .withKey("") + .withProgress(progressable) + .withPutTracker(putTracker) + .withWriteOperations(oHelper); + return builder; + } + + @Before + public void setUp() throws Exception { + final S3ABlockOutputStream.BlockOutputStreamBuilder + builder = mockS3ABuilder(); + stream = spy(new S3ABlockOutputStream(builder)); } + @Test public void testFlushNoOpWhenStreamClosed() throws Exception { doThrow(new IOException()).when(stream).checkOpen(); @@ -108,4 +126,31 @@ public void testCallingCloseAfterCallingAbort() throws Exception { // This will ensure abort() can be called with try-with-resource. stream.close(); } + + + /** + * Unless configured to downgrade, the stream will raise exceptions on + * Syncable API calls. + */ + @Test + public void testSyncableUnsupported() throws Exception { + intercept(UnsupportedOperationException.class, () -> stream.hflush()); + intercept(UnsupportedOperationException.class, () -> stream.hsync()); + } + + /** + * When configured to downgrade, the stream downgrades on + * Syncable API calls. + */ + @Test + public void testSyncableDowngrade() throws Exception { + final S3ABlockOutputStream.BlockOutputStreamBuilder + builder = mockS3ABuilder(); + builder.withDowngradeSyncableExceptions(true); + stream = spy(new S3ABlockOutputStream(builder)); + + stream.hflush(); + stream.hsync(); + } + } From ebf6b14c6794acd89d947c7b1ddb4d8d707b6f45 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 23 Apr 2021 22:48:33 +0200 Subject: [PATCH 08/48] YARN-10123. Error message around yarn app -stop/start can be improved. Contributed by Siddharth Ahuja This is required to highlight that an implementation at framework level is needed for the stop/start functionality to work. --- .../hadoop/yarn/client/api/AppAdminClient.java | 4 +++- .../src/site/markdown/YarnCommands.md | 18 +++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java index 2fd59a863c87b..b9f72484a5d0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java @@ -74,7 +74,9 @@ public static AppAdminClient createAppAdminClient(String appType, } if (!clientClassMap.containsKey(appType)) { throw new IllegalArgumentException("App admin client class name not " + - "specified for type " + appType); + "specified for type " + appType + ". Please ensure " + appType + + " framework has provided a client implementation to handle" + + " this functionality."); } String clientClassName = clientClassMap.get(appType); Class clientClass; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 3082ff81bcb15..123ff6b7c185a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -51,21 +51,21 @@ Usage: `yarn app [options] ` | -changeQueue \ | Moves application to a new queue. ApplicationId can be passed using 'appId' option. 'movetoqueue' command is deprecated, this new command 'changeQueue' performs same functionality. | | -component \ \ | Works with -flex option to change the number of components/containers running for an application / long-running service. Supports absolute or relative changes, such as +1, 2, or -3. | | -components \ | Works with -upgrade option to trigger the upgrade of specified components of the application. Multiple components should be separated by commas. | -| -decommission \ | Decommissions component instances for an application / long-running service. Requires -instances option. Supports -appTypes option to specify which client implementation to use. | -| -destroy \ | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. | -| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. | -| -flex \ | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. | +| -decommission \ | Decommissions component instances for an application / long-running service. Requires -instances option. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | +| -destroy \ | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | +| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | +| -flex \ | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | | -help | Displays help for all commands. | | -instances \ | Works with -upgrade option to trigger the upgrade of specified component instances of the application. Also works with -decommission option to decommission specified component instances. Multiple instances should be separated by commas. | | -kill \ | Kills the application. Set of applications can be provided separated with space | -| -launch \ \ | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. | +| -launch \ \ | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | | -list | List applications. Supports optional use of -appTypes to filter applications based on application type, -appStates to filter applications based on application state and -appTags to filter applications based on application tag. | | -movetoqueue \ | Moves the application to a different queue. Deprecated command. Use 'changeQueue' instead. | | -queue \ | Works with the movetoqueue command to specify which queue to move an application to. | -| -save \ \ | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. | -| -start \ | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. | -| -status \ | Prints the status of the application. If app ID is provided, it prints the generic YARN application status. If name is provided, it prints the application specific status based on app's own implementation, and -appTypes option must be specified unless it is the default `yarn-service` type.| -| -stop \ | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. | +| -save \ \ | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | +| -start \ | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | +| -status \ | Prints the status of the application. If app ID is provided, it prints the generic YARN application status. If name is provided, it prints the application specific status based on app's own implementation, and -appTypes option must be specified unless it is the default `yarn-service` type. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality.| +| -stop \ | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. Please ensure the framework corresponding to appType has provided the appropriate client implementation to handle this specific functionality. | | -updateLifetime \ | Update timeout of an application from NOW. ApplicationId can be passed using 'appId' option. Timeout value is in seconds. | | -updatePriority \ | Update priority of an application. ApplicationId can be passed using 'appId' option. | From 1cbe35946bd8d15e1a0f193cfb64614dc21de6e9 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Fri, 23 Apr 2021 21:35:09 +0000 Subject: [PATCH 09/48] YARN-10743. Add a policy for not aggregating for containers which are killed because exceeding container log size limit. Contributed by Qi Zhu. --- .../api/records/LogAggregationContext.java | 2 ++ ...imitSizeContainerLogAggregationPolicy.java | 32 +++++++++++++++++++ .../TestLogAggregationService.java | 30 +++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LimitSizeContainerLogAggregationPolicy.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java index 5ac7d2dd5e8ae..e58012d68b850 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java @@ -76,6 +76,8 @@ * SampleContainerLogAggregationPolicy: sample logs of successful worker * containers, in addition to application master and failed/killed * containers. + * LimitSizeContainerLogAggregationPolicy: skip aggregation for killed + * containers whose log size exceeds the limit of container log size. * If it isn't specified, it will use the cluster-wide default policy * defined by configuration yarn.nodemanager.log-aggregation.policy.class. * The default value of yarn.nodemanager.log-aggregation.policy.class is diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LimitSizeContainerLogAggregationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LimitSizeContainerLogAggregationPolicy.java new file mode 100644 index 0000000000000..bda175f48a83c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LimitSizeContainerLogAggregationPolicy.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.server.api.ContainerLogContext; + +@Private +public class LimitSizeContainerLogAggregationPolicy extends + AbstractContainerLogAggregationPolicy { + public boolean shouldDoLogAggregation(ContainerLogContext logContext) { + return logContext.getExitCode() + != ContainerExitStatus.KILLED_FOR_EXCESS_LOGS; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index e5ecb724f8c7f..4cc9ac1f3a7a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -99,6 +99,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.Event; @@ -1868,6 +1869,35 @@ public void testFailedContainerPolicy() throws Exception { verifyLogAggFinishEvent(appId); } + @Test (timeout = 50000) + @SuppressWarnings("unchecked") + public void testLimitSizeContainerLogAggregationPolicy() throws Exception { + ApplicationId appId = createApplication(); + LogAggregationService logAggregationService = createLogAggregationService( + appId, LimitSizeContainerLogAggregationPolicy.class, null); + + String[] logFiles = new String[] {"stdout" }; + // exitCode KILLED_FOR_EXCESS_LOGS + finishContainer( + appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, + ContainerExitStatus.KILLED_FOR_EXCESS_LOGS, + logFiles); + ContainerId container2 = + finishContainer(appId, logAggregationService, ContainerType.TASK, 2, 0, + logFiles); + ContainerId container3 = + finishContainer(appId, logAggregationService, ContainerType.TASK, 3, + ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode(), logFiles); + + finishApplication(appId, logAggregationService); + + verifyContainerLogs(logAggregationService, appId, + new ContainerId[] {container2, container3}, + logFiles, 2, false, EMPTY_FILES); + + verifyLogAggFinishEvent(appId); + } + @Test (timeout = 50000) @SuppressWarnings("unchecked") public void testAMOrFailedContainerPolicy() throws Exception { From 6857a05d6ac566a60336c0a28951f09ecda39f24 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Fri, 23 Apr 2021 22:02:04 +0000 Subject: [PATCH 10/48] YARN-10479. Can't remove all node labels after add node label without nodemanager port, broken by YARN-10647. Contributed by D M Murali Krishna Reddy --- .../yarn/nodelabels/CommonNodeLabelsManager.java | 15 ++++++++++++--- .../nodelabels/RMNodeLabelsManager.java | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index c19faaaad3fcd..7dadaeb379047 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -95,6 +95,8 @@ public class CommonNodeLabelsManager extends AbstractService { new ConcurrentHashMap(); protected ConcurrentMap nodeCollections = new ConcurrentHashMap(); + private ConcurrentMap isNodeLabelFromHost = + new ConcurrentHashMap(); protected RMNodeLabel noNodeLabel; @@ -603,6 +605,11 @@ private void replaceLabelsForNode(NodeId node, Set oldLabels, addLabelsToNodeInHost(node, newLabels); } + protected boolean isNodeLabelExplicit(NodeId nodeId) { + return !isNodeLabelFromHost.containsKey(nodeId) || + isNodeLabelFromHost.get(nodeId); + } + @SuppressWarnings("unchecked") protected void internalUpdateLabelsOnNodes( Map> nodeToLabels, NodeLabelUpdateOperation op) @@ -637,6 +644,7 @@ protected void internalUpdateLabelsOnNodes( node.labels.addAll(labels); } addNodeToLabels(node.nodeId, labels); + isNodeLabelFromHost.put(node.nodeId, true); } break; case REPLACE: @@ -646,10 +654,9 @@ protected void internalUpdateLabelsOnNodes( host.labels.addAll(labels); for (Node node : host.nms.values()) { replaceNodeForLabels(node.nodeId, node.labels, labels); - if (node.labels != null) { - replaceLabelsForNode(node.nodeId, node.labels, labels); - } + replaceLabelsForNode(node.nodeId, node.labels, labels); node.labels = null; + isNodeLabelFromHost.put(node.nodeId, true); } break; default: @@ -669,6 +676,7 @@ protected void internalUpdateLabelsOnNodes( nm.labels = new HashSet(); } nm.labels.addAll(labels); + isNodeLabelFromHost.put(nm.nodeId, false); break; case REPLACE: oldLabels = getLabelsByNode(nodeId); @@ -679,6 +687,7 @@ protected void internalUpdateLabelsOnNodes( } nm.labels.clear(); nm.labels.addAll(labels); + isNodeLabelFromHost.put(nm.nodeId, false); break; default: break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 1a08e03e699cc..57f5803f9b457 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -275,7 +275,7 @@ public void deactivateNode(NodeId nodeId) { Map before = cloneNodeMap(ImmutableSet.of(nodeId)); Node nm = getNMInNodeSet(nodeId); if (null != nm) { - if (null == nm.labels) { + if (isNodeLabelExplicit(nm.nodeId)) { // When node deactivated, remove the nm from node collection if no // labels explicitly set for this particular nm From c486dc396c50dc0076009168f80dbb50ea7acca7 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 25 Apr 2021 14:31:51 +0800 Subject: [PATCH 11/48] HDFS-15978. Solve DatanodeManager#getBlockRecoveryCommand() printing IOException. (#2913) Contributed by JiangHua Zhu. --- .../hadoop/hdfs/server/blockmanagement/DatanodeManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 6939e65ba80a1..8e5a24ba60a26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1658,7 +1658,7 @@ private BlockRecoveryCommand getBlockRecoveryCommand(String blockPoolId, BlockUnderConstructionFeature uc = b.getUnderConstructionFeature(); if(uc == null) { throw new IOException("Recovery block " + b + - "where it is not under construction."); + " where it is not under construction."); } final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations(); // Skip stale nodes during recovery From 88a550bc3a73dbbbfb35e2ef29d9c34ad059d1a8 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sun, 25 Apr 2021 18:33:55 +0100 Subject: [PATCH 12/48] HADOOP-17112. S3A committers can't handle whitespace in paths. (#2953) Contributed by Krzysztof Adamski. --- .../java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java | 2 +- .../org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java index c09278fbdcf32..cd8257f97fb91 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java @@ -485,7 +485,7 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile, if (!localFile.isFile()) { throw new FileNotFoundException("Not a file: " + localFile); } - String destURI = destPath.toString(); + String destURI = destPath.toUri().toString(); String destKey = fs.pathToKey(destPath); String uploadId = null; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java index 4d7f81d019b74..ab2f9a2d316ee 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java @@ -673,7 +673,7 @@ public void testBulkCommitFiles() throws Throwable { Path subdir = new Path(destDir, "subdir"); // file 2 Path destFile2 = new Path(subdir, "file2"); - Path destFile3 = new Path(subdir, "file3"); + Path destFile3 = new Path(subdir, "file3 with space"); List destinations = Lists.newArrayList(destFile1, destFile2, destFile3); List commits = new ArrayList<>(3); From 90c6caf6505ea89c90ad59a52dc471ba6a24d102 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 26 Apr 2021 11:27:15 +0800 Subject: [PATCH 13/48] Revert "HDFS-15624. fix the function of setting quota by storage type (#2377)" This reverts commit 394b9f7a5c42ffa38bf7c3a44e7d50a4d19414f9. Ref: HDFS-15995. Had to revert this commit, so we can commit HDFS-15566 (a critical bug preventing rolling upgrade to Hadoop 3.3) Will re-work this fix again later. --- .../java/org/apache/hadoop/fs/StorageType.java | 7 ++++--- .../java/org/apache/hadoop/fs/shell/TestCount.java | 4 ++-- .../server/federation/router/TestRouterQuota.java | 14 +++++++------- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 7 ------- .../server/namenode/NameNodeLayoutVersion.java | 3 +-- .../apache/hadoop/hdfs/TestBlockStoragePolicy.java | 6 +++--- .../hadoop/hdfs/protocol/TestLayoutVersion.java | 3 +-- 7 files changed, 18 insertions(+), 26 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java index 2b5e4f1c5cee2..b17864a22282a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java @@ -34,12 +34,13 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public enum StorageType { + // sorted by the speed of the storage types, from fast to slow RAM_DISK(true, true), + NVDIMM(false, true), SSD(false, false), DISK(false, false), ARCHIVE(false, false), - PROVIDED(false, false), - NVDIMM(false, true); + PROVIDED(false, false); private final boolean isTransient; private final boolean isRAM; @@ -121,4 +122,4 @@ public static String getConf(Configuration conf, StorageType t, String name) { return conf.get(CONF_KEY_HEADER + t.toString() + "." + name); } -} +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 19516b8c40799..618cb0190a99a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -283,10 +283,10 @@ public void processPathWithQuotasByStorageTypesHeader() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> <----13-----> <------17-------> + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA " + // <----13---> <-------17------> "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA " + - " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + "PATHNAME"; verify(out).println(withStorageTypeHeader); verifyNoMoreInteractions(out); @@ -338,11 +338,11 @@ public void processPathWithQuotasByQTVH() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " + " DISK_QUOTA REM_DISK_QUOTA " + "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " + "PROVIDED_QUOTA REM_PROVIDED_QUOTA " + - " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + "PATHNAME"; verify(out).println(withStorageTypeHeader); verifyNoMoreInteractions(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index b69004198eb48..551ae8a8e0612 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -414,13 +414,13 @@ public void testStorageTypeQuota() throws Exception { QuotaUsage usage = client.getQuotaUsage("/type0"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1, -1}, - null, usage); + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, null, + usage); // Verify /type1 quota on NN1. usage = client.getQuotaUsage("/type1"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1, -1}, null, + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, null, usage); FileSystem routerFs = routerContext.getFileSystem(); @@ -431,15 +431,15 @@ public void testStorageTypeQuota() throws Exception { assertEquals(2, u1.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u1.getSpaceQuota()); assertEquals(fileSize * 3, u1.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1, -1}, - new long[] {0, 0, fileSize * 3, 0, 0, 0}, u1); + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, + new long[] {0, 0, 0, fileSize * 3, 0, 0}, u1); // Verify /type0 storage type quota usage on Router. assertEquals(HdfsConstants.QUOTA_RESET, u0.getQuota()); assertEquals(4, u0.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u0.getSpaceQuota()); assertEquals(fileSize * 3 * 2, u0.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1, -1}, - new long[] {0, 0, fileSize * 3 * 2, 0, 0, 0}, u0); + verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, + new long[] {0, 0, 0, fileSize * 3 * 2, 0, 0}, u0); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e559515696d66..926c4e6482414 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2429,10 +2429,6 @@ private void checkStoragePolicyEnabled(final String operationNameReadable, * @throws IOException */ void setStoragePolicy(String src, String policyName) throws IOException { - if (policyName.equalsIgnoreCase( - HdfsConstants.ALLNVDIMM_STORAGE_POLICY_NAME)) { - requireEffectiveLayoutVersionForFeature(Feature.NVDIMM_SUPPORT); - } final String operationName = "setStoragePolicy"; checkOperation(OperationCategory.WRITE); checkStoragePolicyEnabled("set storage policy", true); @@ -3575,9 +3571,6 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type) if (type != null) { requireEffectiveLayoutVersionForFeature(Feature.QUOTA_BY_STORAGE_TYPE); } - if (type == StorageType.NVDIMM) { - requireEffectiveLayoutVersionForFeature(Feature.NVDIMM_SUPPORT); - } checkOperation(OperationCategory.WRITE); final String operationName = getQuotaCommand(nsQuota, ssQuota); final FSPermissionChecker pc = getPermissionChecker(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java index b2477466be9f1..297ca74c5e111 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java @@ -89,8 +89,7 @@ public enum Feature implements LayoutFeature { APPEND_NEW_BLOCK(-62, -61, "Support appending to new block"), QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types"), ERASURE_CODING(-64, -61, "Support erasure coding"), - EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"), - NVDIMM_SUPPORT(-66, -61, "Support NVDIMM storage type"); + EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"); private final FeatureInfo info; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index ef116957fb84d..efb4b4df358f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -1421,29 +1421,29 @@ public void testStorageType() { final EnumMap map = new EnumMap<>(StorageType.class); //put storage type is reversed order - map.put(StorageType.NVDIMM, 1); map.put(StorageType.ARCHIVE, 1); map.put(StorageType.DISK, 1); map.put(StorageType.SSD, 1); map.put(StorageType.RAM_DISK, 1); + map.put(StorageType.NVDIMM, 1); { final Iterator i = map.keySet().iterator(); Assert.assertEquals(StorageType.RAM_DISK, i.next()); + Assert.assertEquals(StorageType.NVDIMM, i.next()); Assert.assertEquals(StorageType.SSD, i.next()); Assert.assertEquals(StorageType.DISK, i.next()); Assert.assertEquals(StorageType.ARCHIVE, i.next()); - Assert.assertEquals(StorageType.NVDIMM, i.next()); } { final Iterator> i = map.entrySet().iterator(); Assert.assertEquals(StorageType.RAM_DISK, i.next().getKey()); + Assert.assertEquals(StorageType.NVDIMM, i.next().getKey()); Assert.assertEquals(StorageType.SSD, i.next().getKey()); Assert.assertEquals(StorageType.DISK, i.next().getKey()); Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey()); - Assert.assertEquals(StorageType.NVDIMM, i.next().getKey()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java index 316480961a9ee..2c9905d6fca4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java @@ -128,8 +128,7 @@ public void testNameNodeFeatureMinimumCompatibleLayoutVersions() { NameNodeLayoutVersion.Feature.APPEND_NEW_BLOCK, NameNodeLayoutVersion.Feature.QUOTA_BY_STORAGE_TYPE, NameNodeLayoutVersion.Feature.ERASURE_CODING, - NameNodeLayoutVersion.Feature.EXPANDED_STRING_TABLE, - NameNodeLayoutVersion.Feature.NVDIMM_SUPPORT); + NameNodeLayoutVersion.Feature.EXPANDED_STRING_TABLE); for (LayoutFeature f : compatibleFeatures) { assertEquals(String.format("Expected minimum compatible layout version " + "%d for feature %s.", baseLV, f), baseLV, From 2621d3f15bfae9b3820f70c0dfda015b1d474a97 Mon Sep 17 00:00:00 2001 From: Brahma Reddy Battula Date: Mon, 26 Apr 2021 11:29:41 +0800 Subject: [PATCH 14/48] HDFS-15566. NN restart fails after RollingUpgrade from 3.1.3/3.2.1 to 3.3.0. Contributed by Brahma Reddy Battula. Signed-off-by: Wei-Chiu Chuang --- .../hdfs/server/namenode/FSEditLogOp.java | 58 +++++++++++++++---- .../namenode/NameNodeLayoutVersion.java | 3 +- .../hdfs/protocol/TestLayoutVersion.java | 3 +- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index feff8b48f7e21..69c95b0521e01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -3477,17 +3477,30 @@ CreateSnapshotOp setSnapshotMTime(long mTime) { void readFields(DataInputStream in, int logVersion) throws IOException { snapshotRoot = FSImageSerialization.readString(in); snapshotName = FSImageSerialization.readString(in); - mtime = FSImageSerialization.readLong(in); - + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + mtime = FSImageSerialization.readLong(in); + } // read RPC ids if necessary readRpcIds(in, logVersion); } @Override public void writeFields(DataOutputStream out) throws IOException { + throw new IOException("Unsupported without logversion"); + } + + @Override + public void writeFields(DataOutputStream out, int logVersion) + throws IOException { FSImageSerialization.writeString(snapshotRoot, out); FSImageSerialization.writeString(snapshotName, out); - FSImageSerialization.writeLong(mtime, out); + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + FSImageSerialization.writeLong(mtime, out); + } writeRpcIds(rpcClientId, rpcCallId, out); } @@ -3569,17 +3582,30 @@ DeleteSnapshotOp setSnapshotMTime(long mTime) { void readFields(DataInputStream in, int logVersion) throws IOException { snapshotRoot = FSImageSerialization.readString(in); snapshotName = FSImageSerialization.readString(in); - mtime = FSImageSerialization.readLong(in); - + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + mtime = FSImageSerialization.readLong(in); + } // read RPC ids if necessary readRpcIds(in, logVersion); } @Override public void writeFields(DataOutputStream out) throws IOException { + throw new IOException("Unsupported without logversion"); + } + + @Override + public void writeFields(DataOutputStream out, int logVersion) + throws IOException { FSImageSerialization.writeString(snapshotRoot, out); FSImageSerialization.writeString(snapshotName, out); - FSImageSerialization.writeLong(mtime, out); + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + FSImageSerialization.writeLong(mtime, out); + } writeRpcIds(rpcClientId, rpcCallId, out); } @@ -3670,19 +3696,31 @@ void readFields(DataInputStream in, int logVersion) throws IOException { snapshotRoot = FSImageSerialization.readString(in); snapshotOldName = FSImageSerialization.readString(in); snapshotNewName = FSImageSerialization.readString(in); - mtime = FSImageSerialization.readLong(in); - + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + mtime = FSImageSerialization.readLong(in); + } // read RPC ids if necessary readRpcIds(in, logVersion); } @Override public void writeFields(DataOutputStream out) throws IOException { + throw new IOException("Unsupported without logversion"); + } + + @Override + public void writeFields(DataOutputStream out, int logVersion) + throws IOException { FSImageSerialization.writeString(snapshotRoot, out); FSImageSerialization.writeString(snapshotOldName, out); FSImageSerialization.writeString(snapshotNewName, out); - FSImageSerialization.writeLong(mtime, out); - + if (NameNodeLayoutVersion + .supports(NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + logVersion)) { + FSImageSerialization.writeLong(mtime, out); + } writeRpcIds(rpcClientId, rpcCallId, out); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java index 297ca74c5e111..bcb3714268aa2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java @@ -89,7 +89,8 @@ public enum Feature implements LayoutFeature { APPEND_NEW_BLOCK(-62, -61, "Support appending to new block"), QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types"), ERASURE_CODING(-64, -61, "Support erasure coding"), - EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"); + EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"), + SNAPSHOT_MODIFICATION_TIME(-66, -61, "Support modification time for snapshot"); private final FeatureInfo info; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java index 2c9905d6fca4b..96629425f8d55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java @@ -128,7 +128,8 @@ public void testNameNodeFeatureMinimumCompatibleLayoutVersions() { NameNodeLayoutVersion.Feature.APPEND_NEW_BLOCK, NameNodeLayoutVersion.Feature.QUOTA_BY_STORAGE_TYPE, NameNodeLayoutVersion.Feature.ERASURE_CODING, - NameNodeLayoutVersion.Feature.EXPANDED_STRING_TABLE); + NameNodeLayoutVersion.Feature.EXPANDED_STRING_TABLE, + NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME); for (LayoutFeature f : compatibleFeatures) { assertEquals(String.format("Expected minimum compatible layout version " + "%d for feature %s.", baseLV, f), baseLV, From 01bad0e92a1f402930bb4c0d18dd605bdf5627e5 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Mon, 26 Apr 2021 13:29:28 +0900 Subject: [PATCH 15/48] HDFS-15967. Improve the log for Short Circuit Local Reads. Contributed by Bhavik Patel. --- .../hadoop/hdfs/server/datanode/DataNode.java | 51 +++++++------------ .../fsdataset/impl/FsDatasetImpl.java | 17 +++---- 2 files changed, 24 insertions(+), 44 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 742b815151d53..5a3a032a658f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1129,7 +1129,7 @@ private synchronized void initDirectoryScanner(Configuration conf) { directoryScanner = new DirectoryScanner(data, conf); directoryScanner.start(); } else { - LOG.info("Periodic Directory Tree Verification scan " + + LOG.warn("Periodic Directory Tree Verification scan " + "is disabled because {}", reason); } @@ -1331,21 +1331,6 @@ public void reportCorruptedBlocks( } } - /** - * Try to send an error report to the NNs associated with the given - * block pool. - * @param bpid the block pool ID - * @param errCode error code to send - * @param errMsg textual message to send - */ - void trySendErrorReport(String bpid, int errCode, String errMsg) { - BPOfferService bpos = blockPoolManager.get(bpid); - if (bpos == null) { - throw new IllegalArgumentException("Bad block pool: " + bpid); - } - bpos.trySendErrorReport(errCode, errMsg); - } - /** * Return the BPOfferService instance corresponding to the given block. * @return the BPOS @@ -2033,7 +2018,7 @@ private void checkBlockToken(ExtendedBlock block, ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); id.readFields(in); - LOG.debug("Got: {}", id); + LOG.debug("BlockTokenIdentifier id: {}", id); blockPoolTokenSecretManager.checkAccess(id, null, block, accessMode, null, null); } @@ -2256,8 +2241,8 @@ private void handleDiskError(String failedVolumes, int failedNumber) { return; // do not shutdown } - LOG.warn("DataNode is shutting down due to failed volumes: [" - + failedVolumes + "]"); + LOG.warn("DataNode is shutting down due to failed volumes: [{}]", + failedVolumes); shouldRun = false; } @@ -2299,7 +2284,7 @@ void incrDatanodeNetworkErrors(String host) { curCount.put("networkErrors", curCount.get("networkErrors") + 1L); datanodeNetworkCounts.put(host, curCount); } catch (ExecutionException e) { - LOG.warn("failed to increment network error counts for " + host); + LOG.warn("failed to increment network error counts for host: {}", host); } } } @@ -2349,7 +2334,7 @@ private void reportBadBlock(final BPOfferService bpos, final ExtendedBlock block, final String msg) { FsVolumeSpi volume = getFSDataset().getVolume(block); if (volume == null) { - LOG.warn("Cannot find FsVolumeSpi to report bad block: " + block); + LOG.warn("Cannot find FsVolumeSpi to report bad block: {}", block); return; } bpos.reportBadBlocks( @@ -2430,7 +2415,7 @@ void transferBlocks(String poolId, Block blocks[], transferBlock(new ExtendedBlock(poolId, blocks[i]), xferTargets[i], xferTargetStorageTypes[i], xferTargetStorageIDs[i]); } catch (IOException ie) { - LOG.warn("Failed to transfer block " + blocks[i], ie); + LOG.warn("Failed to transfer block {}", blocks[i], ie); } } } @@ -2549,15 +2534,13 @@ private class DataTransfer implements Runnable { DataTransfer(DatanodeInfo targets[], StorageType[] targetStorageTypes, String[] targetStorageIds, ExtendedBlock b, BlockConstructionStage stage, final String clientname) { - if (DataTransferProtocol.LOG.isDebugEnabled()) { - DataTransferProtocol.LOG.debug("{}: {} (numBytes={}), stage={}, " + - "clientname={}, targets={}, target storage types={}, " + - "target storage IDs={}", getClass().getSimpleName(), b, - b.getNumBytes(), stage, clientname, Arrays.asList(targets), - targetStorageTypes == null ? "[]" : - Arrays.asList(targetStorageTypes), - targetStorageIds == null ? "[]" : Arrays.asList(targetStorageIds)); - } + DataTransferProtocol.LOG.debug("{}: {} (numBytes={}), stage={}, " + + "clientname={}, targets={}, target storage types={}, " + + "target storage IDs={}", getClass().getSimpleName(), b, + b.getNumBytes(), stage, clientname, Arrays.asList(targets), + targetStorageTypes == null ? "[]" : + Arrays.asList(targetStorageTypes), + targetStorageIds == null ? "[]" : Arrays.asList(targetStorageIds)); this.targets = targets; this.targetStorageTypes = targetStorageTypes; this.targetStorageIds = targetStorageIds; @@ -2661,7 +2644,7 @@ public void run() { LOG.warn("{}:Failed to transfer {} to {} got", bpReg, b, targets[0], ie); } catch (Throwable t) { - LOG.error("Failed to transfer block " + b, t); + LOG.error("Failed to transfer block {}", b, t); } finally { decrementXmitsInProgress(); IOUtils.closeStream(blockSender); @@ -3103,7 +3086,7 @@ private void checkReadAccess(final ExtendedBlock block) throws IOException { } for (TokenIdentifier tokenId : tokenIds) { BlockTokenIdentifier id = (BlockTokenIdentifier) tokenId; - LOG.debug("Got: {}", id); + LOG.debug("BlockTokenIdentifier: {}", id); blockPoolTokenSecretManager.checkAccess(id, null, block, BlockTokenIdentifier.AccessMode.READ, null, null); } @@ -3143,8 +3126,10 @@ void transferReplicaForPipelineRecovery(final ExtendedBlock b, b.setGenerationStamp(storedGS); if (data.isValidRbw(b)) { stage = BlockConstructionStage.TRANSFER_RBW; + LOG.debug("Replica is being written!"); } else if (data.isValidBlock(b)) { stage = BlockConstructionStage.TRANSFER_FINALIZED; + LOG.debug("Replica is finalized!"); } else { final String r = data.getReplicaString(b.getBlockPoolId(), b.getBlockId()); throw new IOException(b + " is neither a RBW nor a Finalized, r=" + r); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index d148f774b98b2..0e7b89a4c4d87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2262,9 +2262,7 @@ ReplicaInfo validateBlockFile(String bpid, long blockId) { datanode.checkDiskErrorAsync(r.getVolume()); } - if (LOG.isDebugEnabled()) { - LOG.debug("blockId=" + blockId + ", replica=" + r); - } + LOG.debug("blockId={}, replica={}", blockId, r); return null; } @@ -2334,15 +2332,12 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async) continue; } } catch(IllegalArgumentException e) { - LOG.warn("Parent directory check failed; replica " + info - + " is not backed by a local file"); + LOG.warn("Parent directory check failed; replica {} is " + + "not backed by a local file", info); } removing = volumeMap.remove(bpid, invalidBlks[i]); addDeletingBlock(bpid, removing.getBlockId()); - if (LOG.isDebugEnabled()) { - LOG.debug("Block file " + removing.getBlockURI() - + " is to be deleted"); - } + LOG.debug("Block file {} is to be deleted", removing.getBlockURI()); if (removing instanceof ReplicaInPipeline) { ((ReplicaInPipeline) removing).releaseAllBytesReserved(); } @@ -2383,8 +2378,8 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async) dataStorage.getTrashDirectoryForReplica(bpid, removing)); } } catch (ClosedChannelException e) { - LOG.warn("Volume " + v + " is closed, ignore the deletion task for " + - "block " + invalidBlks[i]); + LOG.warn("Volume {} is closed, ignore the deletion task for " + + "block: {}", v, invalidBlks[i]); } } if (!errors.isEmpty()) { From 66dda86fecee966c7b0625b19f700a43db9787a3 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Sun, 25 Apr 2021 23:04:52 -0700 Subject: [PATCH 16/48] HADOOP-17661. mvn versions:set fails to parse pom.xml. (#2952) Reviewed-by: Akira Ajisaka Reviewed-by: Hui Fei --- hadoop-project/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d91131e6b980b..b74b2e316ea00 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2283,11 +2283,11 @@ true ${javac.version} ${javac.version} - - - - 9999 - + + -Xlint + -Xlint:unchecked + -Xmaxwarns=9999 + From 538ce9c35403f0c8b595f42e835cc70c91c66621 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 26 Apr 2021 12:18:39 +0530 Subject: [PATCH 17/48] HADOOP-17650. Bump solr to unblock build failure with Maven 3.8.1 (#2939) Reviewed-by: Siyao Meng --- hadoop-project/pom.xml | 2 +- .../pom.xml | 20 +++++++++++++++++++ .../EmbeddedSolrServerFactory.java | 4 +--- .../application/TestAppCatalogSolrClient.java | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b74b2e316ea00..879c5dd8f6182 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -206,7 +206,7 @@ 3.12.2 3.9.0 1.5.6 - 7.7.0 + 8.8.2 1.0.7.Final 1.0.2 5.3.0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index d9481bfee894a..e790cc8cefc70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -110,6 +110,18 @@ org.slf4j slf4j-api + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-client + + + org.eclipse.jetty + jetty-io + @@ -134,6 +146,10 @@ org.eclipse.jetty jetty-http + + org.eclipse.jetty + jetty-client + test @@ -159,6 +175,10 @@ org.eclipse.jetty jetty-http + + org.eclipse.jetty + jetty-client + test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java index 2a4b33853252b..eb78266695988 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java @@ -82,12 +82,10 @@ public static SolrClient create(final String solrHome, solrHomeDir.mkdirs(); } - final SolrResourceLoader loader = new SolrResourceLoader( - solrHomeDir.toPath()); final Path configSetPath = Paths.get(configSetHome).toAbsolutePath(); final NodeConfig config = new NodeConfig.NodeConfigBuilder( - "embeddedSolrServerNode", loader) + "embeddedSolrServerNode", solrHomeDir.toPath()) .setConfigSetBaseDirectory(configSetPath.toString()).build(); final EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(config, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java index d902de5c8cdc2..37a382021ea81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java @@ -47,7 +47,7 @@ public void setup() throws Exception { String targetLocation = EmbeddedSolrServerFactory.class .getProtectionDomain().getCodeSource().getLocation().getFile() + "/.."; - String solrHome = targetLocation + "/solr"; + String solrHome = targetLocation.split("/test-classes")[0] + "/solr"; solrClient = EmbeddedSolrServerFactory.create(solrHome, CONFIGSET_DIR, "exampleCollection"); spy = PowerMockito.spy(new AppCatalogSolrClient()); From bbf62a05623c3ff9943ccbb04e2a97b8997af15a Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 26 Apr 2021 16:42:32 +0800 Subject: [PATCH 18/48] Revert "HADOOP-17661. mvn versions:set fails to parse pom.xml. (#2952)" This reverts commit 66dda86fecee966c7b0625b19f700a43db9787a3. --- hadoop-project/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 879c5dd8f6182..8f229a0fdde74 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2283,11 +2283,11 @@ true ${javac.version} ${javac.version} - - -Xlint - -Xlint:unchecked - -Xmaxwarns=9999 - + + + + 9999 + From b968fa0957ddd0f59c9adfd068507722db7207c6 Mon Sep 17 00:00:00 2001 From: litao Date: Mon, 26 Apr 2021 17:38:43 +0800 Subject: [PATCH 19/48] HDFS-15991. Add location into datanode info for NameNodeMXBean (#2933) Signed-off-by: Takanobu Asanuma --- .../main/webapps/router/federationhealth.html | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 4 +++ .../src/main/webapps/hdfs/dfshealth.html | 10 ++++---- .../server/namenode/TestNameNodeMXBean.java | 25 +++++++++++++++++-- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index 01751c21b1376..87d0e718bdcd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -406,7 +406,7 @@ {#EnteringMaintenanceNodes} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {underReplicatedBlocks} {maintenanceOnlyReplicas} {underReplicateInOpenFiles} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 926c4e6482414..e789f12264ca3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6485,6 +6485,7 @@ public String getLiveNodes() { .put("infoAddr", node.getInfoAddr()) .put("infoSecureAddr", node.getInfoSecureAddr()) .put("xferaddr", node.getXferAddr()) + .put("location", node.getNetworkLocation()) .put("lastContact", getLastContact(node)) .put("usedSpace", getDfsUsed(node)) .put("adminState", node.getAdminState().toString()) @@ -6534,6 +6535,7 @@ public String getDeadNodes() { .put("decommissioned", node.isDecommissioned()) .put("adminState", node.getAdminState().toString()) .put("xferaddr", node.getXferAddr()) + .put("location", node.getNetworkLocation()) .build(); info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo); } @@ -6555,6 +6557,7 @@ public String getDecomNodes() { Map innerinfo = ImmutableMap . builder() .put("xferaddr", node.getXferAddr()) + .put("location", node.getNetworkLocation()) .put("underReplicatedBlocks", node.getLeavingServiceStatus().getUnderReplicatedBlocks()) .put("decommissionOnlyReplicas", @@ -6582,6 +6585,7 @@ public String getEnteringMaintenanceNodes() { Map attrMap = ImmutableMap . builder() .put("xferaddr", node.getXferAddr()) + .put("location", node.getNetworkLocation()) .put("underReplicatedBlocks", node.getLeavingServiceStatus().getUnderReplicatedBlocks()) .put("maintenanceOnlyReplicas", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 8622e4d3a5681..c3ba37165c998 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -326,7 +326,7 @@ {#LiveNodes} {state} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {dnWebAddress} {lastContact}s {lastBlockReport}m @@ -349,7 +349,7 @@ {#DeadNodes} {state} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {#helper_relative_time value="{lastContact}"/} @@ -378,7 +378,7 @@ {#EnteringMaintenanceNodes} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {underReplicatedBlocks} {maintenanceOnlyReplicas} {underReplicateInOpenFiles} @@ -404,7 +404,7 @@ {#DecomNodes} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {underReplicatedBlocks} {decommissionOnlyReplicas} {underReplicateInOpenFiles} @@ -433,7 +433,7 @@ {#LiveNodes} - {name} ({xferaddr}) + {location}/{name} ({xferaddr}) {#helper_date_tostring value="{lastVolumeFailureDate}"/} {volfails} {estimatedCapacityLostTotal|fmt_bytes} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index a309e324f5485..81c9cb8670066 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -112,7 +112,7 @@ public void testNameNodeMXBeanInfo() throws Exception { MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); cluster.waitActive(); // Set upgrade domain on the first DN. @@ -171,7 +171,7 @@ public void testNameNodeMXBeanInfo() throws Exception { "LiveNodes")); Map> liveNodes = (Map>) JSON.parse(alivenodeinfo); - assertTrue(liveNodes.size() == 2); + assertTrue(liveNodes.size() == 4); for (Map liveNode : liveNodes.values()) { assertTrue(liveNode.containsKey("nonDfsUsedSpace")); assertTrue(((Long)liveNode.get("nonDfsUsedSpace")) >= 0); @@ -195,6 +195,27 @@ public void testNameNodeMXBeanInfo() throws Exception { assertFalse(xferAddr.equals(dnXferAddrInMaintenance) ^ inMaintenance); } assertEquals(fsn.getLiveNodes(), alivenodeinfo); + + // Put the third DN to decommissioning state. + DatanodeDescriptor decommissioningNode = dm.getDatanode( + cluster.getDataNodes().get(2).getDatanodeId()); + decommissioningNode.startDecommission(); + + // Put the fourth DN to decommissioned state. + DatanodeDescriptor decommissionedNode = dm.getDatanode( + cluster.getDataNodes().get(3).getDatanodeId()); + decommissionedNode.setDecommissioned(); + + // Assert the location field is included in the mxbeanName + // under different states + String alivenodeinfo1 = (String) (mbs.getAttribute(mxbeanName, + "LiveNodes")); + Map> liveNodes1 = + (Map>) JSON.parse(alivenodeinfo1); + for (Map liveNode : liveNodes1.values()) { + assertTrue(liveNode.containsKey("location")); + } + // get attributes DeadNodes String deadNodeInfo = (String) (mbs.getAttribute(mxbeanName, "DeadNodes")); From 605ed85c291a6250b077da32a49dbb35f3b78bf7 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Mon, 26 Apr 2021 11:00:23 +0100 Subject: [PATCH 20/48] HDFS-15621. Datanode DirectoryScanner uses excessive memory (#2849). Contributed by Stephen O'Donnell --- .../server/datanode/DirectoryScanner.java | 2 +- .../datanode/fsdataset/FsVolumeSpi.java | 118 ++++++++---------- .../datanode/fsdataset/impl/FsVolumeImpl.java | 5 +- .../server/datanode/TestDirectoryScanner.java | 37 +++--- .../fsdataset/impl/TestFsDatasetImpl.java | 4 +- 5 files changed, 77 insertions(+), 89 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 63865f69f29f1..a3bceec9a96a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -582,7 +582,7 @@ private void addDifference(Collection diffRecord, Stats statsRecord, long blockId, FsVolumeSpi vol) { statsRecord.missingBlockFile++; statsRecord.missingMetaFile++; - diffRecord.add(new ScanInfo(blockId, null, null, vol)); + diffRecord.add(new ScanInfo(blockId, null, null, null, vol)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index 68d1a15d5c3b7..8ae204364f05a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -227,27 +227,27 @@ interface BlockIterator extends Closeable { */ public static class ScanInfo implements Comparable { private final long blockId; - /** - * The block file path, relative to the volume's base directory. - * If there was no block file found, this may be null. If 'vol' - * is null, then this is the full path of the block file. + * The full path to the folder containing the block / meta files. */ - private final String blockSuffix; - + private final File basePath; /** - * The suffix of the meta file path relative to the block file. - * If blockSuffix is null, then this will be the entire path relative - * to the volume base directory, or an absolute path if vol is also - * null. + * The block file name, with no path */ - private final String metaSuffix; + private final String blockFile; + /** + * Holds the meta file name, with no path, only if blockFile is null. + * If blockFile is not null, the meta file will be named identically to + * the blockFile, but with a suffix like "_1234.meta". If the blockFile + * is present, we store only the meta file suffix. + */ + private final String metaFile; private final FsVolumeSpi volume; private final FileRegion fileRegion; /** - * Get the file's length in async block scan + * Get the file's length in async block scan. */ private final long blockLength; @@ -257,35 +257,19 @@ public static class ScanInfo implements Comparable { private final static String QUOTED_FILE_SEPARATOR = Matcher.quoteReplacement(File.separator); - /** - * Get the most condensed version of the path. - * - * For example, the condensed version of /foo//bar is /foo/bar - * Unlike {@link File#getCanonicalPath()}, this will never perform I/O - * on the filesystem. - * - * @param path the path to condense - * @return the condensed path - */ - private static String getCondensedPath(String path) { - return CONDENSED_PATH_REGEX.matcher(path). - replaceAll(QUOTED_FILE_SEPARATOR); - } - /** * Get a path suffix. * - * @param f The file to get the suffix for. + * @param f The string to get the suffix for. * @param prefix The prefix we're stripping off. * - * @return A suffix such that prefix + suffix = path to f + * @return A suffix such that prefix + suffix = f */ - private static String getSuffix(File f, String prefix) { - String fullPath = getCondensedPath(f.getAbsolutePath()); - if (fullPath.startsWith(prefix)) { - return fullPath.substring(prefix.length()); + private static String getSuffix(String f, String prefix) { + if (f.startsWith(prefix)) { + return f.substring(prefix.length()); } - throw new RuntimeException(prefix + " is not a prefix of " + fullPath); + throw new RuntimeException(prefix + " is not a prefix of " + f); } /** @@ -293,27 +277,27 @@ private static String getSuffix(File f, String prefix) { * the block data and meta-data files. * * @param blockId the block ID - * @param blockFile the path to the block data file - * @param metaFile the path to the block meta-data file + * @param basePath The full path to the directory the block is stored in + * @param blockFile The block filename, with no path + * @param metaFile The meta filename, with no path. If blockFile is not null + * then the metaFile and blockFile should have the same + * prefix, with the meta file having a suffix like + * "_1234.meta". To save memory, if the blockFile is present + * we store only the meta file suffix in the object * @param vol the volume that contains the block */ - public ScanInfo(long blockId, File blockFile, File metaFile, - FsVolumeSpi vol) { + public ScanInfo(long blockId, File basePath, String blockFile, + String metaFile, FsVolumeSpi vol) { this.blockId = blockId; - String condensedVolPath = - (vol == null || vol.getBaseURI() == null) ? null : - getCondensedPath(new File(vol.getBaseURI()).getAbsolutePath()); - this.blockSuffix = blockFile == null ? null : - getSuffix(blockFile, condensedVolPath); - this.blockLength = (blockFile != null) ? blockFile.length() : 0; - if (metaFile == null) { - this.metaSuffix = null; - } else if (blockFile == null) { - this.metaSuffix = getSuffix(metaFile, condensedVolPath); + this.basePath = basePath; + this.blockFile = blockFile; + if (blockFile != null && metaFile != null) { + this.metaFile = getSuffix(metaFile, blockFile); } else { - this.metaSuffix = getSuffix(metaFile, - condensedVolPath + blockSuffix); + this.metaFile = metaFile; } + this.blockLength = (blockFile != null) ? + new File(basePath, blockFile).length() : 0; this.volume = vol; this.fileRegion = null; } @@ -333,8 +317,9 @@ public ScanInfo(long blockId, FsVolumeSpi vol, FileRegion fileRegion, this.blockLength = length; this.volume = vol; this.fileRegion = fileRegion; - this.blockSuffix = null; - this.metaSuffix = null; + this.basePath = null; + this.blockFile = null; + this.metaFile = null; } /** @@ -343,8 +328,8 @@ public ScanInfo(long blockId, FsVolumeSpi vol, FileRegion fileRegion, * @return the block data file */ public File getBlockFile() { - return (blockSuffix == null) ? null : - new File(new File(volume.getBaseURI()).getAbsolutePath(), blockSuffix); + return (blockFile == null) ? null : + new File(basePath.getAbsolutePath(), blockFile); } /** @@ -363,15 +348,10 @@ public long getBlockLength() { * @return the block meta data file */ public File getMetaFile() { - if (metaSuffix == null) { + if (metaFile == null) { return null; } - String fileSuffix = metaSuffix; - if (blockSuffix != null) { - fileSuffix = blockSuffix + metaSuffix; - } - return new File(new File(volume.getBaseURI()).getAbsolutePath(), - fileSuffix); + return new File(basePath.getAbsolutePath(), fullMetaFile()); } /** @@ -414,14 +394,24 @@ public int hashCode() { } public long getGenStamp() { - return metaSuffix != null ? Block.getGenerationStamp( - getMetaFile().getName()) : - HdfsConstants.GRANDFATHER_GENERATION_STAMP; + return metaFile != null ? Block.getGenerationStamp(fullMetaFile()) + : HdfsConstants.GRANDFATHER_GENERATION_STAMP; } public FileRegion getFileRegion() { return fileRegion; } + + private String fullMetaFile() { + if (metaFile == null) { + return null; + } + if (blockFile == null) { + return metaFile; + } else { + return blockFile + metaFile; + } + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 6681f6fd64c0b..1dda8584f56ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -1451,7 +1451,7 @@ private void compileReport(File bpFinalizedDir, File dir, long blockId = Block.getBlockId(file.getName()); verifyFileLocation(file, bpFinalizedDir, blockId); - report.add(new ScanInfo(blockId, null, file, this)); + report.add(new ScanInfo(blockId, dir, null, fileNames.get(i), this)); } continue; } @@ -1474,7 +1474,8 @@ private void compileReport(File bpFinalizedDir, File dir, } } verifyFileLocation(blockFile, bpFinalizedDir, blockId); - report.add(new ScanInfo(blockId, blockFile, metaFile, this)); + report.add(new ScanInfo(blockId, dir, blockFile.getName(), + metaFile == null ? null : metaFile.getName(), this)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 44d99a292b4b6..e2a15a8da039a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -1040,19 +1040,21 @@ public VolumeCheckResult check(VolumeCheckContext context) private final static String BPID_2 = "BP-367845636-127.0.0.1-5895645674231"; - void testScanInfoObject(long blockId, File blockFile, File metaFile) + void testScanInfoObject(long blockId, File baseDir, String blockFile, + String metaFile) throws Exception { FsVolumeSpi.ScanInfo scanInfo = - new FsVolumeSpi.ScanInfo(blockId, blockFile, metaFile, TEST_VOLUME); + new FsVolumeSpi.ScanInfo(blockId, baseDir, blockFile, metaFile, + TEST_VOLUME); assertEquals(blockId, scanInfo.getBlockId()); if (blockFile != null) { - assertEquals(blockFile.getAbsolutePath(), + assertEquals(new File(baseDir, blockFile).getAbsolutePath(), scanInfo.getBlockFile().getAbsolutePath()); } else { assertNull(scanInfo.getBlockFile()); } if (metaFile != null) { - assertEquals(metaFile.getAbsolutePath(), + assertEquals(new File(baseDir, metaFile).getAbsolutePath(), scanInfo.getMetaFile().getAbsolutePath()); } else { assertNull(scanInfo.getMetaFile()); @@ -1062,7 +1064,7 @@ void testScanInfoObject(long blockId, File blockFile, File metaFile) void testScanInfoObject(long blockId) throws Exception { FsVolumeSpi.ScanInfo scanInfo = - new FsVolumeSpi.ScanInfo(blockId, null, null, null); + new FsVolumeSpi.ScanInfo(blockId, null, null, null, null); assertEquals(blockId, scanInfo.getBlockId()); assertNull(scanInfo.getBlockFile()); assertNull(scanInfo.getMetaFile()); @@ -1071,24 +1073,19 @@ void testScanInfoObject(long blockId) throws Exception { @Test(timeout = 120000) public void TestScanInfo() throws Exception { testScanInfoObject(123, - new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath(), - "blk_123"), - new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath(), - "blk_123__1001.meta")); + new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath()), + "blk_123", "blk_123__1001.meta"); testScanInfoObject(464, - new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath(), - "blk_123"), - null); - testScanInfoObject(523, null, - new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath(), - "blk_123__1009.meta")); - testScanInfoObject(789, null, null); + new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath()), + "blk_123", null); + testScanInfoObject(523, + new File(TEST_VOLUME.getFinalizedDir(BPID_1).getAbsolutePath()), + null, "blk_123__1009.meta"); + testScanInfoObject(789, null, null, null); testScanInfoObject(456); testScanInfoObject(123, - new File(TEST_VOLUME.getFinalizedDir(BPID_2).getAbsolutePath(), - "blk_567"), - new File(TEST_VOLUME.getFinalizedDir(BPID_2).getAbsolutePath(), - "blk_567__1004.meta")); + new File(TEST_VOLUME.getFinalizedDir(BPID_2).getAbsolutePath()), + "blk_567", "blk_567__1004.meta"); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 778ef97180b41..fbd9f005e3895 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -1786,8 +1786,8 @@ public void testNotifyNamenodeMissingOrNewBlock() throws Exception { assertFalse(metaFile.exists()); FsVolumeSpi.ScanInfo info = new FsVolumeSpi.ScanInfo( - replicaInfo.getBlockId(), blockFile.getAbsoluteFile(), - metaFile.getAbsoluteFile(), replicaInfo.getVolume()); + replicaInfo.getBlockId(), blockFile.getParentFile().getAbsoluteFile(), + blockFile.getName(), metaFile.getName(), replicaInfo.getVolume()); fsdataset.checkAndUpdate(bpid, info); BlockManager blockManager = cluster.getNameNode(). From 9166bfeb74df249c6e5e5a4aca50d3ded81303b4 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Sat, 24 Apr 2021 10:10:10 +0200 Subject: [PATCH 21/48] YARN-10637. fs2cs: add queue autorefresh policy during conversion. Contributed by Qi Zhu. --- .../fair/converter/FSYarnSiteConverter.java | 6 ++++++ .../fair/converter/TestFSYarnSiteConverter.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java index 401c056cc999e..4f9029a79d0fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueConfigurationAutoRefreshPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -111,6 +112,11 @@ public void convertSiteProperties(Configuration conf, CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, false); } + // Make auto cs conf refresh enabled. + yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + addMonitorPolicy(QueueConfigurationAutoRefreshPolicy + .class.getCanonicalName(), yarnSiteConfig)); + int maxAssign = conf.getInt(FairSchedulerConfiguration.MAX_ASSIGN, FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN); if (maxAssign != FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java index 55ac242332309..4498373d73d01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -21,6 +21,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueConfigurationAutoRefreshPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -69,6 +70,19 @@ public void testSiteContinuousSchedulingConversion() { ".scheduling-interval-ms", -1)); } + @Test + public void testSiteQueueConfAutoRefreshConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false, false, null); + assertTrue(yarnConvertedConfig.get(YarnConfiguration. + RM_SCHEDULER_ENABLE_MONITORS), true); + assertTrue("Scheduling Policies contains queue conf auto refresh", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(QueueConfigurationAutoRefreshPolicy. + class.getCanonicalName())); + } + @Test public void testSitePreemptionConversion() { yarnConfig.setBoolean(FairSchedulerConfiguration.PREEMPTION, true); From 64264d726d81c919d3f682228ff4bd7fe349fcae Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 26 Apr 2021 18:42:42 -0700 Subject: [PATCH 22/48] HADOOP-17661. mvn versions:set fails to parse pom.xml. (#2956) Signed-off-by: Akira Ajisaka Change-Id: I7de255cb9a167e3a5e25ae339b050e82cb4b7c30 --- hadoop-project/pom.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8f229a0fdde74..99e39a3e490c8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2283,11 +2283,12 @@ true ${javac.version} ${javac.version} - - - - 9999 - + + -Xlint + -Xlint:unchecked + -Xmaxwarns + 9999 + From ef13f8ad6b269a3d9a9dc5cadd990ab87add0a33 Mon Sep 17 00:00:00 2001 From: bshashikant Date: Tue, 27 Apr 2021 09:15:06 +0530 Subject: [PATCH 23/48] HDFS-15961. standby namenode failed to start ordered snapshot deletion is enabled while having snapshottable directories (#2881) --- .../hdfs/server/namenode/FSNamesystem.java | 39 +++++++++++++------ .../hadoop/hdfs/server/namenode/NameNode.java | 1 + .../server/namenode/ha/TestHASafeMode.java | 37 ++++++++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e789f12264ca3..7b9efea538e18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -124,7 +124,12 @@ import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.ipc.ObserverRetryOnActiveException; -import org.apache.hadoop.util.*; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.VersionInfo; import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.monotonicNow; @@ -8560,25 +8565,36 @@ void checkAccess(String src, FsAction mode) throws IOException { /** * Check if snapshot roots are created for all existing snapshottable * directories. Create them if not. + * Only the active NameNode needs to execute this in HA setup once it is out + * of safe mode. + * + * The function gets called while exiting safe mode or post starting the + * services in Active NameNode, but comes into effect post whichever event + * happens later. */ @Override - public void checkAndProvisionSnapshotTrashRoots() { - if (isSnapshotTrashRootEnabled) { + public synchronized void checkAndProvisionSnapshotTrashRoots() { + if (isSnapshotTrashRootEnabled && (haEnabled && inActiveState() + || !haEnabled) && !blockManager.isInSafeMode()) { + SnapshottableDirectoryStatus dirStatus = null; try { SnapshottableDirectoryStatus[] dirStatusList = getSnapshottableDirListing(); if (dirStatusList == null) { return; } - for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + for (SnapshottableDirectoryStatus status : dirStatusList) { + dirStatus = status; String currDir = dirStatus.getFullPath().toString(); if (!currDir.endsWith(Path.SEPARATOR)) { currDir += Path.SEPARATOR; } String trashPath = currDir + FileSystem.TRASH_PREFIX; - HdfsFileStatus fileStatus = getFileInfo(trashPath, false, false, false); + HdfsFileStatus fileStatus = + getFileInfo(trashPath, false, false, false); if (fileStatus == null) { - LOG.info("Trash doesn't exist for snapshottable directory {}. " + "Creating trash at {}", currDir, trashPath); + LOG.info("Trash doesn't exist for snapshottable directory {}. " + + "Creating trash at {}", currDir, trashPath); PermissionStatus permissionStatus = new PermissionStatus(getRemoteUser().getShortUserName(), null, SHARED_TRASH_PERMISSION); @@ -8586,12 +8602,13 @@ public void checkAndProvisionSnapshotTrashRoots() { } } } catch (IOException e) { - final String msg = - "Could not provision Trash directory for existing " - + "snapshottable directories. Exiting Namenode."; - ExitUtil.terminate(1, msg); + if (dirStatus == null) { + LOG.error("Failed to get snapshottable directory list", e); + } else { + LOG.error("Could not provision Trash directory for existing " + + "snapshottable directory {}", dirStatus, e); + } } - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index ed3d80bbfc201..4692229be2f4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -2021,6 +2021,7 @@ public HAState getState() { public void startActiveServices() throws IOException { try { namesystem.startActiveServices(); + namesystem.checkAndProvisionSnapshotTrashRoots(); startTrashEmptier(getConf()); } catch (Throwable t) { doImmediateShutdown(t); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index e17bb6f53cb49..562eb198fdb8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -98,6 +98,7 @@ public void setupCluster() throws Exception { conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", false); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) @@ -909,6 +910,42 @@ public Boolean get() { assertSafeMode(nn1, 3, 3, 3, 0); } + @Test + public void testNameNodeCreateSnapshotTrashRootOnHASetup() throws Exception { + DistributedFileSystem dfs = cluster.getFileSystem(0); + final Path testDir = new Path("/disallowss/test2/"); + final Path file0path = new Path(testDir, "file-0"); + dfs.create(file0path).close(); + dfs.allowSnapshot(testDir); + // .Trash won't be created right now since snapshot trash is disabled + final Path trashRoot = new Path(testDir, FileSystem.TRASH_PREFIX); + assertFalse(dfs.exists(trashRoot)); + // Set dfs.namenode.snapshot.trashroot.enabled=true + cluster.getNameNode(0).getConf() + .setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + cluster.getNameNode(1).getConf() + .setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + restartActive(); + cluster.transitionToActive(1); + dfs = cluster.getFileSystem(1); + // Make sure .Trash path does not exist yet as on NN1 trash root is not + // enabled + assertFalse(dfs.exists(trashRoot)); + cluster.transitionToStandby(1); + cluster.transitionToActive(0); + dfs = cluster.getFileSystem(0); + // Check .Trash existence, should be created now + assertTrue(dfs.exists(trashRoot)); + assertFalse(cluster.getNameNode(0).isInSafeMode()); + restartStandby(); + // Ensure Standby namenode is up and running + assertTrue(cluster.getNameNode(1).isStandbyState()); + // Cleanup + dfs.delete(trashRoot, true); + dfs.disallowSnapshot(testDir); + dfs.delete(testDir, true); + } + /** * Test transition to active when namenode in safemode. * From f54e7646cf5fa4bd8498cab824e4693a5d1a9717 Mon Sep 17 00:00:00 2001 From: bilaharith <52483117+bilaharith@users.noreply.github.com> Date: Tue, 27 Apr 2021 15:45:52 +0530 Subject: [PATCH 24/48] HADOOP-17536. ABFS: Supporting customer provided encryption key (#2707) Contributed by bilahari t h --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 5 + .../azurebfs/constants/ConfigurationKeys.java | 2 + .../constants/FileSystemConfigurations.java | 2 + .../constants/HttpHeaderConfigurations.java | 5 + .../fs/azurebfs/services/AbfsClient.java | 66 +- .../fs/azurebfs/ITestCustomerProvidedKey.java | 936 ++++++++++++++++++ .../constants/TestConfigurationKeys.java | 4 + .../fs/azurebfs/services/TestAbfsClient.java | 6 +- 8 files changed, 1019 insertions(+), 7 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 0a8224aaaeb58..1c4a09be3c9a7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -948,6 +948,11 @@ public boolean enableAbfsListIterator() { return this.enableAbfsListIterator; } + public String getClientProvidedEncryptionKey() { + String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY); + return rawConfig.get(accSpecEncKey, null); + } + @VisibleForTesting void setReadBufferSize(int bufferSize) { this.readBufferSize = bufferSize; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 4fe1d1c276db5..2dbb2b9b08db8 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -143,6 +143,8 @@ public final class ConfigurationKeys { public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script"; /** Setting this true will make the driver use it's own RemoteIterator implementation */ public static final String FS_AZURE_ENABLE_ABFS_LIST_ITERATOR = "fs.azure.enable.abfslistiterator"; + /** Server side encryption key */ + public static final String FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY = "fs.azure.client-provided-encryption-key"; /** End point of ABFS account: {@value}. */ public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 040b18ae4c281..dc4caa98a5e60 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -68,6 +68,8 @@ public final class FileSystemConfigurations { public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost"; public static final int DEFAULT_AZURE_LIST_MAX_RESULTS = 5000; + public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM = "AES256"; + public static final int MAX_CONCURRENT_READ_THREADS = 12; public static final int MAX_CONCURRENT_WRITE_THREADS = 8; public static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java index 232553844fcf3..d4065ac2836d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java @@ -60,6 +60,11 @@ public final class HttpHeaderConfigurations { public static final String X_MS_UMASK = "x-ms-umask"; public static final String X_MS_NAMESPACE_ENABLED = "x-ms-namespace-enabled"; public static final String X_MS_ABFS_CLIENT_LATENCY = "x-ms-abfs-client-latency"; + public static final String X_MS_ENCRYPTION_KEY = "x-ms-encryption-key"; + public static final String X_MS_ENCRYPTION_KEY_SHA256 = "x-ms-encryption-key-sha256"; + public static final String X_MS_ENCRYPTION_ALGORITHM = "x-ms-encryption-algorithm"; + public static final String X_MS_REQUEST_SERVER_ENCRYPTED = "x-ms-request-server-encrypted"; + public static final String X_MS_SERVER_ENCRYPTED = "x-ms-server-encrypted"; public static final String X_MS_LEASE_ACTION = "x-ms-lease-action"; public static final String X_MS_LEASE_DURATION = "x-ms-lease-duration"; public static final String X_MS_LEASE_ID = "x-ms-lease-id"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 7c8a2112bfa46..c5c218d3fb257 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -25,8 +25,12 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Locale; import java.util.UUID; @@ -65,6 +69,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*; import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*; @@ -74,6 +79,7 @@ */ public class AbfsClient implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); + private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; private final String xMsVersion = "2019-12-12"; @@ -82,6 +88,8 @@ public class AbfsClient implements Closeable { private final AbfsConfiguration abfsConfiguration; private final String userAgent; private final AbfsPerfTracker abfsPerfTracker; + private final String clientProvidedEncryptionKey; + private final String clientProvidedEncryptionKeySHA; private final String accountName; private final AuthType authType; @@ -93,7 +101,8 @@ public class AbfsClient implements Closeable { private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, - final AbfsClientContext abfsClientContext) { + final AbfsClientContext abfsClientContext) + throws IOException { this.baseUrl = baseUrl; this.sharedKeyCredentials = sharedKeyCredentials; String baseUrlString = baseUrl.toString(); @@ -103,6 +112,17 @@ private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCreden this.accountName = abfsConfiguration.getAccountName().substring(0, abfsConfiguration.getAccountName().indexOf(AbfsHttpConstants.DOT)); this.authType = abfsConfiguration.getAuthType(accountName); + String encryptionKey = this.abfsConfiguration + .getClientProvidedEncryptionKey(); + if (encryptionKey != null) { + this.clientProvidedEncryptionKey = getBase64EncodedString(encryptionKey); + this.clientProvidedEncryptionKeySHA = getBase64EncodedString( + getSHA256Hash(encryptionKey)); + } else { + this.clientProvidedEncryptionKey = null; + this.clientProvidedEncryptionKeySHA = null; + } + String sslProviderName = null; if (this.baseUrl.toString().startsWith(HTTPS_SCHEME)) { @@ -131,7 +151,8 @@ private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCreden public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, final AccessTokenProvider tokenProvider, - final AbfsClientContext abfsClientContext) { + final AbfsClientContext abfsClientContext) + throws IOException { this(baseUrl, sharedKeyCredentials, abfsConfiguration, abfsClientContext); this.tokenProvider = tokenProvider; } @@ -139,11 +160,29 @@ public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredent public AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials, final AbfsConfiguration abfsConfiguration, final SASTokenProvider sasTokenProvider, - final AbfsClientContext abfsClientContext) { + final AbfsClientContext abfsClientContext) + throws IOException { this(baseUrl, sharedKeyCredentials, abfsConfiguration, abfsClientContext); this.sasTokenProvider = sasTokenProvider; } + private byte[] getSHA256Hash(String key) throws IOException { + try { + final MessageDigest digester = MessageDigest.getInstance("SHA-256"); + return digester.digest(key.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + throw new IOException(e); + } + } + + private String getBase64EncodedString(String key) { + return getBase64EncodedString(key.getBytes(StandardCharsets.UTF_8)); + } + + private String getBase64EncodedString(byte[] bytes) { + return Base64.getEncoder().encodeToString(bytes); + } + @Override public void close() throws IOException { if (tokenProvider instanceof Closeable) { @@ -180,6 +219,18 @@ List createDefaultHeaders() { return requestHeaders; } + private void addCustomerProvidedKeyHeaders( + final List requestHeaders) { + if (clientProvidedEncryptionKey != null) { + requestHeaders.add( + new AbfsHttpHeader(X_MS_ENCRYPTION_KEY, clientProvidedEncryptionKey)); + requestHeaders.add(new AbfsHttpHeader(X_MS_ENCRYPTION_KEY_SHA256, + clientProvidedEncryptionKeySHA)); + requestHeaders.add(new AbfsHttpHeader(X_MS_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_ALGORITHM)); + } + } + AbfsUriQueryBuilder createDefaultUriQueryBuilder() { final AbfsUriQueryBuilder abfsUriQueryBuilder = new AbfsUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_TIMEOUT, DEFAULT_TIMEOUT); @@ -289,6 +340,9 @@ public AbfsRestOperation createPath(final String path, final boolean isFile, fin final String permission, final String umask, final boolean isAppendBlob, final String eTag) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); + if (isFile) { + addCustomerProvidedKeyHeaders(requestHeaders); + } if (!overwrite) { requestHeaders.add(new AbfsHttpHeader(IF_NONE_MATCH, AbfsHttpConstants.STAR)); } @@ -510,6 +564,7 @@ public AbfsRestOperation append(final String path, final byte[] buffer, AppendRequestParameters reqParams, final String cachedSasToken) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); + addCustomerProvidedKeyHeaders(requestHeaders); // JDK7 does not support PATCH, so to workaround the issue we will use // PUT and specify the real method in the X-Http-Method-Override header. requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE, @@ -596,6 +651,7 @@ public AbfsRestOperation flush(final String path, final long position, boolean r boolean isClose, final String cachedSasToken, final String leaseId) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); + addCustomerProvidedKeyHeaders(requestHeaders); // JDK7 does not support PATCH, so to workaround the issue we will use // PUT and specify the real method in the X-Http-Method-Override header. requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE, @@ -627,6 +683,7 @@ public AbfsRestOperation flush(final String path, final long position, boolean r public AbfsRestOperation setPathProperties(final String path, final String properties) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); + addCustomerProvidedKeyHeaders(requestHeaders); // JDK7 does not support PATCH, so to workaround the issue we will use // PUT and specify the real method in the X-Http-Method-Override header. requestHeaders.add(new AbfsHttpHeader(X_HTTP_METHOD_OVERRIDE, @@ -660,6 +717,8 @@ public AbfsRestOperation getPathStatus(final String path, final boolean includeP // only traversal (execute) permission is required. abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); operation = SASTokenProvider.GET_STATUS_OPERATION; + } else { + addCustomerProvidedKeyHeaders(requestHeaders); } abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); @@ -678,6 +737,7 @@ public AbfsRestOperation getPathStatus(final String path, final boolean includeP public AbfsRestOperation read(final String path, final long position, final byte[] buffer, final int bufferOffset, final int bufferLength, final String eTag, String cachedSasToken) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); + addCustomerProvidedKeyHeaders(requestHeaders); requestHeaders.add(new AbfsHttpHeader(RANGE, String.format("bytes=%d-%d", position, position + bufferLength - 1))); requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java new file mode 100644 index 0000000000000..9229905b4623c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java @@ -0,0 +1,936 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 ("License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.EnumSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; + +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters; +import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode; +import org.apache.hadoop.fs.azurebfs.services.AbfsAclHelper; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsHttpHeader; +import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; +import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.utils.Base64; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.test.LambdaTestUtils; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_KEY_SHA256; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_SERVER_ENCRYPTED; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_SERVER_ENCRYPTED; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_CPK_ENABLED; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME; +import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry; +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; + +public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest { + private static final Logger LOG = LoggerFactory + .getLogger(ITestCustomerProvidedKey.class); + + private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1"; + private static final int INT_512 = 512; + private static final int INT_50 = 50; + private static final int ENCRYPTION_KEY_LEN = 32; + private static final int FILE_SIZE = 10 * ONE_MB; + private static final int FILE_SIZE_FOR_COPY_BETWEEN_ACCOUNTS = 24 * ONE_MB; + + public ITestCustomerProvidedKey() throws Exception { + boolean isCPKTestsEnabled = getConfiguration() + .getBoolean(FS_AZURE_TEST_CPK_ENABLED, false); + Assume.assumeTrue(isCPKTestsEnabled); + } + + @Test + public void testReadWithCPK() throws Exception { + final AzureBlobFileSystem fs = getAbfs(true); + String fileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, fileName, FILE_SIZE); + + AbfsClient abfsClient = fs.getAbfsClient(); + int length = FILE_SIZE; + byte[] buffer = new byte[length]; + final AbfsRestOperation op = abfsClient.getPathStatus(fileName, false); + final String eTag = op.getResult() + .getResponseHeader(HttpHeaderConfigurations.ETAG); + AbfsRestOperation abfsRestOperation = abfsClient + .read(fileName, 0, buffer, 0, length, eTag, null); + assertCPKHeaders(abfsRestOperation, true); + assertResponseHeader(abfsRestOperation, true, X_MS_ENCRYPTION_KEY_SHA256, + getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, true, X_MS_SERVER_ENCRYPTED, + "true"); + assertResponseHeader(abfsRestOperation, false, + X_MS_REQUEST_SERVER_ENCRYPTED, ""); + + // Trying to read with different CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "different-1234567890123456789012"); + try (AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + FSDataInputStream iStream = fs2.open(new Path(fileName))) { + int len = 8 * ONE_MB; + byte[] b = new byte[len]; + LambdaTestUtils.intercept(IOException.class, () -> { + iStream.read(b, 0, len); + }); + } + + // Trying to read with no CPK headers + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + try (AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem + .get(conf); FSDataInputStream iStream = fs3.open(new Path(fileName))) { + int len = 8 * ONE_MB; + byte[] b = new byte[len]; + LambdaTestUtils.intercept(IOException.class, () -> { + iStream.read(b, 0, len); + }); + } + } + + @Test + public void testReadWithoutCPK() throws Exception { + final AzureBlobFileSystem fs = getAbfs(false); + String fileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, fileName, FILE_SIZE); + + AbfsClient abfsClient = fs.getAbfsClient(); + int length = INT_512; + byte[] buffer = new byte[length * 4]; + final AbfsRestOperation op = abfsClient.getPathStatus(fileName, false); + final String eTag = op.getResult() + .getResponseHeader(HttpHeaderConfigurations.ETAG); + AbfsRestOperation abfsRestOperation = abfsClient + .read(fileName, 0, buffer, 0, length, eTag, null); + assertCPKHeaders(abfsRestOperation, false); + assertResponseHeader(abfsRestOperation, false, X_MS_ENCRYPTION_KEY_SHA256, + getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, true, X_MS_SERVER_ENCRYPTED, + "true"); + assertResponseHeader(abfsRestOperation, false, + X_MS_REQUEST_SERVER_ENCRYPTED, ""); + + // Trying to read with CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "12345678901234567890123456789012"); + + try (AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + AbfsClient abfsClient2 = fs2.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient2.read(fileName, 0, buffer, 0, length, eTag, null); + }); + } + } + + @Test + public void testAppendWithCPK() throws Exception { + final AzureBlobFileSystem fs = getAbfs(true); + final String fileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, fileName, FILE_SIZE); + + // Trying to append with correct CPK headers + AppendRequestParameters appendRequestParameters = + new AppendRequestParameters( + 0, 0, 5, Mode.APPEND_MODE, false, null); + byte[] buffer = getRandomBytesArray(5); + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .append(fileName, buffer, appendRequestParameters, null); + assertCPKHeaders(abfsRestOperation, true); + assertResponseHeader(abfsRestOperation, true, X_MS_ENCRYPTION_KEY_SHA256, + getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, true, X_MS_REQUEST_SERVER_ENCRYPTED, + "true"); + + // Trying to append with different CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "different-1234567890123456789012"); + try (AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + AbfsClient abfsClient2 = fs2.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient2.append(fileName, buffer, appendRequestParameters, null); + }); + } + + // Trying to append with no CPK headers + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + try (AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem + .get(conf); AbfsClient abfsClient3 = fs3.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient3.append(fileName, buffer, appendRequestParameters, null); + }); + } + } + + @Test + public void testAppendWithoutCPK() throws Exception { + final AzureBlobFileSystem fs = getAbfs(false); + final String fileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, fileName, FILE_SIZE); + + // Trying to append without CPK headers + AppendRequestParameters appendRequestParameters = + new AppendRequestParameters( + 0, 0, 5, Mode.APPEND_MODE, false, null); + byte[] buffer = getRandomBytesArray(5); + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .append(fileName, buffer, appendRequestParameters, null); + assertCPKHeaders(abfsRestOperation, false); + assertResponseHeader(abfsRestOperation, false, X_MS_ENCRYPTION_KEY_SHA256, + ""); + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, true, X_MS_REQUEST_SERVER_ENCRYPTED, + "true"); + + // Trying to append with CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "12345678901234567890123456789012"); + try (AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + AbfsClient abfsClient2 = fs2.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient2.append(fileName, buffer, appendRequestParameters, null); + }); + } + } + + @Test + public void testSetGetXAttr() throws Exception { + final AzureBlobFileSystem fs = getAbfs(true); + String fileName = methodName.getMethodName(); + createFileAndGetContent(fs, fileName, FILE_SIZE); + + String valSent = "testValue"; + String attrName = "testXAttr"; + + // set get and verify + fs.setXAttr(new Path(fileName), attrName, + valSent.getBytes(StandardCharsets.UTF_8), + EnumSet.of(XAttrSetFlag.CREATE)); + byte[] valBytes = fs.getXAttr(new Path(fileName), attrName); + String valRecieved = new String(valBytes); + assertEquals(valSent, valRecieved); + + // set new value get and verify + valSent = "new value"; + fs.setXAttr(new Path(fileName), attrName, + valSent.getBytes(StandardCharsets.UTF_8), + EnumSet.of(XAttrSetFlag.REPLACE)); + valBytes = fs.getXAttr(new Path(fileName), attrName); + valRecieved = new String(valBytes); + assertEquals(valSent, valRecieved); + + // Read without CPK header + LambdaTestUtils.intercept(IOException.class, () -> { + getAbfs(false).getXAttr(new Path(fileName), attrName); + }); + + // Wrong CPK + LambdaTestUtils.intercept(IOException.class, () -> { + getSameFSWithWrongCPK(fs).getXAttr(new Path(fileName), attrName); + }); + } + + @Test + public void testCopyBetweenAccounts() throws Exception { + String accountName = getRawConfiguration() + .get(FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT); + String accountKey = getRawConfiguration() + .get(FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT_KEY); + Assume.assumeTrue(accountName != null && !accountName.isEmpty()); + Assume.assumeTrue(accountKey != null && !accountKey.isEmpty()); + String fileSystemName = "cpkfs"; + + // Create fs1 and a file with CPK + AzureBlobFileSystem fs1 = getAbfs(true); + int fileSize = FILE_SIZE_FOR_COPY_BETWEEN_ACCOUNTS; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs1, "fs1-file.txt", fileContent); + + // Create fs2 with different CPK + Configuration conf = new Configuration(); + conf.addResource(TEST_CONFIGURATION_FILE_NAME); + conf.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); + conf.unset(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_ABFS_ACCOUNT_NAME, accountName); + conf.set(FS_AZURE_ACCOUNT_KEY + "." + accountName, accountKey); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "123456789012345678901234567890ab"); + conf.set("fs.defaultFS", "abfs://" + fileSystemName + "@" + accountName); + AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + + // Read from fs1 and write to fs2, fs1 and fs2 are having different CPK + Path fs2DestFilePath = new Path("fs2-dest-file.txt"); + FSDataOutputStream ops = fs2.create(fs2DestFilePath); + try (FSDataInputStream iStream = fs1.open(testFilePath)) { + long totalBytesRead = 0; + do { + int length = 8 * ONE_MB; + byte[] buffer = new byte[length]; + int bytesRead = iStream.read(buffer, 0, length); + totalBytesRead += bytesRead; + ops.write(buffer); + } while (totalBytesRead < fileContent.length); + ops.close(); + } + + // Trying to read fs2DestFilePath with different CPK headers + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "different-1234567890123456789012"); + try (AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem + .get(conf); FSDataInputStream iStream = fs3.open(fs2DestFilePath)) { + int length = 8 * ONE_MB; + byte[] buffer = new byte[length]; + LambdaTestUtils.intercept(IOException.class, () -> { + iStream.read(buffer, 0, length); + }); + } + + // Trying to read fs2DestFilePath with no CPK headers + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + try (AzureBlobFileSystem fs4 = (AzureBlobFileSystem) FileSystem + .get(conf); FSDataInputStream iStream = fs4.open(fs2DestFilePath)) { + int length = 8 * ONE_MB; + byte[] buffer = new byte[length]; + LambdaTestUtils.intercept(IOException.class, () -> { + iStream.read(buffer, 0, length); + }); + } + + // Read fs2DestFilePath and verify the content with the initial random + // bytes created and wrote into the source file at fs1 + try (FSDataInputStream iStream = fs2.open(fs2DestFilePath)) { + long totalBytesRead = 0; + int pos = 0; + do { + int length = 8 * ONE_MB; + byte[] buffer = new byte[length]; + int bytesRead = iStream.read(buffer, 0, length); + totalBytesRead += bytesRead; + for (int i = 0; i < bytesRead; i++) { + assertEquals(fileContent[pos + i], buffer[i]); + } + pos = pos + bytesRead; + } while (totalBytesRead < fileContent.length); + } + } + + @Test + public void testListPathWithCPK() throws Exception { + testListPath(true); + } + + @Test + public void testListPathWithoutCPK() throws Exception { + testListPath(false); + } + + private void testListPath(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + String testDirName = "/" + methodName.getMethodName(); + final Path testPath = new Path(testDirName); + fs.mkdirs(testPath); + createFileAndGetContent(fs, testDirName + "/aaa", FILE_SIZE); + createFileAndGetContent(fs, testDirName + "/bbb", FILE_SIZE); + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .listPath(testDirName, false, INT_50, null); + assertListstatus(fs, abfsRestOperation, testPath); + + // Trying with different CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "different-1234567890123456789012"); + AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + AbfsClient abfsClient2 = fs2.getAbfsClient(); + abfsRestOperation = abfsClient2.listPath(testDirName, false, INT_50, null); + assertListstatus(fs, abfsRestOperation, testPath); + + if (isWithCPK) { + // Trying with no CPK headers + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem.get(conf); + AbfsClient abfsClient3 = fs3.getAbfsClient(); + abfsRestOperation = abfsClient3 + .listPath(testDirName, false, INT_50, null); + assertListstatus(fs, abfsRestOperation, testPath); + } + } + + private void assertListstatus(AzureBlobFileSystem fs, + AbfsRestOperation abfsRestOperation, Path testPath) throws IOException { + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + + FileStatus[] listStatuses = fs.listStatus(testPath); + Assertions.assertThat(listStatuses.length) + .describedAs("listStatuses should have 2 entries").isEqualTo(2); + + listStatuses = getSameFSWithWrongCPK(fs).listStatus(testPath); + Assertions.assertThat(listStatuses.length) + .describedAs("listStatuses should have 2 entries").isEqualTo(2); + } + + @Test + public void testCreatePathWithCPK() throws Exception { + testCreatePath(true); + } + + @Test + public void testCreatePathWithoutCPK() throws Exception { + testCreatePath(false); + } + + private void testCreatePath(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + + AbfsClient abfsClient = fs.getAbfsClient(); + FsPermission permission = new FsPermission(FsAction.EXECUTE, + FsAction.EXECUTE, FsAction.EXECUTE); + FsPermission umask = new FsPermission(FsAction.NONE, FsAction.NONE, + FsAction.NONE); + boolean isNamespaceEnabled = fs.getIsNamespaceEnabled(); + AbfsRestOperation abfsRestOperation = abfsClient + .createPath(testFileName, true, true, + isNamespaceEnabled ? getOctalNotation(permission) : null, + isNamespaceEnabled ? getOctalNotation(umask) : null, false, null); + assertCPKHeaders(abfsRestOperation, isWithCPK); + assertResponseHeader(abfsRestOperation, isWithCPK, + X_MS_ENCRYPTION_KEY_SHA256, getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, true, X_MS_REQUEST_SERVER_ENCRYPTED, + "true"); + + FileStatus[] listStatuses = fs.listStatus(new Path(testFileName)); + Assertions.assertThat(listStatuses.length) + .describedAs("listStatuses should have 1 entry").isEqualTo(1); + + listStatuses = getSameFSWithWrongCPK(fs).listStatus(new Path(testFileName)); + Assertions.assertThat(listStatuses.length) + .describedAs("listStatuses should have 1 entry").isEqualTo(1); + } + + @Test + public void testRenamePathWithCPK() throws Exception { + testRenamePath(true); + } + + @Test + public void testRenamePathWithoutCPK() throws Exception { + testRenamePath(false); + } + + private void testRenamePath(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + + FileStatus fileStatusBeforeRename = fs + .getFileStatus(new Path(testFileName)); + + String newName = "/newName"; + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .renamePath(testFileName, newName, null); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + + LambdaTestUtils.intercept(FileNotFoundException.class, + (() -> fs.getFileStatus(new Path(testFileName)))); + + FileStatus fileStatusAfterRename = fs.getFileStatus(new Path(newName)); + Assertions.assertThat(fileStatusAfterRename.getLen()) + .describedAs("File size has to be same before and after rename") + .isEqualTo(fileStatusBeforeRename.getLen()); + } + + @Test + public void testFlushWithCPK() throws Exception { + testFlush(true); + } + + @Test + public void testFlushWithoutCPK() throws Exception { + testFlush(false); + } + + private void testFlush(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + fs.create(new Path(testFileName)); + AbfsClient abfsClient = fs.getAbfsClient(); + String expectedCPKSha = getCPKSha(fs); + + byte[] fileContent = getRandomBytesArray(FILE_SIZE); + Path testFilePath = new Path(testFileName + "1"); + FSDataOutputStream oStream = fs.create(testFilePath); + oStream.write(fileContent); + + // Trying to read with different CPK headers + Configuration conf = fs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + "different-1234567890123456789012"); + try (AzureBlobFileSystem fs2 = (AzureBlobFileSystem) FileSystem.newInstance(conf); + AbfsClient abfsClient2 = fs2.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient2.flush(testFileName, 0, false, false, null, null); + }); + } + + // Trying to read with no CPK headers + if (isWithCPK) { + conf.unset(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + try (AzureBlobFileSystem fs3 = (AzureBlobFileSystem) FileSystem + .get(conf); AbfsClient abfsClient3 = fs3.getAbfsClient()) { + LambdaTestUtils.intercept(IOException.class, () -> { + abfsClient3.flush(testFileName, 0, false, false, null, null); + }); + } + } + + // With correct CPK + AbfsRestOperation abfsRestOperation = abfsClient + .flush(testFileName, 0, false, false, null, null); + assertCPKHeaders(abfsRestOperation, isWithCPK); + assertResponseHeader(abfsRestOperation, isWithCPK, + X_MS_ENCRYPTION_KEY_SHA256, expectedCPKSha); + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, true, X_MS_REQUEST_SERVER_ENCRYPTED, + isWithCPK + ""); + } + + @Test + public void testSetPathPropertiesWithCPK() throws Exception { + testSetPathProperties(true); + } + + @Test + public void testSetPathPropertiesWithoutCPK() throws Exception { + testSetPathProperties(false); + } + + private void testSetPathProperties(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + + AbfsClient abfsClient = fs.getAbfsClient(); + final Hashtable properties = new Hashtable<>(); + properties.put("key", "val"); + AbfsRestOperation abfsRestOperation = abfsClient + .setPathProperties(testFileName, + convertXmsPropertiesToCommaSeparatedString(properties)); + assertCPKHeaders(abfsRestOperation, isWithCPK); + assertResponseHeader(abfsRestOperation, isWithCPK, + X_MS_ENCRYPTION_KEY_SHA256, getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, true, X_MS_REQUEST_SERVER_ENCRYPTED, + "true"); + } + + @Test + public void testGetPathStatusFileWithCPK() throws Exception { + testGetPathStatusFile(true); + } + + @Test + public void testGetPathStatusFileWithoutCPK() throws Exception { + testGetPathStatusFile(false); + } + + private void testGetPathStatusFile(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .getPathStatus(testFileName, false); + assertCPKHeaders(abfsRestOperation, false); + assertResponseHeader(abfsRestOperation, isWithCPK, + X_MS_ENCRYPTION_KEY_SHA256, getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, true, X_MS_SERVER_ENCRYPTED, + "true"); + assertResponseHeader(abfsRestOperation, false, + X_MS_REQUEST_SERVER_ENCRYPTED, ""); + + abfsRestOperation = abfsClient.getPathStatus(testFileName, true); + assertCPKHeaders(abfsRestOperation, isWithCPK); + assertResponseHeader(abfsRestOperation, isWithCPK, + X_MS_ENCRYPTION_KEY_SHA256, getCPKSha(fs)); + assertResponseHeader(abfsRestOperation, true, X_MS_SERVER_ENCRYPTED, + "true"); + assertResponseHeader(abfsRestOperation, false, + X_MS_REQUEST_SERVER_ENCRYPTED, ""); + } + + @Test + public void testDeletePathWithCPK() throws Exception { + testDeletePath(false); + } + + @Test + public void testDeletePathWithoutCPK() throws Exception { + testDeletePath(false); + } + + private void testDeletePath(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + + FileStatus[] listStatuses = fs.listStatus(new Path(testFileName)); + Assertions.assertThat(listStatuses.length) + .describedAs("listStatuses should have 1 entry").isEqualTo(1); + + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .deletePath(testFileName, false, null); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + + Assertions.assertThatThrownBy(() -> fs.listStatus(new Path(testFileName))) + .isInstanceOf(FileNotFoundException.class); + } + + @Test + public void testSetPermissionWithCPK() throws Exception { + testSetPermission(true); + } + + @Test + public void testSetPermissionWithoutCPK() throws Exception { + testSetPermission(false); + } + + private void testSetPermission(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + Assume.assumeTrue(fs.getIsNamespaceEnabled()); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + AbfsClient abfsClient = fs.getAbfsClient(); + FsPermission permission = new FsPermission(FsAction.EXECUTE, + FsAction.EXECUTE, FsAction.EXECUTE); + AbfsRestOperation abfsRestOperation = abfsClient + .setPermission(testFileName, permission.toString()); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + } + + @Test + public void testSetAclWithCPK() throws Exception { + testSetAcl(true); + } + + @Test + public void testSetAclWithoutCPK() throws Exception { + testSetAcl(false); + } + + private void testSetAcl(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + Assume.assumeTrue(fs.getIsNamespaceEnabled()); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + AbfsClient abfsClient = fs.getAbfsClient(); + + List aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, ALL)); + final Map aclEntries = AbfsAclHelper + .deserializeAclSpec(AclEntry.aclSpecToString(aclSpec)); + + AbfsRestOperation abfsRestOperation = abfsClient + .setAcl(testFileName, AbfsAclHelper.serializeAclSpec(aclEntries)); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + } + + @Test + public void testGetAclWithCPK() throws Exception { + testGetAcl(true); + } + + @Test + public void testGetAclWithoutCPK() throws Exception { + testGetAcl(false); + } + + private void testGetAcl(final boolean isWithCPK) throws Exception { + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + Assume.assumeTrue(fs.getIsNamespaceEnabled()); + createFileAndGetContent(fs, testFileName, FILE_SIZE); + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient.getAclStatus(testFileName); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + } + + @Test + public void testCheckAccessWithCPK() throws Exception { + testCheckAccess(true); + } + + @Test + public void testCheckAccessWithoutCPK() throws Exception { + testCheckAccess(false); + } + + private void testCheckAccess(final boolean isWithCPK) throws Exception { + boolean isHNSEnabled = getConfiguration() + .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + Assume.assumeTrue(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT + " is false", + isHNSEnabled); + Assume.assumeTrue("AuthType has to be OAuth", + getAuthType() == AuthType.OAuth); + + final AzureBlobFileSystem fs = getAbfs(isWithCPK); + final String testFileName = "/" + methodName.getMethodName(); + fs.create(new Path(testFileName)); + AbfsClient abfsClient = fs.getAbfsClient(); + AbfsRestOperation abfsRestOperation = abfsClient + .checkAccess(testFileName, "rwx"); + assertCPKHeaders(abfsRestOperation, false); + assertNoCPKResponseHeadersPresent(abfsRestOperation); + } + + private byte[] createFileAndGetContent(AzureBlobFileSystem fs, + String fileName, int fileSize) throws IOException { + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + ContractTestUtils.verifyFileContents(fs, testFilePath, fileContent); + return fileContent; + } + + private void assertCPKHeaders(AbfsRestOperation abfsRestOperation, + boolean isCPKHeaderExpected) { + assertHeader(abfsRestOperation, X_MS_ENCRYPTION_KEY, isCPKHeaderExpected); + assertHeader(abfsRestOperation, X_MS_ENCRYPTION_KEY_SHA256, + isCPKHeaderExpected); + assertHeader(abfsRestOperation, X_MS_ENCRYPTION_ALGORITHM, + isCPKHeaderExpected); + } + + private void assertNoCPKResponseHeadersPresent( + AbfsRestOperation abfsRestOperation) { + assertResponseHeader(abfsRestOperation, false, X_MS_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, false, + X_MS_REQUEST_SERVER_ENCRYPTED, ""); + assertResponseHeader(abfsRestOperation, false, X_MS_ENCRYPTION_KEY_SHA256, + ""); + } + + private void assertResponseHeader(AbfsRestOperation abfsRestOperation, + boolean isHeaderExpected, String headerName, String expectedValue) { + final AbfsHttpOperation result = abfsRestOperation.getResult(); + final String value = result.getResponseHeader(headerName); + if (isHeaderExpected) { + Assertions.assertThat(value).isEqualTo(expectedValue); + } else { + Assertions.assertThat(value).isNull(); + } + } + + private void assertHeader(AbfsRestOperation abfsRestOperation, + String headerName, boolean isCPKHeaderExpected) { + assertTrue(abfsRestOperation != null); + Optional header = abfsRestOperation.getRequestHeaders() + .stream().filter(abfsHttpHeader -> abfsHttpHeader.getName() + .equalsIgnoreCase(headerName)).findFirst(); + String desc; + if (isCPKHeaderExpected) { + desc = + "CPK header " + headerName + " is expected, but the same is absent."; + } else { + desc = "CPK header " + headerName + + " is not expected, but the same is present."; + } + Assertions.assertThat(header.isPresent()).describedAs(desc) + .isEqualTo(isCPKHeaderExpected); + } + + private byte[] getSHA256Hash(String key) throws IOException { + try { + final MessageDigest digester = MessageDigest.getInstance("SHA-256"); + return digester.digest(key.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + throw new IOException(e); + } + } + + private String getCPKSha(final AzureBlobFileSystem abfs) throws IOException { + Configuration conf = abfs.getConf(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + String encryptionKey = conf + .get(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + if (encryptionKey == null || encryptionKey.isEmpty()) { + return ""; + } + return getBase64EncodedString(getSHA256Hash(encryptionKey)); + } + + private String getBase64EncodedString(byte[] bytes) { + return java.util.Base64.getEncoder().encodeToString(bytes); + } + + private Path createFileWithContent(FileSystem fs, String fileName, + byte[] fileContent) throws IOException { + Path testFilePath = new Path(fileName); + try (FSDataOutputStream oStream = fs.create(testFilePath)) { + oStream.write(fileContent); + oStream.flush(); + } + return testFilePath; + } + + private String convertXmsPropertiesToCommaSeparatedString( + final Hashtable properties) + throws CharacterCodingException { + StringBuilder commaSeparatedProperties = new StringBuilder(); + final CharsetEncoder encoder = Charset.forName(XMS_PROPERTIES_ENCODING) + .newEncoder(); + for (Map.Entry propertyEntry : properties.entrySet()) { + String key = propertyEntry.getKey(); + String value = propertyEntry.getValue(); + Boolean canEncodeValue = encoder.canEncode(value); + if (!canEncodeValue) { + throw new CharacterCodingException(); + } + String encodedPropertyValue = Base64 + .encode(encoder.encode(CharBuffer.wrap(value)).array()); + commaSeparatedProperties.append(key).append(AbfsHttpConstants.EQUAL) + .append(encodedPropertyValue); + commaSeparatedProperties.append(AbfsHttpConstants.COMMA); + } + if (commaSeparatedProperties.length() != 0) { + commaSeparatedProperties + .deleteCharAt(commaSeparatedProperties.length() - 1); + } + return commaSeparatedProperties.toString(); + } + + private String getOctalNotation(FsPermission fsPermission) { + Preconditions.checkNotNull(fsPermission, "fsPermission"); + return String + .format(AbfsHttpConstants.PERMISSION_FORMAT, fsPermission.toOctal()); + } + + private byte[] getRandomBytesArray(int length) { + final byte[] b = new byte[length]; + new Random().nextBytes(b); + return b; + } + + private AzureBlobFileSystem getAbfs(boolean withCPK) throws IOException { + return getAbfs(withCPK, "12345678901234567890123456789012"); + } + + private AzureBlobFileSystem getAbfs(boolean withCPK, String cpk) + throws IOException { + Configuration conf = getRawConfiguration(); + if (withCPK) { + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName(), + cpk); + } else { + conf.unset( + FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + getAccountName()); + } + return (AzureBlobFileSystem) FileSystem.newInstance(conf); + } + + private AzureBlobFileSystem getSameFSWithWrongCPK( + final AzureBlobFileSystem fs) throws IOException { + AbfsConfiguration abfsConf = fs.getAbfsStore().getAbfsConfiguration(); + Configuration conf = abfsConf.getRawConfiguration(); + String accountName = conf.get(FS_AZURE_ABFS_ACCOUNT_NAME); + String cpk = conf + .get(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName); + if (cpk == null || cpk.isEmpty()) { + cpk = "01234567890123456789012345678912"; + } + cpk = "different-" + cpk; + String differentCpk = cpk.substring(0, ENCRYPTION_KEY_LEN - 1); + conf.set(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY + "." + accountName, + differentCpk); + conf.set("fs.defaultFS", + "abfs://" + getFileSystemName() + "@" + accountName); + AzureBlobFileSystem sameFSWithDifferentCPK = + (AzureBlobFileSystem) FileSystem.newInstance(conf); + return sameFSWithDifferentCPK; + } + +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 72ea7661b5a90..565eb38c4f70a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -28,6 +28,7 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled"; public static final String FS_AZURE_TEST_APPENDBLOB_ENABLED = "fs.azure.test.appendblob.enabled"; + public static final String FS_AZURE_TEST_CPK_ENABLED = "fs.azure.test.cpk.enabled"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET = "fs.azure.account.oauth2.contributor.client.secret"; @@ -54,6 +55,9 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_TEST_APP_SECRET = "fs.azure.test.app.secret"; + public static final String FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT = "fs.azure.test.cpk-enabled-secondary-account"; + public static final String FS_AZURE_TEST_CPK_ENABLED_SECONDARY_ACCOUNT_KEY = "fs.azure.test.cpk-enabled-secondary-account.key"; + public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-"; public static final int TEST_TIMEOUT = 15 * 60 * 1000; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 4facc10aeff0b..a725bf3175a5c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.net.MalformedURLException; import java.net.URL; import java.util.List; import java.util.regex.Pattern; @@ -103,7 +102,7 @@ public TestAbfsClient(){ } private String getUserAgentString(AbfsConfiguration config, - boolean includeSSLProvider) throws MalformedURLException { + boolean includeSSLProvider) throws IOException { AbfsClientContext abfsClientContext = new AbfsClientContextBuilder().build(); AbfsClient client = new AbfsClient(new URL("https://azure.com"), null, config, (AccessTokenProvider) null, abfsClientContext); @@ -250,8 +249,7 @@ public void verifyUserAgentClusterType() throws Exception { public static AbfsClient createTestClientFromCurrentContext( AbfsClient baseAbfsClientInstance, - AbfsConfiguration abfsConfig) - throws AzureBlobFileSystemException { + AbfsConfiguration abfsConfig) throws IOException { AuthType currentAuthType = abfsConfig.getAuthType( abfsConfig.getAccountName()); From a967ab06f2e802f30ed34eb4ceb2742e4df8be52 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Tue, 27 Apr 2021 14:01:53 +0200 Subject: [PATCH 25/48] YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu. --- .../hadoop/yarn/event/AsyncDispatcher.java | 18 +++++++++++++++++- .../hadoop/yarn/event/TestAsyncDispatcher.java | 2 -- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 667515d00c104..ba6bb435ec25b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -25,7 +25,11 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.yarn.metrics.EventTypeMetrics; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -93,6 +97,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher { private Clock clock = new MonotonicClock(); + private ThreadPoolExecutor printEventDetailsExecutor; + /** * The thread name for dispatcher. */ @@ -179,6 +185,15 @@ protected void serviceInit(Configuration conf) throws Exception{ YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD, YarnConfiguration. DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD); + + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("PrintEventDetailsThread #%d") + .build(); + // Thread pool for async print event details, + // to prevent wasting too much time for RM. + printEventDetailsExecutor = new ThreadPoolExecutor( + 1, 5, 10, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), threadFactory); } @Override @@ -222,6 +237,7 @@ protected void serviceStop() throws Exception { LOG.warn("Interrupted Exception while stopping", ie); } } + printEventDetailsExecutor.shutdownNow(); // stop all the components super.serviceStop(); @@ -319,7 +335,7 @@ public void handle(Event event) { if (qSize != 0 && qSize % detailsInterval == 0 && lastEventDetailsQueueSizeLogged != qSize) { lastEventDetailsQueueSizeLogged = qSize; - printEventQueueDetails(); + printEventDetailsExecutor.submit(this::printEventQueueDetails); printTrigger = true; } int remCapacity = eventQueue.remainingCapacity(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java index 7d2572a4c118f..8b2dfa08b0dd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java @@ -187,8 +187,6 @@ public void testPrintDispatcherEventDetails() throws Exception { when(event.getType()).thenReturn(TestEnum.TestEventType); dispatcher.getEventHandler().handle(event); } - verify(log, atLeastOnce()).info("Event type: TestEventType, " + - "Event record counter: 5000"); Thread.sleep(2000); //Make sure more than one event to take verify(log, atLeastOnce()). From 633007a9b6fb079644cbca1d18f9d66b295b0624 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 28 Apr 2021 10:03:39 +0900 Subject: [PATCH 26/48] HADOOP-17674. Use spotbugs-maven-plugin in hadoop-huaweicloud. (#2962) Reviewed-by: Wei-Chiu Chuang --- hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index 963fa37426cc6..43360c11cd9d2 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -60,10 +60,9 @@ - org.codehaus.mojo - findbugs-maven-plugin + com.github.spotbugs + spotbugs-maven-plugin - true true ${basedir}/dev-support/findbugs-exclude.xml From 5b09ba75eccab4c2f1850bcc4cd8e60241cb500e Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Tue, 27 Apr 2021 20:44:13 -0700 Subject: [PATCH 27/48] YARN-10747. Bump YARN CSI protobuf version to 3.7.1 (#2946) Signed-off-by: Akira Ajisaka Change-Id: Ib2a4bf775a5804641a91c6f9a486772c9d87c17e --- .../hadoop-yarn/hadoop-yarn-csi/pom.xml | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml index 22565c14bcc43..62707eadb9d22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml @@ -26,7 +26,6 @@ jar - 3.6.1 1.26.0 1.5.0.Final @@ -39,7 +38,7 @@ com.google.protobuf protobuf-java - ${protobuf.version} + ${hadoop.protobuf.version} io.netty @@ -54,6 +53,13 @@ io.grpc grpc-protobuf ${grpc.version} + + + + com.google.protobuf + protobuf-java + + io.grpc @@ -91,21 +97,49 @@ hadoop-common test-jar test + + + + com.google.protobuf + protobuf-java + + org.apache.hadoop hadoop-common provided + + + + com.google.protobuf + protobuf-java + + org.apache.hadoop hadoop-yarn-common provided + + + + com.google.protobuf + protobuf-java + + org.apache.hadoop hadoop-yarn-api provided + + + + com.google.protobuf + protobuf-java + + javax.annotation @@ -158,7 +192,7 @@ protobuf-maven-plugin ${protobuf-maven-plugin.version} - com.google.protobuf:protoc:3.6.1:exe:${os.detected.classifier} + com.google.protobuf:protoc:${hadoop.protobuf.version}:exe:${os.detected.classifier} grpc-java io.grpc:protoc-gen-grpc-java:1.26.0:exe:${os.detected.classifier} From f41a368c146ab5ebea770017621256d3ff526046 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 28 Apr 2021 23:43:11 -0700 Subject: [PATCH 28/48] HADOOP-11245. Update NFS gateway to use Netty4 (#2832) Reviewed-by: Tsz-Wo Nicholas Sze --- hadoop-common-project/hadoop-nfs/pom.xml | 2 +- .../org/apache/hadoop/mount/MountdBase.java | 14 +- .../org/apache/hadoop/nfs/nfs3/Nfs3Base.java | 7 +- .../hadoop/oncrpc/RegistrationClient.java | 13 +- .../org/apache/hadoop/oncrpc/RpcInfo.java | 12 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 19 ++- .../org/apache/hadoop/oncrpc/RpcResponse.java | 23 ++-- .../org/apache/hadoop/oncrpc/RpcUtil.java | 123 +++++++++-------- .../apache/hadoop/oncrpc/SimpleTcpClient.java | 78 ++++++----- .../hadoop/oncrpc/SimpleTcpClientHandler.java | 30 ++--- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 76 ++++++----- .../apache/hadoop/oncrpc/SimpleUdpServer.java | 65 +++++---- .../java/org/apache/hadoop/oncrpc/XDR.java | 12 +- .../org/apache/hadoop/portmap/Portmap.java | 126 +++++++++++------- .../hadoop/portmap/RpcProgramPortmap.java | 46 ++++--- .../hadoop/oncrpc/TestFrameDecoder.java | 100 +++++++------- .../apache/hadoop/portmap/TestPortmap.java | 2 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +- .../hdfs/nfs/mount/RpcProgramMountd.java | 12 +- .../hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 12 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 2 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 14 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 2 +- .../hadoop/hdfs/nfs/nfs3/WriteManager.java | 2 +- .../hadoop/hdfs/nfs/TestOutOfOrderWrite.java | 32 ++--- .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 2 +- .../hadoop/hdfs/nfs/nfs3/TestWrites.java | 2 +- 27 files changed, 472 insertions(+), 358 deletions(-) diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index f5de8407a0920..33d8b3710c528 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -90,7 +90,7 @@ io.netty - netty + netty-all compile diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java index 0ff3084bf3eb9..58d3e51f2bdfb 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java @@ -41,6 +41,8 @@ abstract public class MountdBase { private final RpcProgram rpcProgram; private int udpBoundPort; // Will set after server starts private int tcpBoundPort; // Will set after server starts + private SimpleUdpServer udpServer = null; + private SimpleTcpServer tcpServer = null; public RpcProgram getRpcProgram() { return rpcProgram; @@ -57,7 +59,7 @@ public MountdBase(RpcProgram program) throws IOException { /* Start UDP server */ private void startUDPServer() { - SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.getPort(), + udpServer = new SimpleUdpServer(rpcProgram.getPort(), rpcProgram, 1); rpcProgram.startDaemons(); try { @@ -76,7 +78,7 @@ private void startUDPServer() { /* Start TCP server */ private void startTCPServer() { - SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(), + tcpServer = new SimpleTcpServer(rpcProgram.getPort(), rpcProgram, 1); rpcProgram.startDaemons(); try { @@ -118,6 +120,14 @@ public void stop() { rpcProgram.unregister(PortmapMapping.TRANSPORT_TCP, tcpBoundPort); tcpBoundPort = 0; } + if (udpServer != null) { + udpServer.shutdown(); + udpServer = null; + } + if (tcpServer != null) { + tcpServer.shutdown(); + tcpServer = null; + } } /** diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java index ff83a5f19bee1..e6ea29b42bff4 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java @@ -35,6 +35,7 @@ public abstract class Nfs3Base { public static final Logger LOG = LoggerFactory.getLogger(Nfs3Base.class); private final RpcProgram rpcProgram; private int nfsBoundPort; // Will set after server starts + private SimpleTcpServer tcpServer = null; public RpcProgram getRpcProgram() { return rpcProgram; @@ -61,7 +62,7 @@ public void start(boolean register) { } private void startTCPServer() { - SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(), + tcpServer = new SimpleTcpServer(rpcProgram.getPort(), rpcProgram, 0); rpcProgram.startDaemons(); try { @@ -84,6 +85,10 @@ public void stop() { nfsBoundPort = 0; } rpcProgram.stopDaemons(); + if (tcpServer != null) { + tcpServer.shutdown(); + tcpServer = null; + } } /** * Priority of the nfsd shutdown hook. diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RegistrationClient.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RegistrationClient.java index c8528ba4d558f..c96f1d53bb4c5 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RegistrationClient.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RegistrationClient.java @@ -19,10 +19,9 @@ import java.util.Arrays; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; import org.apache.hadoop.oncrpc.RpcAcceptedReply.AcceptState; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.MessageEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,10 +57,10 @@ private boolean validMessageLength(int len) { } @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { - ChannelBuffer buf = (ChannelBuffer) e.getMessage(); // Read reply + public void channelRead(ChannelHandlerContext ctx, Object msg) { + ByteBuf buf = (ByteBuf) msg; // Read reply if (!validMessageLength(buf.readableBytes())) { - e.getChannel().close(); + ctx.channel().close(); return; } @@ -83,7 +82,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { RpcDeniedReply deniedReply = (RpcDeniedReply) reply; handle(deniedReply); } - e.getChannel().close(); // shutdown now that request is complete + ctx.channel().close(); // shutdown now that request is complete } private void handle(RpcDeniedReply deniedReply) { diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcInfo.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcInfo.java index b434d79285c6f..aba8e9ea2624e 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcInfo.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcInfo.java @@ -19,9 +19,9 @@ import java.net.SocketAddress; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; /** * RpcInfo records all contextual information of an RPC message. It contains @@ -29,11 +29,11 @@ */ public final class RpcInfo { private final RpcMessage header; - private final ChannelBuffer data; + private final ByteBuf data; private final Channel channel; private final SocketAddress remoteAddress; - public RpcInfo(RpcMessage header, ChannelBuffer data, + public RpcInfo(RpcMessage header, ByteBuf data, ChannelHandlerContext channelContext, Channel channel, SocketAddress remoteAddress) { this.header = header; @@ -46,7 +46,7 @@ public RpcMessage header() { return header; } - public ChannelBuffer data() { + public ByteBuf data() { return data; } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java index d4b2261e05f22..be927b37efbc3 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java @@ -22,17 +22,16 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.oncrpc.RpcAcceptedReply.AcceptState; import org.apache.hadoop.oncrpc.security.Verifier; import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.portmap.PortmapMapping; import org.apache.hadoop.portmap.PortmapRequest; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.MessageEvent; -import org.jboss.netty.channel.SimpleChannelUpstreamHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +39,7 @@ * Class for writing RPC server programs based on RFC 1050. Extend this class * and implement {@link #handleInternal} to handle the requests received. */ -public abstract class RpcProgram extends SimpleChannelUpstreamHandler { +public abstract class RpcProgram extends ChannelInboundHandlerAdapter { static final Logger LOG = LoggerFactory.getLogger(RpcProgram.class); public static final int RPCB_PORT = 111; private final String program; @@ -162,9 +161,9 @@ public void startDaemons() {} public void stopDaemons() {} @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - RpcInfo info = (RpcInfo) e.getMessage(); + RpcInfo info = (RpcInfo) msg; RpcCall call = (RpcCall) info.header(); SocketAddress remoteAddress = info.remoteAddress(); @@ -222,7 +221,7 @@ private void sendAcceptedReply(RpcCall call, SocketAddress remoteAddress, out.writeInt(lowProgVersion); out.writeInt(highProgVersion); } - ChannelBuffer b = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap() + ByteBuf b = Unpooled.wrappedBuffer(out.asReadOnlyWrap() .buffer()); RpcResponse rsp = new RpcResponse(b, remoteAddress); RpcUtil.sendRpcResponse(ctx, rsp); @@ -235,7 +234,7 @@ protected static void sendRejectedReply(RpcCall call, RpcReply.ReplyState.MSG_DENIED, RpcDeniedReply.RejectState.AUTH_ERROR, new VerifierNone()); reply.write(out); - ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap() + ByteBuf buf = Unpooled.wrappedBuffer(out.asReadOnlyWrap() .buffer()); RpcResponse rsp = new RpcResponse(buf, remoteAddress); RpcUtil.sendRpcResponse(ctx, rsp); diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcResponse.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcResponse.java index 2e45e6100b108..0d6431f68bd5a 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcResponse.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcResponse.java @@ -19,27 +19,30 @@ import java.net.SocketAddress; -import org.jboss.netty.buffer.ChannelBuffer; +import io.netty.buffer.ByteBuf; +import io.netty.channel.DefaultAddressedEnvelope; /** * RpcResponse encapsulates a response to a RPC request. It contains the data * that is going to cross the wire, as well as the information of the remote * peer. */ -public class RpcResponse { - private final ChannelBuffer data; - private final SocketAddress remoteAddress; +public class RpcResponse extends + DefaultAddressedEnvelope { + public RpcResponse(ByteBuf message, SocketAddress recipient) { + super(message, recipient, null); + } - public RpcResponse(ChannelBuffer data, SocketAddress remoteAddress) { - this.data = data; - this.remoteAddress = remoteAddress; + public RpcResponse(ByteBuf message, SocketAddress recipient, + SocketAddress sender) { + super(message, recipient, sender); } - public ChannelBuffer data() { - return data; + public ByteBuf data() { + return this.content(); } public SocketAddress remoteAddress() { - return remoteAddress; + return this.recipient(); } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java index cebebd27d0c4b..e8bc27d687fea 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java @@ -17,16 +17,18 @@ */ package org.apache.hadoop.oncrpc; +import java.net.SocketAddress; import java.nio.ByteBuffer; - -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.MessageEvent; -import org.jboss.netty.channel.SimpleChannelUpstreamHandler; -import org.jboss.netty.handler.codec.frame.FrameDecoder; +import java.util.List; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.socket.DatagramPacket; +import io.netty.handler.codec.ByteToMessageDecoder; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,16 +45,16 @@ public static int getNewXid(String caller) { public static void sendRpcResponse(ChannelHandlerContext ctx, RpcResponse response) { - Channels.fireMessageReceived(ctx, response); + ctx.fireChannelRead(response); } - public static FrameDecoder constructRpcFrameDecoder() { + public static ByteToMessageDecoder constructRpcFrameDecoder() { return new RpcFrameDecoder(); } - public static final SimpleChannelUpstreamHandler STAGE_RPC_MESSAGE_PARSER = new RpcMessageParserStage(); - public static final SimpleChannelUpstreamHandler STAGE_RPC_TCP_RESPONSE = new RpcTcpResponseStage(); - public static final SimpleChannelUpstreamHandler STAGE_RPC_UDP_RESPONSE = new RpcUdpResponseStage(); + public static final ChannelInboundHandlerAdapter STAGE_RPC_MESSAGE_PARSER = new RpcMessageParserStage(); + public static final ChannelInboundHandlerAdapter STAGE_RPC_TCP_RESPONSE = new RpcTcpResponseStage(); + public static final ChannelInboundHandlerAdapter STAGE_RPC_UDP_RESPONSE = new RpcUdpResponseStage(); /** * An RPC client can separate a RPC message into several frames (i.e., @@ -62,44 +64,39 @@ public static FrameDecoder constructRpcFrameDecoder() { * RpcFrameDecoder is a stateful pipeline stage. It has to be constructed for * each RPC client. */ - static class RpcFrameDecoder extends FrameDecoder { + static class RpcFrameDecoder extends ByteToMessageDecoder { public static final Logger LOG = LoggerFactory.getLogger(RpcFrameDecoder.class); - private ChannelBuffer currentFrame; + private volatile boolean isLast; @Override - protected Object decode(ChannelHandlerContext ctx, Channel channel, - ChannelBuffer buf) { + protected void decode(ChannelHandlerContext ctx, ByteBuf buf, + List out) { - if (buf.readableBytes() < 4) - return null; + if (buf.readableBytes() < 4) { + return; + } buf.markReaderIndex(); byte[] fragmentHeader = new byte[4]; buf.readBytes(fragmentHeader); int length = XDR.fragmentSize(fragmentHeader); - boolean isLast = XDR.isLastFragment(fragmentHeader); + isLast = XDR.isLastFragment(fragmentHeader); if (buf.readableBytes() < length) { buf.resetReaderIndex(); - return null; + return; } - ChannelBuffer newFragment = buf.readSlice(length); - if (currentFrame == null) { - currentFrame = newFragment; - } else { - currentFrame = ChannelBuffers.wrappedBuffer(currentFrame, newFragment); - } + ByteBuf newFragment = buf.readSlice(length); + newFragment.retain(); + out.add(newFragment); + } - if (isLast) { - ChannelBuffer completeFrame = currentFrame; - currentFrame = null; - return completeFrame; - } else { - return null; - } + @VisibleForTesting + public boolean isLast() { + return isLast; } } @@ -107,30 +104,44 @@ protected Object decode(ChannelHandlerContext ctx, Channel channel, * RpcMessageParserStage parses the network bytes and encapsulates the RPC * request into a RpcInfo instance. */ - static final class RpcMessageParserStage extends SimpleChannelUpstreamHandler { + @ChannelHandler.Sharable + static final class RpcMessageParserStage extends ChannelInboundHandlerAdapter { private static final Logger LOG = LoggerFactory .getLogger(RpcMessageParserStage.class); @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - ChannelBuffer buf = (ChannelBuffer) e.getMessage(); - ByteBuffer b = buf.toByteBuffer().asReadOnlyBuffer(); + ByteBuf buf; + SocketAddress remoteAddress; + if (msg instanceof DatagramPacket) { + DatagramPacket packet = (DatagramPacket)msg; + buf = packet.content(); + remoteAddress = packet.sender(); + } else { + buf = (ByteBuf) msg; + remoteAddress = ctx.channel().remoteAddress(); + } + + ByteBuffer b = buf.nioBuffer().asReadOnlyBuffer(); XDR in = new XDR(b, XDR.State.READING); RpcInfo info = null; try { RpcCall callHeader = RpcCall.read(in); - ChannelBuffer dataBuffer = ChannelBuffers.wrappedBuffer(in.buffer() + ByteBuf dataBuffer = Unpooled.wrappedBuffer(in.buffer() .slice()); - info = new RpcInfo(callHeader, dataBuffer, ctx, e.getChannel(), - e.getRemoteAddress()); + + info = new RpcInfo(callHeader, dataBuffer, ctx, ctx.channel(), + remoteAddress); } catch (Exception exc) { - LOG.info("Malformed RPC request from " + e.getRemoteAddress()); + LOG.info("Malformed RPC request from " + remoteAddress); + } finally { + buf.release(); } if (info != null) { - Channels.fireMessageReceived(ctx, info); + ctx.fireChannelRead(info); } } } @@ -139,16 +150,17 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) * RpcTcpResponseStage sends an RpcResponse across the wire with the * appropriate fragment header. */ - private static class RpcTcpResponseStage extends SimpleChannelUpstreamHandler { + @ChannelHandler.Sharable + private static class RpcTcpResponseStage extends ChannelInboundHandlerAdapter { @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - RpcResponse r = (RpcResponse) e.getMessage(); + RpcResponse r = (RpcResponse) msg; byte[] fragmentHeader = XDR.recordMark(r.data().readableBytes(), true); - ChannelBuffer header = ChannelBuffers.wrappedBuffer(fragmentHeader); - ChannelBuffer d = ChannelBuffers.wrappedBuffer(header, r.data()); - e.getChannel().write(d); + ByteBuf header = Unpooled.wrappedBuffer(fragmentHeader); + ByteBuf d = Unpooled.wrappedBuffer(header, r.data()); + ctx.channel().writeAndFlush(d); } } @@ -156,14 +168,17 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) * RpcUdpResponseStage sends an RpcResponse as a UDP packet, which does not * require a fragment header. */ + @ChannelHandler.Sharable private static final class RpcUdpResponseStage extends - SimpleChannelUpstreamHandler { + ChannelInboundHandlerAdapter { @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - RpcResponse r = (RpcResponse) e.getMessage(); - e.getChannel().write(r.data(), r.remoteAddress()); + RpcResponse r = (RpcResponse) msg; + // TODO: check out https://github.com/netty/netty/issues/1282 for + // correct usage + ctx.channel().writeAndFlush(r.data()); } } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClient.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClient.java index 32e1b4b839218..7cfef6439b059 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClient.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClient.java @@ -18,15 +18,16 @@ package org.apache.hadoop.oncrpc; import java.net.InetSocketAddress; -import java.util.concurrent.Executors; -import org.jboss.netty.bootstrap.ClientBootstrap; -import org.jboss.netty.channel.ChannelFactory; -import org.jboss.netty.channel.ChannelFuture; -import org.jboss.netty.channel.ChannelPipeline; -import org.jboss.netty.channel.ChannelPipelineFactory; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * A simple TCP based RPC client which just sends a request to a server. @@ -35,8 +36,9 @@ public class SimpleTcpClient { protected final String host; protected final int port; protected final XDR request; - protected ChannelPipelineFactory pipelineFactory; protected final boolean oneShot; + private NioEventLoopGroup workerGroup; + private ChannelFuture future; public SimpleTcpClient(String host, int port, XDR request) { this(host,port, request, true); @@ -48,40 +50,54 @@ public SimpleTcpClient(String host, int port, XDR request, Boolean oneShot) { this.request = request; this.oneShot = oneShot; } - - protected ChannelPipelineFactory setPipelineFactory() { - this.pipelineFactory = new ChannelPipelineFactory() { + + protected ChannelInitializer setChannelHandler() { + return new ChannelInitializer() { @Override - public ChannelPipeline getPipeline() { - return Channels.pipeline( + protected void initChannel(SocketChannel ch) throws Exception { + ChannelPipeline p = ch.pipeline(); + p.addLast( RpcUtil.constructRpcFrameDecoder(), - new SimpleTcpClientHandler(request)); + new SimpleTcpClientHandler(request) + ); } }; - return this.pipelineFactory; } + @VisibleForTesting public void run() { // Configure the client. - ChannelFactory factory = new NioClientSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), 1, 1); - ClientBootstrap bootstrap = new ClientBootstrap(factory); - - // Set up the pipeline factory. - bootstrap.setPipelineFactory(setPipelineFactory()); - - bootstrap.setOption("tcpNoDelay", true); - bootstrap.setOption("keepAlive", true); + workerGroup = new NioEventLoopGroup(); + Bootstrap bootstrap = new Bootstrap() + .group(workerGroup) + .channel(NioSocketChannel.class); - // Start the connection attempt. - ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port)); + try { + future = bootstrap.handler(setChannelHandler()) + .option(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_KEEPALIVE, true) + .connect(new InetSocketAddress(host, port)).sync(); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + if (oneShot) { + stop(); + } + } + } - if (oneShot) { - // Wait until the connection is closed or the connection attempt fails. - future.getChannel().getCloseFuture().awaitUninterruptibly(); + public void stop() { + try { + if (future != null) { + // Wait until the connection is closed or the connection attempt fails. + future.channel().closeFuture().sync(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { // Shut down thread pools to exit. - bootstrap.releaseExternalResources(); + workerGroup.shutdownGracefully(); } } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java index 23b6682361c9b..1acefc857f830 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java @@ -17,19 +17,19 @@ */ package org.apache.hadoop.oncrpc; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.ChannelStateEvent; -import org.jboss.netty.channel.ExceptionEvent; -import org.jboss.netty.channel.MessageEvent; -import org.jboss.netty.channel.SimpleChannelHandler; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.util.ReferenceCountUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A simple TCP based RPC client handler used by {@link SimpleTcpServer}. */ -public class SimpleTcpClientHandler extends SimpleChannelHandler { +public class SimpleTcpClientHandler extends ChannelInboundHandlerAdapter { public static final Logger LOG = LoggerFactory.getLogger(SimpleTcpClient.class); protected final XDR request; @@ -39,13 +39,13 @@ public SimpleTcpClientHandler(XDR request) { } @Override - public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) { + public void channelActive(ChannelHandlerContext ctx) throws Exception { // Send the request if (LOG.isDebugEnabled()) { LOG.debug("sending PRC request"); } - ChannelBuffer outBuf = XDR.writeMessageTcp(request, true); - e.getChannel().write(outBuf); + ByteBuf outBuf = XDR.writeMessageTcp(request, true); + ctx.channel().writeAndFlush(outBuf); } /** @@ -53,13 +53,13 @@ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) { * more interaction with the server. */ @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { - e.getChannel().close(); + public void channelRead(ChannelHandlerContext ctx, Object msg) { + ctx.channel().close(); } @Override - public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { - LOG.warn("Unexpected exception from downstream: ", e.getCause()); - e.getChannel().close(); + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + LOG.warn("Unexpected exception from downstream: ", cause.getCause()); + ctx.channel().close(); } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java index 177fa3d80b1b4..29155c80b1846 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java @@ -20,14 +20,17 @@ import java.net.InetSocketAddress; import java.util.concurrent.Executors; -import org.jboss.netty.bootstrap.ServerBootstrap; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelFactory; -import org.jboss.netty.channel.ChannelPipeline; -import org.jboss.netty.channel.ChannelPipelineFactory; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.SimpleChannelUpstreamHandler; -import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,9 +42,11 @@ public class SimpleTcpServer { LoggerFactory.getLogger(SimpleTcpServer.class); protected final int port; protected int boundPort = -1; // Will be set after server starts - protected final SimpleChannelUpstreamHandler rpcProgram; + protected final ChannelInboundHandlerAdapter rpcProgram; private ServerBootstrap server; private Channel ch; + private EventLoopGroup bossGroup; + private EventLoopGroup workerGroup; /** The maximum number of I/O worker threads */ protected final int workerCount; @@ -57,37 +62,32 @@ public SimpleTcpServer(int port, RpcProgram program, int workercount) { this.workerCount = workercount; } - public void run() { + public void run() throws InterruptedException { // Configure the Server. - ChannelFactory factory; - if (workerCount == 0) { - // Use default workers: 2 * the number of available processors - factory = new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool()); - } else { - factory = new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), - workerCount); - } + bossGroup = new NioEventLoopGroup(); + workerGroup = new NioEventLoopGroup(workerCount, Executors.newCachedThreadPool()); - server = new ServerBootstrap(factory); - server.setPipelineFactory(new ChannelPipelineFactory() { + server = new ServerBootstrap(); + server.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { @Override - public ChannelPipeline getPipeline() throws Exception { - return Channels.pipeline(RpcUtil.constructRpcFrameDecoder(), + protected void initChannel(SocketChannel ch) throws Exception { + ChannelPipeline p = ch.pipeline(); + p.addLast(RpcUtil.constructRpcFrameDecoder(), RpcUtil.STAGE_RPC_MESSAGE_PARSER, rpcProgram, RpcUtil.STAGE_RPC_TCP_RESPONSE); - } - }); - server.setOption("child.tcpNoDelay", true); - server.setOption("child.keepAlive", true); - server.setOption("child.reuseAddress", true); - server.setOption("reuseAddress", true); + }}) + .childOption(ChannelOption.TCP_NODELAY, true) + .childOption(ChannelOption.SO_KEEPALIVE, true) + .childOption(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_REUSEADDR, true); // Listen to TCP port - ch = server.bind(new InetSocketAddress(port)); - InetSocketAddress socketAddr = (InetSocketAddress) ch.getLocalAddress(); + ChannelFuture f = server.bind(new InetSocketAddress(port)).sync(); + ch = f.channel(); + InetSocketAddress socketAddr = (InetSocketAddress) ch.localAddress(); boundPort = socketAddr.getPort(); LOG.info("Started listening to TCP requests at port " + boundPort + " for " @@ -102,9 +102,17 @@ public int getBoundPort() { public void shutdown() { if (ch != null) { ch.close().awaitUninterruptibly(); + ch = null; + } + + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + workerGroup = null; } - if (server != null) { - server.releaseExternalResources(); + + if (bossGroup != null) { + bossGroup.shutdownGracefully(); + bossGroup = null; } } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java index e65003ca64beb..516503c323a08 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java @@ -20,12 +20,16 @@ import java.net.InetSocketAddress; import java.util.concurrent.Executors; -import org.jboss.netty.bootstrap.ConnectionlessBootstrap; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.SimpleChannelUpstreamHandler; -import org.jboss.netty.channel.socket.DatagramChannelFactory; -import org.jboss.netty.channel.socket.nio.NioDatagramChannelFactory; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioDatagramChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,36 +43,45 @@ public class SimpleUdpServer { private final int RECEIVE_BUFFER_SIZE = 65536; protected final int port; - protected final SimpleChannelUpstreamHandler rpcProgram; + protected final ChannelInboundHandlerAdapter rpcProgram; protected final int workerCount; protected int boundPort = -1; // Will be set after server starts - private ConnectionlessBootstrap server; + private Bootstrap server; private Channel ch; + private EventLoopGroup workerGroup; - public SimpleUdpServer(int port, SimpleChannelUpstreamHandler program, + public SimpleUdpServer(int port, ChannelInboundHandlerAdapter program, int workerCount) { this.port = port; this.rpcProgram = program; this.workerCount = workerCount; } - public void run() { - // Configure the client. - DatagramChannelFactory f = new NioDatagramChannelFactory( - Executors.newCachedThreadPool(), workerCount); + public void run() throws InterruptedException { + workerGroup = new NioEventLoopGroup(workerCount, Executors.newCachedThreadPool()); - server = new ConnectionlessBootstrap(f); - server.setPipeline(Channels.pipeline(RpcUtil.STAGE_RPC_MESSAGE_PARSER, - rpcProgram, RpcUtil.STAGE_RPC_UDP_RESPONSE)); - - server.setOption("broadcast", "false"); - server.setOption("sendBufferSize", SEND_BUFFER_SIZE); - server.setOption("receiveBufferSize", RECEIVE_BUFFER_SIZE); - server.setOption("reuseAddress", true); + server = new Bootstrap(); + server.group(workerGroup) + .channel(NioDatagramChannel.class) + .option(ChannelOption.SO_BROADCAST, true) + .option(ChannelOption.SO_SNDBUF, SEND_BUFFER_SIZE) + .option(ChannelOption.SO_RCVBUF, RECEIVE_BUFFER_SIZE) + .option(ChannelOption.SO_REUSEADDR, true) + .handler(new ChannelInitializer() { + @Override protected void initChannel(NioDatagramChannel ch) + throws Exception { + ChannelPipeline p = ch.pipeline(); + p.addLast( + RpcUtil.STAGE_RPC_MESSAGE_PARSER, + rpcProgram, + RpcUtil.STAGE_RPC_UDP_RESPONSE); + } + }); // Listen to the UDP port - ch = server.bind(new InetSocketAddress(port)); - InetSocketAddress socketAddr = (InetSocketAddress) ch.getLocalAddress(); + ChannelFuture f = server.bind(new InetSocketAddress(port)).sync(); + ch = f.channel(); + InetSocketAddress socketAddr = (InetSocketAddress) ch.localAddress(); boundPort = socketAddr.getPort(); LOG.info("Started listening to UDP requests at port " + boundPort + " for " @@ -83,9 +96,11 @@ public int getBoundPort() { public void shutdown() { if (ch != null) { ch.close().awaitUninterruptibly(); + ch = null; } - if (server != null) { - server.releaseExternalResources(); + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + workerGroup = null; } } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java index 419eff831f0e7..6000fd57a1b65 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java @@ -20,8 +20,8 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; @@ -242,7 +242,7 @@ static byte[] recordMark(int size, boolean last) { * @param last specifies last request or not * @return TCP buffer */ - public static ChannelBuffer writeMessageTcp(XDR request, boolean last) { + public static ByteBuf writeMessageTcp(XDR request, boolean last) { Preconditions.checkState(request.state == XDR.State.WRITING); ByteBuffer b = request.buf.duplicate(); b.flip(); @@ -250,7 +250,7 @@ public static ChannelBuffer writeMessageTcp(XDR request, boolean last) { ByteBuffer headerBuf = ByteBuffer.wrap(fragmentHeader); // TODO: Investigate whether making a copy of the buffer is necessary. - return ChannelBuffers.copiedBuffer(headerBuf, b); + return Unpooled.wrappedBuffer(headerBuf, b); } /** @@ -258,10 +258,10 @@ public static ChannelBuffer writeMessageTcp(XDR request, boolean last) { * @param response XDR response * @return UDP buffer */ - public static ChannelBuffer writeMessageUdp(XDR response) { + public static ByteBuf writeMessageUdp(XDR response) { Preconditions.checkState(response.state == XDR.State.READING); // TODO: Investigate whether making a copy of the buffer is necessary. - return ChannelBuffers.copiedBuffer(response.buf); + return Unpooled.copiedBuffer(response.buf); } public static int fragmentSize(byte[] mark) { diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java index 80f43828ea83b..1a8a305436ce9 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java @@ -22,21 +22,27 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.group.ChannelGroup; +import io.netty.channel.group.DefaultChannelGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioDatagramChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.GlobalEventExecutor; import org.apache.hadoop.oncrpc.RpcProgram; import org.apache.hadoop.oncrpc.RpcUtil; import org.apache.hadoop.util.StringUtils; -import org.jboss.netty.bootstrap.ConnectionlessBootstrap; -import org.jboss.netty.bootstrap.ServerBootstrap; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelPipeline; -import org.jboss.netty.channel.ChannelPipelineFactory; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.group.ChannelGroup; -import org.jboss.netty.channel.group.DefaultChannelGroup; -import org.jboss.netty.channel.socket.nio.NioDatagramChannelFactory; -import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory; -import org.jboss.netty.handler.timeout.IdleStateHandler; -import org.jboss.netty.util.HashedWheelTimer; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -49,11 +55,17 @@ final class Portmap { private static final Logger LOG = LoggerFactory.getLogger(Portmap.class); private static final int DEFAULT_IDLE_TIME_MILLISECONDS = 5000; - private ConnectionlessBootstrap udpServer; + private Bootstrap udpServer; private ServerBootstrap tcpServer; - private ChannelGroup allChannels = new DefaultChannelGroup(); + private ChannelGroup allChannels = new DefaultChannelGroup( + GlobalEventExecutor.INSTANCE); private Channel udpChannel; private Channel tcpChannel; + + EventLoopGroup bossGroup; + EventLoopGroup workerGroup; + EventLoopGroup udpGroup; + private final RpcProgramPortmap handler = new RpcProgramPortmap(allChannels); public static void main(String[] args) { @@ -73,18 +85,19 @@ public static void main(String[] args) { void shutdown() { allChannels.close().awaitUninterruptibly(); - tcpServer.releaseExternalResources(); - udpServer.releaseExternalResources(); + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + udpGroup.shutdownGracefully(); } @VisibleForTesting SocketAddress getTcpServerLocalAddress() { - return tcpChannel.getLocalAddress(); + return tcpChannel.localAddress(); } @VisibleForTesting SocketAddress getUdpServerLoAddress() { - return udpChannel.getLocalAddress(); + return udpChannel.localAddress(); } @VisibleForTesting @@ -93,38 +106,55 @@ RpcProgramPortmap getHandler() { } void start(final int idleTimeMilliSeconds, final SocketAddress tcpAddress, - final SocketAddress udpAddress) { - - tcpServer = new ServerBootstrap(new NioServerSocketChannelFactory( - Executors.newCachedThreadPool(), Executors.newCachedThreadPool())); - tcpServer.setPipelineFactory(new ChannelPipelineFactory() { - private final HashedWheelTimer timer = new HashedWheelTimer(); - private final IdleStateHandler idleStateHandler = new IdleStateHandler( - timer, 0, 0, idleTimeMilliSeconds, TimeUnit.MILLISECONDS); - - @Override - public ChannelPipeline getPipeline() throws Exception { - return Channels.pipeline(RpcUtil.constructRpcFrameDecoder(), - RpcUtil.STAGE_RPC_MESSAGE_PARSER, idleStateHandler, handler, - RpcUtil.STAGE_RPC_TCP_RESPONSE); - } - }); - tcpServer.setOption("reuseAddress", true); - tcpServer.setOption("child.reuseAddress", true); - - udpServer = new ConnectionlessBootstrap(new NioDatagramChannelFactory( - Executors.newCachedThreadPool())); - - udpServer.setPipeline(Channels.pipeline(RpcUtil.STAGE_RPC_MESSAGE_PARSER, - handler, RpcUtil.STAGE_RPC_UDP_RESPONSE)); - udpServer.setOption("reuseAddress", true); - - tcpChannel = tcpServer.bind(tcpAddress); - udpChannel = udpServer.bind(udpAddress); + final SocketAddress udpAddress) throws InterruptedException { + + bossGroup = new NioEventLoopGroup(); + workerGroup = new NioEventLoopGroup(0, Executors.newCachedThreadPool()); + + tcpServer = new ServerBootstrap(); + tcpServer.group(bossGroup, workerGroup) + .option(ChannelOption.SO_REUSEADDR, true) + .childOption(ChannelOption.SO_REUSEADDR, true) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + private final IdleStateHandler idleStateHandler = new IdleStateHandler( + 0, 0, idleTimeMilliSeconds, TimeUnit.MILLISECONDS); + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ChannelPipeline p = ch.pipeline(); + + p.addLast(RpcUtil.constructRpcFrameDecoder(), + RpcUtil.STAGE_RPC_MESSAGE_PARSER, idleStateHandler, handler, + RpcUtil.STAGE_RPC_TCP_RESPONSE); + }}); + + udpGroup = new NioEventLoopGroup(0, Executors.newCachedThreadPool()); + + udpServer = new Bootstrap(); + udpServer.group(udpGroup) + .channel(NioDatagramChannel.class) + .handler(new ChannelInitializer() { + @Override protected void initChannel(NioDatagramChannel ch) + throws Exception { + ChannelPipeline p = ch.pipeline(); + p.addLast( + new LoggingHandler(LogLevel.DEBUG), + RpcUtil.STAGE_RPC_MESSAGE_PARSER, handler, RpcUtil.STAGE_RPC_UDP_RESPONSE); + } + }) + .option(ChannelOption.SO_REUSEADDR, true); + + ChannelFuture tcpChannelFuture = null; + tcpChannelFuture = tcpServer.bind(tcpAddress); + ChannelFuture udpChannelFuture = udpServer.bind(udpAddress); + tcpChannel = tcpChannelFuture.sync().channel(); + udpChannel = udpChannelFuture.sync().channel(); + allChannels.add(tcpChannel); allChannels.add(udpChannel); - LOG.info("Portmap server started at tcp://" + tcpChannel.getLocalAddress() - + ", udp://" + udpChannel.getLocalAddress()); + LOG.info("Portmap server started at tcp://" + tcpChannel.localAddress() + + ", udp://" + udpChannel.localAddress()); } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java index 0bc380f614c1c..7b33a644fbe76 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java @@ -19,6 +19,14 @@ import java.util.concurrent.ConcurrentHashMap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.group.ChannelGroup; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.handler.timeout.IdleStateHandler; import org.apache.hadoop.oncrpc.RpcAcceptedReply; import org.apache.hadoop.oncrpc.RpcCall; import org.apache.hadoop.oncrpc.RpcInfo; @@ -27,20 +35,12 @@ import org.apache.hadoop.oncrpc.RpcUtil; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.VerifierNone; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.ChannelStateEvent; -import org.jboss.netty.channel.ExceptionEvent; -import org.jboss.netty.channel.MessageEvent; -import org.jboss.netty.channel.group.ChannelGroup; -import org.jboss.netty.handler.timeout.IdleState; -import org.jboss.netty.handler.timeout.IdleStateAwareChannelUpstreamHandler; -import org.jboss.netty.handler.timeout.IdleStateEvent; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -final class RpcProgramPortmap extends IdleStateAwareChannelUpstreamHandler { +@ChannelHandler.Sharable +final class RpcProgramPortmap extends IdleStateHandler { static final int PROGRAM = 100000; static final int VERSION = 2; @@ -60,6 +60,8 @@ final class RpcProgramPortmap extends IdleStateAwareChannelUpstreamHandler { private final ChannelGroup allChannels; RpcProgramPortmap(ChannelGroup allChannels) { + super(1, 1, 1); + // FIXME: set default idle timeout 1 second. this.allChannels = allChannels; PortmapMapping m = new PortmapMapping(PROGRAM, VERSION, PortmapMapping.TRANSPORT_TCP, RpcProgram.RPCB_PORT); @@ -151,14 +153,14 @@ private XDR dump(int xid, XDR in, XDR out) { } @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - RpcInfo info = (RpcInfo) e.getMessage(); + RpcInfo info = (RpcInfo) msg; RpcCall rpcCall = (RpcCall) info.header(); final int portmapProc = rpcCall.getProcedure(); int xid = rpcCall.getXid(); - XDR in = new XDR(info.data().toByteBuffer().asReadOnlyBuffer(), + XDR in = new XDR(info.data().nioBuffer().asReadOnlyBuffer(), XDR.State.READING); XDR out = new XDR(); @@ -181,29 +183,29 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) reply.write(out); } - ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap() + ByteBuf buf = Unpooled.wrappedBuffer(out.asReadOnlyWrap() .buffer()); RpcResponse rsp = new RpcResponse(buf, info.remoteAddress()); RpcUtil.sendRpcResponse(ctx, rsp); } @Override - public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) + public void channelActive(ChannelHandlerContext ctx) throws Exception { - allChannels.add(e.getChannel()); + allChannels.add(ctx.channel()); } @Override public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception { - if (e.getState() == IdleState.ALL_IDLE) { - e.getChannel().close(); + if (e.state() == IdleState.ALL_IDLE) { + ctx.channel().close(); } } @Override - public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) { - LOG.warn("Encountered ", e.getCause()); - e.getChannel().close(); + public void exceptionCaught(ChannelHandlerContext ctx, Throwable t) { + LOG.warn("Encountered ", t); + ctx.channel().close(); } } diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java index 0e416b3738d20..6d103fdd781c6 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java @@ -22,19 +22,19 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Random; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelException; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; import org.apache.hadoop.oncrpc.RpcUtil.RpcFrameDecoder; import org.apache.hadoop.oncrpc.security.CredentialsNone; import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.test.GenericTestUtils; -import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelException; -import org.jboss.netty.channel.ChannelHandlerContext; import org.junit.Test; import org.mockito.Mockito; import org.slf4j.event.Level; @@ -55,6 +55,7 @@ static void testRequest(XDR request, int serverPort) { tcpClient.run(); } + @ChannelHandler.Sharable static class TestRpcProgram extends RpcProgram { protected TestRpcProgram(String program, String host, int port, @@ -83,7 +84,7 @@ protected void handleInternal(ChannelHandlerContext ctx, RpcInfo info) { new VerifierNone()); XDR out = new XDR(); reply.write(out); - ChannelBuffer b = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap().buffer()); + ByteBuf b = Unpooled.wrappedBuffer(out.asReadOnlyWrap().buffer()); RpcResponse rsp = new RpcResponse(b, info.remoteAddress()); RpcUtil.sendRpcResponse(ctx, rsp); } @@ -99,13 +100,14 @@ public void testSingleFrame() { RpcFrameDecoder decoder = new RpcFrameDecoder(); // Test "Length field is not received yet" - ByteBuffer buffer = ByteBuffer.allocate(1); - ChannelBuffer buf = new ByteBufferBackedChannelBuffer(buffer); - ChannelBuffer channelBuffer = (ChannelBuffer) decoder.decode( - Mockito.mock(ChannelHandlerContext.class), Mockito.mock(Channel.class), - buf); - assertTrue(channelBuffer == null); + ByteBuf buf = Unpooled.directBuffer(1); + List outputBufs = new ArrayList<>(); + decoder.decode( + Mockito.mock(ChannelHandlerContext.class), buf, + outputBufs); + assertTrue(outputBufs.isEmpty()); + decoder = new RpcFrameDecoder(); // Test all bytes are not received yet byte[] fragment = new byte[4 + 9]; fragment[0] = (byte) (1 << 7); // final fragment @@ -114,15 +116,16 @@ public void testSingleFrame() { fragment[3] = (byte) 10; // fragment size = 10 bytes assertTrue(XDR.isLastFragment(fragment)); assertTrue(XDR.fragmentSize(fragment)==10); + buf.release(); - buffer = ByteBuffer.allocate(4 + 9); - buffer.put(fragment); - buffer.flip(); - buf = new ByteBufferBackedChannelBuffer(buffer); - channelBuffer = (ChannelBuffer) decoder.decode( - Mockito.mock(ChannelHandlerContext.class), Mockito.mock(Channel.class), - buf); - assertTrue(channelBuffer == null); + buf = Unpooled.directBuffer(4 + 9); + buf.writeBytes(fragment); + outputBufs = new ArrayList<>(); + decoder.decode( + Mockito.mock(ChannelHandlerContext.class), buf, + outputBufs); + assertTrue(decoder.isLast()); + buf.release(); } @Test @@ -137,16 +140,15 @@ public void testMultipleFrames() { fragment1[3] = (byte) 10; // fragment size = 10 bytes assertFalse(XDR.isLastFragment(fragment1)); assertTrue(XDR.fragmentSize(fragment1)==10); + + List outputBufs = new ArrayList<>(); // decoder should wait for the final fragment - ByteBuffer buffer = ByteBuffer.allocate(4 + 10); - buffer.put(fragment1); - buffer.flip(); - ChannelBuffer buf = new ByteBufferBackedChannelBuffer(buffer); - ChannelBuffer channelBuffer = (ChannelBuffer) decoder.decode( - Mockito.mock(ChannelHandlerContext.class), Mockito.mock(Channel.class), - buf); - assertTrue(channelBuffer == null); + ByteBuf buf = Unpooled.directBuffer(4 + 10, 4 + 10); + buf.writeBytes(fragment1); + decoder.decode( + Mockito.mock(ChannelHandlerContext.class), buf, + outputBufs); byte[] fragment2 = new byte[4 + 10]; fragment2[0] = (byte) (1 << 7); // final fragment @@ -155,21 +157,22 @@ public void testMultipleFrames() { fragment2[3] = (byte) 10; // fragment size = 10 bytes assertTrue(XDR.isLastFragment(fragment2)); assertTrue(XDR.fragmentSize(fragment2)==10); + buf.release(); - buffer = ByteBuffer.allocate(4 + 10); - buffer.put(fragment2); - buffer.flip(); - buf = new ByteBufferBackedChannelBuffer(buffer); - channelBuffer = (ChannelBuffer) decoder.decode( - Mockito.mock(ChannelHandlerContext.class), Mockito.mock(Channel.class), - buf); - assertTrue(channelBuffer != null); - // Complete frame should have to total size 10+10=20 - assertEquals(20, channelBuffer.readableBytes()); + buf = Unpooled.directBuffer(4 + 10, 4 + 10); + buf.writeBytes(fragment2); + decoder.decode( + Mockito.mock(ChannelHandlerContext.class), buf, + outputBufs); + // Expect two completed frames each 10 bytes + decoder.isLast(); + assertEquals(2, outputBufs.size()); + outputBufs.forEach(b -> assertEquals(((ByteBuf)b).readableBytes(), 10)); + buf.release(); } @Test - public void testFrames() { + public void testFrames() throws InterruptedException { int serverPort = startRpcServer(true); XDR xdrOut = createGetportMount(); @@ -187,7 +190,7 @@ public void testFrames() { } @Test - public void testUnprivilegedPort() { + public void testUnprivilegedPort() throws InterruptedException { // Don't allow connections from unprivileged ports. Given that this test is // presumably not being run by root, this will be the case. int serverPort = startRpcServer(false); @@ -218,23 +221,28 @@ public void testUnprivilegedPort() { assertEquals(requestSize, resultSize); } - private static int startRpcServer(boolean allowInsecurePorts) { + private static int startRpcServer(boolean allowInsecurePorts) + throws InterruptedException { Random rand = new Random(); int serverPort = 30000 + rand.nextInt(10000); int retries = 10; // A few retries in case initial choice is in use. while (true) { + SimpleTcpServer tcpServer = null; try { RpcProgram program = new TestFrameDecoder.TestRpcProgram("TestRpcProgram", "localhost", serverPort, 100000, 1, 2, allowInsecurePorts); - SimpleTcpServer tcpServer = new SimpleTcpServer(serverPort, program, 1); + tcpServer = new SimpleTcpServer(serverPort, program, 1); tcpServer.run(); break; // Successfully bound a port, break out. - } catch (ChannelException ce) { + } catch (InterruptedException | ChannelException e) { + if (tcpServer != null) { + tcpServer.shutdown(); + } if (retries-- > 0) { serverPort += rand.nextInt(20); // Port in use? Try another. } else { - throw ce; // Out of retries. + throw e; // Out of retries. } } } diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java index 6941c4a04e998..8ebf9d03c6c30 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java @@ -43,7 +43,7 @@ public class TestPortmap { private int xid; @BeforeClass - public static void setup() { + public static void setup() throws InterruptedException { pm.start(SHORT_TIMEOUT_MILLISECONDS, new InetSocketAddress("localhost", 0), new InetSocketAddress("localhost", 0)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 18a91cbcdc6f8..ab21583964b5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -47,7 +47,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty - netty + netty-all compile diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java index 3b0327ad4a149..2ba1bb060ce43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java @@ -26,6 +26,10 @@ import java.util.List; import java.util.HashMap; +import io.netty.channel.ChannelHandler; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileSystem; @@ -51,15 +55,13 @@ import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.ChannelHandlerContext; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; /** * RPC program corresponding to mountd daemon. See {@link Mountd}. */ +@ChannelHandler.Sharable public class RpcProgramMountd extends RpcProgram implements MountInterface { private static final Logger LOG = LoggerFactory.getLogger(RpcProgramMountd.class); @@ -262,8 +264,8 @@ RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()) RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write( out); } - ChannelBuffer buf = - ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap().buffer()); + ByteBuf buf = + Unpooled.wrappedBuffer(out.asReadOnlyWrap().buffer()); RpcResponse rsp = new RpcResponse(buf, info.remoteAddress()); RpcUtil.sendRpcResponse(ctx, rsp); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java index c6da1981f3716..c58dc5976b37d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java @@ -22,6 +22,8 @@ import java.net.URI; import java.nio.file.FileSystemException; +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsConstants; @@ -39,8 +41,6 @@ import org.apache.hadoop.nfs.nfs3.response.WccData; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.security.IdMappingServiceProvider; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.Channel; /** * Utility/helper methods related to NFS @@ -147,16 +147,16 @@ public static void writeChannel(Channel channel, XDR out, int xid) { if (RpcProgramNfs3.LOG.isDebugEnabled()) { RpcProgramNfs3.LOG.debug(WRITE_RPC_END + xid); } - ChannelBuffer outBuf = XDR.writeMessageTcp(out, true); - channel.write(outBuf); + ByteBuf outBuf = XDR.writeMessageTcp(out, true); + channel.writeAndFlush(outBuf); } public static void writeChannelCommit(Channel channel, XDR out, int xid) { if (RpcProgramNfs3.LOG.isDebugEnabled()) { RpcProgramNfs3.LOG.debug("Commit done:" + xid); } - ChannelBuffer outBuf = XDR.writeMessageTcp(out, true); - channel.write(outBuf); + ByteBuf outBuf = XDR.writeMessageTcp(out, true); + channel.writeAndFlush(outBuf); } private static boolean isSet(int access, int bits) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index 528ead7a003b3..8358c056cac4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -31,6 +31,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; +import io.netty.channel.Channel; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; @@ -55,7 +56,6 @@ import org.apache.hadoop.security.IdMappingServiceProvider; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; -import org.jboss.netty.channel.Channel; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index d436eac598be0..f6cb4350e4050 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -28,6 +28,11 @@ import java.nio.charset.Charset; import java.util.EnumSet; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException; @@ -129,10 +134,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.util.JvmPauseMonitor; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffers; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -141,6 +142,7 @@ /** * RPC program corresponding to nfs daemon. See {@link Nfs3}. */ +@ChannelHandler.Sharable public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { public static final int DEFAULT_UMASK = 0022; public static final FsPermission umask = new FsPermission( @@ -2180,7 +2182,7 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) { RpcDeniedReply.RejectState.AUTH_ERROR, new VerifierNone()); rdr.write(reply); - ChannelBuffer buf = ChannelBuffers.wrappedBuffer(reply.asReadOnlyWrap() + ByteBuf buf = Unpooled.wrappedBuffer(reply.asReadOnlyWrap() .buffer()); RpcResponse rsp = new RpcResponse(buf, info.remoteAddress()); RpcUtil.sendRpcResponse(ctx, rsp); @@ -2291,7 +2293,7 @@ RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write( } // TODO: currently we just return VerifierNone out = response.serialize(out, xid, new VerifierNone()); - ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap() + ByteBuf buf = Unpooled.wrappedBuffer(out.asReadOnlyWrap() .buffer()); RpcResponse rsp = new RpcResponse(buf, info.remoteAddress()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java index 76859247bf2a3..d5c9d4f55924a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java @@ -22,12 +22,12 @@ import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import io.netty.channel.Channel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow; -import org.jboss.netty.channel.Channel; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java index 288937104084b..a1b6e12eebfcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.EnumSet; +import io.netty.channel.Channel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -43,7 +44,6 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.security.IdMappingServiceProvider; -import org.jboss.netty.channel.Channel; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java index 4e53c72bec8a8..31528a2db87a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java @@ -21,6 +21,12 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.socket.SocketChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys; @@ -42,13 +48,6 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.CredentialsNone; import org.apache.hadoop.oncrpc.security.VerifierNone; -import org.jboss.netty.buffer.ChannelBuffer; -import org.jboss.netty.channel.Channel; -import org.jboss.netty.channel.ChannelHandlerContext; -import org.jboss.netty.channel.ChannelPipeline; -import org.jboss.netty.channel.ChannelPipelineFactory; -import org.jboss.netty.channel.Channels; -import org.jboss.netty.channel.MessageEvent; public class TestOutOfOrderWrite { public final static Logger LOG = @@ -100,9 +99,9 @@ public WriteHandler(XDR request) { } @Override - public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { + public void channelRead(ChannelHandlerContext ctx, Object msg) { // Get handle from create response - ChannelBuffer buf = (ChannelBuffer) e.getMessage(); + ByteBuf buf = (ByteBuf) msg; XDR rsp = new XDR(buf.array()); if (rsp.getBytes().length == 0) { LOG.info("rsp length is zero, why?"); @@ -125,7 +124,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) { rsp.readBoolean(); // value follow handle = new FileHandle(); handle.deserialize(rsp); - channel = e.getChannel(); + channel = ctx.channel(); } } @@ -136,16 +135,17 @@ public WriteClient(String host, int port, XDR request, Boolean oneShot) { } @Override - protected ChannelPipelineFactory setPipelineFactory() { - this.pipelineFactory = new ChannelPipelineFactory() { + protected ChannelInitializer setChannelHandler() { + return new ChannelInitializer() { @Override - public ChannelPipeline getPipeline() { - return Channels.pipeline( + protected void initChannel(SocketChannel ch) throws Exception { + ChannelPipeline p = ch.pipeline(); + p.addLast( RpcUtil.constructRpcFrameDecoder(), - new WriteHandler(request)); + new WriteHandler(request) + ); } }; - return this.pipelineFactory; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java index 30ecc0b824b9e..07954c00d64e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java @@ -28,6 +28,7 @@ import java.nio.ByteBuffer; import java.util.EnumSet; +import io.netty.channel.Channel; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -92,7 +93,6 @@ import org.apache.hadoop.security.IdMappingConstant; import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; -import org.jboss.netty.channel.Channel; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java index f7a92fac53501..0f03c6da93bf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.concurrent.ConcurrentNavigableMap; +import io.netty.channel.Channel; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -52,7 +53,6 @@ import org.apache.hadoop.security.ShellBasedIdMapping; import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; -import org.jboss.netty.channel.Channel; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; From b2e54762a480214bd9331514a85bebc05a4ecd0e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 28 Apr 2021 23:54:39 -0700 Subject: [PATCH 29/48] HDFS-15624. fix the function of setting quota by storage type (#2377) (#2955) 1. puts NVDIMM to the end of storage type enum to make sure compatibility. 2. adds check to make sure the software layout version is satisfied Co-authored-by: su xu Co-authored-by: huangtianhua Co-authored-by: YaYun-Wang <34060507+YaYun-Wang@users.noreply.github.com> Signed-off-by: Mingliang Liu Signed-off-by: Ayush Saxena Signed-off-by: Vinayakumar B Change-Id: I3c58beef50730827a09b3c968e9ad637baa57d44 --- .../java/org/apache/hadoop/fs/StorageType.java | 7 +++---- .../java/org/apache/hadoop/fs/shell/TestCount.java | 4 ++-- .../server/federation/router/TestRouterQuota.java | 14 +++++++------- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 7 +++++++ .../server/namenode/NameNodeLayoutVersion.java | 3 ++- .../apache/hadoop/hdfs/TestBlockStoragePolicy.java | 6 +++--- .../hadoop/hdfs/protocol/TestLayoutVersion.java | 3 ++- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java index b17864a22282a..2b5e4f1c5cee2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java @@ -34,13 +34,12 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public enum StorageType { - // sorted by the speed of the storage types, from fast to slow RAM_DISK(true, true), - NVDIMM(false, true), SSD(false, false), DISK(false, false), ARCHIVE(false, false), - PROVIDED(false, false); + PROVIDED(false, false), + NVDIMM(false, true); private final boolean isTransient; private final boolean isRAM; @@ -122,4 +121,4 @@ public static String getConf(Configuration conf, StorageType t, String name) { return conf.get(CONF_KEY_HEADER + t.toString() + "." + name); } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 618cb0190a99a..19516b8c40799 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -283,10 +283,10 @@ public void processPathWithQuotasByStorageTypesHeader() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> <----13-----> <------17-------> - " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA DISK_QUOTA REM_DISK_QUOTA " + // <----13---> <-------17------> "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA PROVIDED_QUOTA REM_PROVIDED_QUOTA " + + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + "PATHNAME"; verify(out).println(withStorageTypeHeader); verifyNoMoreInteractions(out); @@ -338,11 +338,11 @@ public void processPathWithQuotasByQTVH() throws Exception { count.processOptions(options); String withStorageTypeHeader = // <----13---> <-------17------> - " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + " SSD_QUOTA REM_SSD_QUOTA " + " DISK_QUOTA REM_DISK_QUOTA " + "ARCHIVE_QUOTA REM_ARCHIVE_QUOTA " + "PROVIDED_QUOTA REM_PROVIDED_QUOTA " + + " NVDIMM_QUOTA REM_NVDIMM_QUOTA " + "PATHNAME"; verify(out).println(withStorageTypeHeader); verifyNoMoreInteractions(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index 551ae8a8e0612..b69004198eb48 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -414,13 +414,13 @@ public void testStorageTypeQuota() throws Exception { QuotaUsage usage = client.getQuotaUsage("/type0"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, null, - usage); + verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1, -1}, + null, usage); // Verify /type1 quota on NN1. usage = client.getQuotaUsage("/type1"); assertEquals(HdfsConstants.QUOTA_RESET, usage.getQuota()); assertEquals(HdfsConstants.QUOTA_RESET, usage.getSpaceQuota()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, null, + verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1, -1}, null, usage); FileSystem routerFs = routerContext.getFileSystem(); @@ -431,15 +431,15 @@ public void testStorageTypeQuota() throws Exception { assertEquals(2, u1.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u1.getSpaceQuota()); assertEquals(fileSize * 3, u1.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota, -1, -1}, - new long[] {0, 0, 0, fileSize * 3, 0, 0}, u1); + verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota, -1, -1, -1}, + new long[] {0, 0, fileSize * 3, 0, 0, 0}, u1); // Verify /type0 storage type quota usage on Router. assertEquals(HdfsConstants.QUOTA_RESET, u0.getQuota()); assertEquals(4, u0.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_RESET, u0.getSpaceQuota()); assertEquals(fileSize * 3 * 2, u0.getSpaceConsumed()); - verifyTypeQuotaAndConsume(new long[] {-1, -1, -1, ssQuota * 2, -1, -1}, - new long[] {0, 0, 0, fileSize * 3 * 2, 0, 0}, u0); + verifyTypeQuotaAndConsume(new long[] {-1, -1, ssQuota * 2, -1, -1, -1}, + new long[] {0, 0, fileSize * 3 * 2, 0, 0, 0}, u0); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7b9efea538e18..095959b45bf67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2434,6 +2434,10 @@ private void checkStoragePolicyEnabled(final String operationNameReadable, * @throws IOException */ void setStoragePolicy(String src, String policyName) throws IOException { + if (policyName.equalsIgnoreCase( + HdfsConstants.ALLNVDIMM_STORAGE_POLICY_NAME)) { + requireEffectiveLayoutVersionForFeature(Feature.NVDIMM_SUPPORT); + } final String operationName = "setStoragePolicy"; checkOperation(OperationCategory.WRITE); checkStoragePolicyEnabled("set storage policy", true); @@ -3576,6 +3580,9 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type) if (type != null) { requireEffectiveLayoutVersionForFeature(Feature.QUOTA_BY_STORAGE_TYPE); } + if (type == StorageType.NVDIMM) { + requireEffectiveLayoutVersionForFeature(Feature.NVDIMM_SUPPORT); + } checkOperation(OperationCategory.WRITE); final String operationName = getQuotaCommand(nsQuota, ssQuota); final FSPermissionChecker pc = getPermissionChecker(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java index bcb3714268aa2..f5e261dc78aae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java @@ -90,7 +90,8 @@ public enum Feature implements LayoutFeature { QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types"), ERASURE_CODING(-64, -61, "Support erasure coding"), EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"), - SNAPSHOT_MODIFICATION_TIME(-66, -61, "Support modification time for snapshot"); + SNAPSHOT_MODIFICATION_TIME(-66, -61, "Support modification time for snapshot"), + NVDIMM_SUPPORT(-67, -61, "Support NVDIMM storage type"); private final FeatureInfo info; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index efb4b4df358f4..ef116957fb84d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -1421,29 +1421,29 @@ public void testStorageType() { final EnumMap map = new EnumMap<>(StorageType.class); //put storage type is reversed order + map.put(StorageType.NVDIMM, 1); map.put(StorageType.ARCHIVE, 1); map.put(StorageType.DISK, 1); map.put(StorageType.SSD, 1); map.put(StorageType.RAM_DISK, 1); - map.put(StorageType.NVDIMM, 1); { final Iterator i = map.keySet().iterator(); Assert.assertEquals(StorageType.RAM_DISK, i.next()); - Assert.assertEquals(StorageType.NVDIMM, i.next()); Assert.assertEquals(StorageType.SSD, i.next()); Assert.assertEquals(StorageType.DISK, i.next()); Assert.assertEquals(StorageType.ARCHIVE, i.next()); + Assert.assertEquals(StorageType.NVDIMM, i.next()); } { final Iterator> i = map.entrySet().iterator(); Assert.assertEquals(StorageType.RAM_DISK, i.next().getKey()); - Assert.assertEquals(StorageType.NVDIMM, i.next().getKey()); Assert.assertEquals(StorageType.SSD, i.next().getKey()); Assert.assertEquals(StorageType.DISK, i.next().getKey()); Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey()); + Assert.assertEquals(StorageType.NVDIMM, i.next().getKey()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java index 96629425f8d55..f074897001469 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java @@ -129,7 +129,8 @@ public void testNameNodeFeatureMinimumCompatibleLayoutVersions() { NameNodeLayoutVersion.Feature.QUOTA_BY_STORAGE_TYPE, NameNodeLayoutVersion.Feature.ERASURE_CODING, NameNodeLayoutVersion.Feature.EXPANDED_STRING_TABLE, - NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME); + NameNodeLayoutVersion.Feature.SNAPSHOT_MODIFICATION_TIME, + NameNodeLayoutVersion.Feature.NVDIMM_SUPPORT); for (LayoutFeature f : compatibleFeatures) { assertEquals(String.format("Expected minimum compatible layout version " + "%d for feature %s.", baseLV, f), baseLV, From 552e9dcc6c778df02e24d514750bf68151d0a77f Mon Sep 17 00:00:00 2001 From: lfengnan Date: Thu, 29 Apr 2021 06:14:18 -0700 Subject: [PATCH 30/48] HDFS-15561. RBF: Remove NPE when local namenode is not configured (#2954). Contributed by Fengnan Li. Reviewed-by: He Xiaoqiao --- .../hdfs/server/federation/router/Router.java | 17 ++++++++- .../router/TestRouterNamenodeHeartbeat.java | 38 +++++++++++++++++++ .../router/TestRouterWebHdfsMethods.java | 1 + 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java index d6e5a1cfe9626..ae9b62b7b4105 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java @@ -550,14 +550,20 @@ public void verifyToken(DelegationTokenIdentifier tokenId, byte[] password) * * @return Updater of the status for the local Namenode. */ - protected NamenodeHeartbeatService createLocalNamenodeHeartbeatService() { + @VisibleForTesting + public NamenodeHeartbeatService createLocalNamenodeHeartbeatService() { // Detect NN running in this machine String nsId = DFSUtil.getNamenodeNameServiceId(conf); + if (nsId == null) { + LOG.error("Cannot find local nameservice id"); + return null; + } String nnId = null; if (HAUtil.isHAEnabled(conf, nsId)) { nnId = HAUtil.getNameNodeId(conf, nsId); if (nnId == null) { LOG.error("Cannot find namenode id for local {}", nsId); + return null; } } @@ -787,4 +793,13 @@ public RouterAdminServer getAdminServer() { return adminServer; } + /** + * Set router configuration. + * @param conf + */ + @VisibleForTesting + public void setConf(Configuration conf) { + this.conf = conf; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java index d2bc5d6eb2fcb..38419ed84926f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java @@ -17,15 +17,21 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.federation.MockResolver; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; @@ -105,6 +111,38 @@ public void testNamenodeHeartbeatService() throws IOException { server.close(); } + @Test + public void testLocalNamenodeHeartbeatService() throws IOException { + Router router = new Router(); + Configuration conf = new Configuration(); + assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf)); + + // case 1: no local nn is configured + router.setConf(conf); + assertNull(router.createLocalNamenodeHeartbeatService()); + + // case 2: local nn is configured + conf.set(DFS_NAMESERVICES, "ns1"); + assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf)); + conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), + "nn1,nn2"); + conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), + "localhost:8020"); + conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), + "ns1-nn2.example.com:8020"); + router.setConf(conf); + NamenodeHeartbeatService heartbeatService = + router.createLocalNamenodeHeartbeatService(); + assertNotNull(heartbeatService); + // we have to start the service to get the serviceAddress assigned + heartbeatService.init(conf); + assertEquals("ns1-nn1:localhost:8020", + heartbeatService.getNamenodeDesc()); + heartbeatService.stop(); + } + @Test public void testHearbeat() throws InterruptedException, IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java index 7028928041452..8e82d44c4ddfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java @@ -61,6 +61,7 @@ public static void globalSetUp() throws Exception { .admin() .build(); cluster.addRouterOverrides(conf); + cluster.setIndependentDNs(); cluster.startCluster(); cluster.startRouters(); cluster.waitClusterUp(); From 803ac4b1a0fc5c0b6c25a5df0733b3ebcdb2f294 Mon Sep 17 00:00:00 2001 From: Eric Badger Date: Thu, 29 Apr 2021 17:42:13 +0000 Subject: [PATCH 31/48] YARN-10707. Support custom resources in ResourceUtilization, and update Node GPU Utilization to use. Contributed by Qi Zhu --- .../yarn/api/records/ResourceUtilization.java | 128 +++++++++++++++++- .../src/main/proto/yarn_protos.proto | 6 + .../yarn/api/records/impl/pb/ProtoUtils.java | 29 ++++ .../impl/pb/ResourceUtilizationPBImpl.java | 31 ++++- .../yarn/api/BasePBImplRecordsTest.java | 2 + .../api/records/TestResourceUtilization.java | 49 +++++++ .../nodemanager/NodeResourceMonitorImpl.java | 30 ++-- .../gpu/GpuNodeResourceUpdateHandler.java | 43 +++++- .../gpu/TestGpuResourcePlugin.java | 4 +- 9 files changed, 297 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java index f6c5a69eade00..ff3cec32b6522 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -22,6 +22,9 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; +import java.util.HashMap; +import java.util.Map; + /** *

* ResourceUtilization models the utilization of a set of computer @@ -33,14 +36,26 @@ public abstract class ResourceUtilization implements Comparable { + private Map customResources + = new HashMap<>(); + @Public @Unstable - public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) { + public static ResourceUtilization newInstance(int pmem, int vmem, + float cpu) { + return newInstance(pmem, vmem, cpu, null); + } + + @Public + @Unstable + public static ResourceUtilization newInstance(int pmem, int vmem, + float cpu, Map customResources) { ResourceUtilization utilization = Records.newRecord(ResourceUtilization.class); utilization.setPhysicalMemory(pmem); utilization.setVirtualMemory(vmem); utilization.setCPU(cpu); + utilization.setCustomResources(customResources); return utilization; } @@ -49,7 +64,9 @@ public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) { public static ResourceUtilization newInstance( ResourceUtilization resourceUtil) { return newInstance(resourceUtil.getPhysicalMemory(), - resourceUtil.getVirtualMemory(), resourceUtil.getCPU()); + resourceUtil.getVirtualMemory(), + resourceUtil.getCPU(), + resourceUtil.getCustomResources()); } /** @@ -106,6 +123,51 @@ public static ResourceUtilization newInstance( @Unstable public abstract void setCPU(float cpu); + /** + * Get custom resource utilization + * (The amount of custom resource used). + * + * @param resourceName resourceName of custom resource + * @return resourceName utilization + */ + @Public + @Unstable + public float getCustomResource(String resourceName) { + if (customResources != null && resourceName != null) { + return customResources.get(resourceName); + } + return 0f; + } + + @Public + @Unstable + public Map getCustomResources() { + return customResources; + } + + @Public + @Unstable + public void setCustomResources(Map customResources) { + if (customResources != null) { + this.customResources = customResources; + } + } + + /** + * Set custom resource utilization + * (The amount of custom resource used). + * @param resourceName resourceName + * @param utilization utilization of custom resource + * + */ + @Public + @Unstable + public void setCustomResource(String resourceName, float utilization) { + if (resourceName != null && !resourceName.isEmpty()) { + customResources.put(resourceName, utilization); + } + } + @Override public int hashCode() { final int prime = 263167; @@ -113,6 +175,12 @@ public int hashCode() { result = prime * result + getVirtualMemory(); result = prime * result + getPhysicalMemory(); result = 31 * result + Float.valueOf(getCPU()).hashCode(); + if (customResources != null && !customResources.isEmpty()) { + for (Map.Entry entry : customResources.entrySet()) { + result = 31 * result + + customResources.get(entry.getKey()).hashCode(); + } + } return result; } @@ -130,7 +198,8 @@ public boolean equals(Object obj) { ResourceUtilization other = (ResourceUtilization) obj; if (getVirtualMemory() != other.getVirtualMemory() || getPhysicalMemory() != other.getPhysicalMemory() - || getCPU() != other.getCPU()) { + || getCPU() != other.getCPU() + || !customResources.equals(other.customResources)) { return false; } return true; @@ -138,8 +207,19 @@ public boolean equals(Object obj) { @Override public String toString() { - return ""; + StringBuilder utilizationString = new StringBuilder(); + utilizationString.append( + " entry : getCustomResources().entrySet()) { + utilizationString.append(", " + + entry.getKey() + ":" + entry.getValue()); + } + } + + utilizationString.append(">"); + return utilizationString.toString(); } /** @@ -151,9 +231,28 @@ public String toString() { @Public @Unstable public void addTo(int pmem, int vmem, float cpu) { + addTo(pmem, vmem, cpu, null, 0f); + } + + /** + * Add utilization to the current one. + * @param pmem Physical memory used to add. + * @param vmem Virtual memory used to add. + * @param cpu CPU utilization to add. + * @param resourceName of custom resource to add. + * @param utilization of custom resource to add. + */ + @Public + @Unstable + public void addTo(int pmem, int vmem, float cpu, + String resourceName, float utilization) { this.setPhysicalMemory(this.getPhysicalMemory() + pmem); this.setVirtualMemory(this.getVirtualMemory() + vmem); this.setCPU(this.getCPU() + cpu); + if (resourceName != null) { + this.setCustomResource(resourceName, + getCustomResource(resourceName) + utilization); + } } /** @@ -165,8 +264,27 @@ public void addTo(int pmem, int vmem, float cpu) { @Public @Unstable public void subtractFrom(int pmem, int vmem, float cpu) { + subtractFrom(pmem, vmem, cpu, null, 0f); + } + + /** + * Subtract utilization from the current one. + * @param pmem Physical memory to be subtracted. + * @param vmem Virtual memory to be subtracted. + * @param cpu CPU utilization to be subtracted. + * @param resourceName of custom resource to be subtracted. + * @param utilization of custom resource to be subtracted. + */ + @Public + @Unstable + public void subtractFrom(int pmem, int vmem, float cpu, + String resourceName, float utilization) { this.setPhysicalMemory(this.getPhysicalMemory() - pmem); this.setVirtualMemory(this.getVirtualMemory() - vmem); this.setCPU(this.getCPU() - cpu); + if (resourceName != null) { + this.setCustomResource(resourceName, + getCustomResource(resourceName) - utilization); + } } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 0b360dfca5941..558e724f8ac6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -83,6 +83,7 @@ message ResourceUtilizationProto { optional int32 pmem = 1; optional int32 vmem = 2; optional float cpu = 3; + repeated StringFloatMapProto customResources = 4; } message ResourceOptionProto { @@ -243,6 +244,11 @@ message StringLongMapProto { required int64 value = 2; } +message StringFloatMapProto { + required string key = 1; + required float value = 2; +} + message ApplicationResourceUsageReportProto { optional int32 num_used_containers = 1; optional int32 num_reserved_containers = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index cdeb417243e37..64bf8cf5d3876 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -587,6 +587,22 @@ public static List convertMapToStringLongMapProto return ret; } + public static List + convertMapToStringFloatMapProtoList( + Map map) { + List ret = new ArrayList<>(); + if (map != null) { + for (Map.Entry entry : map.entrySet()) { + YarnProtos.StringFloatMapProto.Builder tmp = + YarnProtos.StringFloatMapProto.newBuilder(); + tmp.setKey(entry.getKey()); + tmp.setValue(entry.getValue()); + ret.add(tmp.build()); + } + } + return ret; + } + public static Map convertStringStringMapProtoListToMap( List pList) { Map ret = new HashMap<>(); @@ -600,6 +616,19 @@ public static Map convertStringStringMapProtoListToMap( return ret; } + public static Map convertStringFloatMapProtoListToMap( + List pList) { + Map ret = new HashMap<>(); + if (pList != null) { + for (YarnProtos.StringFloatMapProto p : pList) { + if (p.hasKey()) { + ret.put(p.getKey(), p.getValue()); + } + } + } + return ret; + } + public static List convertToProtoFormat( Map stringMap) { List pList = new ArrayList<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java index e37adbe63f97e..023d1e9ff68c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java @@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProtoOrBuilder; import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import java.util.Map; + @Private @Unstable public class ResourceUtilizationPBImpl extends ResourceUtilization { @@ -69,7 +71,7 @@ public void setPhysicalMemory(int pmem) { @Override public int getVirtualMemory() { ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; - return (p.getVmem()); + return p.getVmem(); } @Override @@ -90,6 +92,28 @@ public void setCPU(float cpu) { builder.setCpu(cpu); } + @Override + public float getCustomResource(String resourceName) { + return getCustomResources().get(resourceName); + } + + @Override + public Map getCustomResources() { + ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder; + return ProtoUtils. + convertStringFloatMapProtoListToMap(p. + getCustomResourcesList()); + } + + @Override + public void setCustomResources(Map customResources) { + if (customResources != null) { + maybeInitBuilder(); + builder.addAllCustomResources(ProtoUtils. + convertMapToStringFloatMapProtoList(customResources)); + } + } + @Override public int compareTo(ResourceUtilization other) { int diff = this.getPhysicalMemory() - other.getPhysicalMemory(); @@ -97,6 +121,11 @@ public int compareTo(ResourceUtilization other) { diff = this.getVirtualMemory() - other.getVirtualMemory(); if (diff == 0) { diff = Float.compare(this.getCPU(), other.getCPU()); + if (diff == 0) { + diff = this.getCustomResources().size() - + other.getCustomResources().size(); + // todo how to compare custom resource in same size + } } } return diff; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index affa08f082437..22b687c159e15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -76,6 +76,8 @@ private static Object genTypeValue(Type type) { 'a' + rand.nextInt(26), 'a' + rand.nextInt(26), 'a' + rand.nextInt(26)); + } else if (type.equals(Float.class)) { + return rand.nextFloat(); } else if (type instanceof Class) { Class clazz = (Class)type; if (clazz.isArray()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java index 5934846e2f36f..a2b05708326b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java @@ -21,6 +21,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + public class TestResourceUtilization { @Test @@ -60,4 +63,50 @@ public void testResourceUtilization() { u1.subtractFrom(10, 0, 0.0f); Assert.assertEquals(u1, u3); } + + @Test + public void testResourceUtilizationWithCustomResource() { + Map customResources = new HashMap<>(); + customResources.put(ResourceInformation.GPU_URI, 5.0f); + ResourceUtilization u1 = ResourceUtilization. + newInstance(10, 20, 0.5f, customResources); + ResourceUtilization u2 = ResourceUtilization.newInstance(u1); + ResourceUtilization u3 = ResourceUtilization. + newInstance(10, 20, 0.5f, customResources); + ResourceUtilization u4 = ResourceUtilization. + newInstance(20, 20, 0.5f, customResources); + ResourceUtilization u5 = ResourceUtilization. + newInstance(30, 40, 0.8f, customResources); + + Assert.assertEquals(u1, u2); + Assert.assertEquals(u1, u3); + Assert.assertNotEquals(u1, u4); + Assert.assertNotEquals(u2, u5); + Assert.assertNotEquals(u4, u5); + + Assert.assertTrue(u1.hashCode() == u2.hashCode()); + Assert.assertTrue(u1.hashCode() == u3.hashCode()); + Assert.assertFalse(u1.hashCode() == u4.hashCode()); + Assert.assertFalse(u2.hashCode() == u5.hashCode()); + Assert.assertFalse(u4.hashCode() == u5.hashCode()); + + Assert.assertTrue(u1.getPhysicalMemory() == 10); + Assert.assertFalse(u1.getVirtualMemory() == 10); + Assert.assertTrue(u1.getCPU() == 0.5f); + Assert.assertTrue(u1. + getCustomResource(ResourceInformation.GPU_URI) == 5.0f); + + Assert.assertEquals("", u1.toString()); + + u1.addTo(10, 0, 0.0f); + Assert.assertNotEquals(u1, u2); + Assert.assertEquals(u1, u4); + u1.addTo(10, 20, 0.3f); + Assert.assertEquals(u1, u5); + u1.subtractFrom(10, 20, 0.3f); + Assert.assertEquals(u1, u4); + u1.subtractFrom(10, 0, 0.0f); + Assert.assertEquals(u1, u3); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java index 7577b55899781..37fa33e14fcce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; + /** * Implementation of the node resource monitor. It periodically tracks the * resource utilization of the node and reports it to the NM. @@ -54,8 +57,11 @@ public class NodeResourceMonitorImpl extends AbstractService implements private GpuNodeResourceUpdateHandler gpuNodeResourceUpdateHandler; /** Current resource utilization of the node. */ + + private Map customResources = new HashMap<>(); + private ResourceUtilization nodeUtilization = - ResourceUtilization.newInstance(0, 0, 0f); + ResourceUtilization.newInstance(0, 0, 0f, customResources); private Context nmContext; /** @@ -165,22 +171,26 @@ public void run() { resourceCalculatorPlugin.getVirtualMemorySize() - resourceCalculatorPlugin.getAvailableVirtualMemorySize(); float vcores = resourceCalculatorPlugin.getNumVCoresUsed(); - nodeUtilization = - ResourceUtilization.newInstance( - (int) (pmem >> 20), // B -> MB - (int) (vmem >> 20), // B -> MB - vcores); // Used Virtual Cores - float nodeGpuUtilization = 0F; + float totalNodeGpuUtilization = 0F; try { if (gpuNodeResourceUpdateHandler != null) { - nodeGpuUtilization = - gpuNodeResourceUpdateHandler.getNodeGpuUtilization(); + totalNodeGpuUtilization = + gpuNodeResourceUpdateHandler.getTotalNodeGpuUtilization(); } } catch (Exception e) { LOG.error("Get Node GPU Utilization error: " + e); } + customResources. + put(ResourceInformation.GPU_URI, totalNodeGpuUtilization); + nodeUtilization = + ResourceUtilization.newInstance( + (int) (pmem >> 20), // B -> MB + (int) (vmem >> 20), // B -> MB + vcores, // Used Virtual Cores + customResources); // Used GPUs + // Publish the node utilization metrics to node manager // metrics system. NodeManagerMetrics nmMetrics = nmContext.getNodeManagerMetrics(); @@ -188,7 +198,7 @@ public void run() { nmMetrics.setNodeUsedMemGB(nodeUtilization.getPhysicalMemory()); nmMetrics.setNodeUsedVMemGB(nodeUtilization.getVirtualMemory()); nmMetrics.setNodeCpuUtilization(nodeUtilization.getCPU()); - nmMetrics.setNodeGpuUtilization(nodeGpuUtilization); + nmMetrics.setNodeGpuUtilization(totalNodeGpuUtilization); } try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java index af81709566d18..c31555ed962a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java @@ -79,19 +79,48 @@ public void updateConfiguredResource(Resource res) throws YarnException { res.setResourceValue(GPU_URI, nUsableGpus); } - public float getNodeGpuUtilization() throws Exception{ + /** + * + * @return The average physical GPUs used in this node. + * + * For example: + * Node with total 4 GPUs + * Physical used 2.4 GPUs + * Will return 2.4/4 = 0.6f + * + * @throws Exception when any error happens + */ + public float getAvgNodeGpuUtilization() throws Exception{ List gpuList = gpuDiscoverer.getGpuDeviceInformation().getGpus(); - Float totalGpuUtilization = 0F; + Float avgGpuUtilization = 0F; if (gpuList != null && gpuList.size() != 0) { - totalGpuUtilization = gpuList - .stream() - .map(g -> g.getGpuUtilizations().getOverallGpuUtilization()) - .collect(Collectors.summingDouble(Float::floatValue)) - .floatValue() / gpuList.size(); + avgGpuUtilization = getTotalNodeGpuUtilization() / gpuList.size(); } + return avgGpuUtilization; + } + + /** + * + * @return The total physical GPUs used in this node. + * + * For example: + * Node with total 4 GPUs + * Physical used 2.4 GPUs + * Will return 2.4f + * + * @throws Exception when any error happens + */ + public float getTotalNodeGpuUtilization() throws Exception{ + List gpuList = + gpuDiscoverer.getGpuDeviceInformation().getGpus(); + Float totalGpuUtilization = gpuList + .stream() + .map(g -> g.getGpuUtilizations().getOverallGpuUtilization()) + .collect(Collectors.summingDouble(Float::floatValue)) + .floatValue(); return totalGpuUtilization; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuResourcePlugin.java index 749e0cc14de91..da1a57ec005be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuResourcePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuResourcePlugin.java @@ -126,7 +126,7 @@ public void testGetNMResourceInfoAutoDiscoveryDisabled() } @Test - public void testNodeGPUUtilization() + public void testAvgNodeGpuUtilization() throws Exception { GpuDiscoverer gpuDiscoverer = createNodeGPUUtilizationDiscoverer(); @@ -134,7 +134,7 @@ public void testNodeGPUUtilization() new GpuNodeResourceUpdateHandler(gpuDiscoverer, new Configuration()); Assert.assertEquals(0.5F, - gpuNodeResourceUpdateHandler.getNodeGpuUtilization(), 1e-6); + gpuNodeResourceUpdateHandler.getAvgNodeGpuUtilization(), 1e-6); } private GpuDiscoverer createNodeGPUUtilizationDiscoverer() From bd3da73a0ff75231340b1168f7805164710bf4fe Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Sat, 1 May 2021 11:05:31 -0700 Subject: [PATCH 32/48] HDFS-15865. Interrupt DataStreamer thread if no ack (#2728) --- .../java/org/apache/hadoop/hdfs/DataStreamer.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 96c86c3569b6b..e04268eddce79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -897,6 +897,8 @@ void waitForAckedSeqno(long seqno) throws IOException { try (TraceScope ignored = dfsClient.getTracer(). newScope("waitForAckedSeqno")) { LOG.debug("{} waiting for ack for: {}", this, seqno); + int dnodes = nodes != null ? nodes.length : 3; + int writeTimeout = dfsClient.getDatanodeWriteTimeout(dnodes); long begin = Time.monotonicNow(); try { synchronized (dataQueue) { @@ -907,6 +909,16 @@ void waitForAckedSeqno(long seqno) throws IOException { } try { dataQueue.wait(1000); // when we receive an ack, we notify on + long duration = Time.monotonicNow() - begin; + if (duration > writeTimeout) { + LOG.error("No ack received, took {}ms (threshold={}ms). " + + "File being written: {}, block: {}, " + + "Write pipeline datanodes: {}.", + duration, writeTimeout, src, block, nodes); + throw new InterruptedIOException("No ack received after " + + duration / 1000 + "s and a timeout of " + + writeTimeout / 1000 + "s"); + } // dataQueue } catch (InterruptedException ie) { throw new InterruptedIOException( From f1e180902956fd547ad6a7b440588fdf583870c2 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Sat, 1 May 2021 19:10:32 -0700 Subject: [PATCH 33/48] HADOOP-17653. Do not use guava's Files.createTempDir(). (#2945) Reviewed-by: Steve Loughran Signed-off-by: Akira Ajisaka --- .../store/driver/impl/StateStoreFileImpl.java | 13 ++++++++++++- .../impl/TestInMemoryLevelDBAliasMapClient.java | 9 ++++++--- .../impl/TestLevelDbMockAliasMapClient.java | 7 +++++-- .../hadoop/yarn/service/client/ServiceClient.java | 7 +++++-- .../hadoop/yarn/csi/client/TestCsiClient.java | 7 +++++-- .../resourceplugin/com/nec/TestNECVEPlugin.java | 2 +- .../fpga/TestIntelFpgaOpenclPlugin.java | 2 +- .../TestDominantResourceFairnessPolicy.java | 2 +- 8 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java index cedc784e39f15..9d2b1ab2fb73a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java @@ -88,7 +88,18 @@ protected String getRootDir() { if (this.rootDirectory == null) { String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY); if (dir == null) { - File tempDir = Files.createTempDir(); + File tempDirBase = + new File(System.getProperty("java.io.tmpdir")); + File tempDir = null; + try { + tempDir = java.nio.file.Files.createTempDirectory( + tempDirBase.toPath(), System.currentTimeMillis() + "-").toFile(); + } catch (IOException e) { + // fallback to the base upon exception. + LOG.debug("Unable to create a temporary directory. Fall back to " + + " the default system temp directory {}", tempDirBase, e); + tempDir = tempDirBase; + } dir = tempDir.getAbsolutePath(); LOG.warn("The root directory is not available, using {}", dir); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java index 8971730707bfb..1996f4455af5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -16,8 +16,6 @@ */ package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; -import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -28,7 +26,9 @@ import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -44,6 +44,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.List; import java.util.Optional; import java.util.Random; @@ -74,7 +75,9 @@ public void setUp() throws IOException { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); - tempDir = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + tempDir = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); File levelDBDir = new File(tempDir, BPID); levelDBDir.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java index 91e07cfa4c8e4..84b0d23fda5b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -26,12 +25,14 @@ import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.apache.hadoop.test.GenericTestUtils; import org.iq80.leveldb.DBException; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.mockito.Mockito.doThrow; @@ -60,7 +61,9 @@ public void setUp() throws IOException { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); - tempDir = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + tempDir = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDir.getAbsolutePath()); levelDBAliasMapServer.setConf(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 901b81fe14946..8d350158fffdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -27,7 +27,6 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryNTimes; -import org.apache.curator.shaded.com.google.common.io.Files; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -117,6 +116,7 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.text.MessageFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -1140,7 +1140,10 @@ private void addYarnSysFs(Path path, return; } String buffer = ServiceApiUtil.jsonSerDeser.toJson(app); - File tmpDir = Files.createTempDir(); + File testDir = + new File(System.getProperty("java.io.tmpdir")); + File tmpDir = Files.createTempDirectory( + testDir.toPath(), System.currentTimeMillis() + "-").toFile(); if (tmpDir.exists()) { String serviceJsonPath = tmpDir.getAbsolutePath() + "/app.json"; File localFile = new File(serviceJsonPath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java index f025bc6d72fda..b60d5cdb86080 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java @@ -20,7 +20,7 @@ import csi.v0.Csi; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Assume; @@ -30,6 +30,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; /** * Test class for CSI client. @@ -42,7 +43,9 @@ public class TestCsiClient { @BeforeClass public static void setUp() throws IOException { - testRoot = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + testRoot = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); File socketPath = new File(testRoot, "csi.sock"); FileUtils.forceMkdirParent(socketPath); domainSocket = "unix://" + socketPath.getAbsolutePath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index 06d1d2de6f909..58fc4c7dc1c47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -46,7 +46,7 @@ import org.apache.commons.compress.utils.Sets; import org.apache.commons.io.FileUtils; -import org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.util.Shell.CommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java index 663df5e26cbcd..0b093953eef34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; -import org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java index c963e0dab63d2..197651a64c16c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java @@ -29,7 +29,7 @@ import java.util.Map; import java.util.TreeSet; -import org.apache.curator.shaded.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; From 6e525ab81cc2d93eb1e4aeea6b13769635076c06 Mon Sep 17 00:00:00 2001 From: lfengnan Date: Sun, 2 May 2021 03:18:47 -0700 Subject: [PATCH 34/48] HDFS-15810. RBF: RBFMetrics's TotalCapacity out of bounds (#2910) Reviewed-by: Inigo Goiri Signed-off-by: Akira Ajisaka --- .../src/site/markdown/Metrics.md | 9 +++-- .../federation/metrics/FederationMBean.java | 26 ++++++++++++++ .../server/federation/metrics/RBFMetrics.java | 36 +++++++++++++++++-- .../main/webapps/router/federationhealth.html | 6 ++-- .../federation/metrics/TestMetricsBase.java | 11 ++++++ .../federation/metrics/TestRBFMetrics.java | 36 ++++++++++++++++++- 6 files changed, 115 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 8423b5351b11f..6cec030ee2557 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -527,9 +527,12 @@ RBFMetrics shows the metrics which are the aggregated values of sub-clusters' in | `NumInMaintenanceLiveDataNodes` | Number of live Datanodes which are in maintenance state | | `NumInMaintenanceDeadDataNodes` | Number of dead Datanodes which are in maintenance state | | `NumEnteringMaintenanceDataNodes` | Number of Datanodes that are entering the maintenance state | -| `TotalCapacity` | Current raw capacity of DataNodes in bytes | -| `UsedCapacity` | Current used capacity across all DataNodes in bytes | -| `RemainingCapacity` | Current remaining capacity in bytes | +| `TotalCapacity` | Current raw capacity of DataNodes in bytes (long primitive, may overflow) | +| `UsedCapacity` | Current used capacity across all DataNodes in bytes (long primitive, may overflow) | +| `RemainingCapacity` | Current remaining capacity in bytes (long primitive, may overflow) | +| `TotalCapacityBigInt` | Current raw capacity of DataNodes in bytes (using BigInteger) | +| `UsedCapacityBigInt` | Current used capacity across all DataNodes in bytes (using BigInteger) | +| `RemainingCapacityBigInt` | Current remaining capacity in bytes (using BigInteger) | | `NumOfMissingBlocks` | Current number of missing blocks | | `NumLiveNodes` | Number of datanodes which are currently live | | `NumDeadNodes` | Number of datanodes which are currently dead | diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java index c06a2e08548ae..b9ea8709e90f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.federation.metrics; +import java.math.BigInteger; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -54,22 +56,46 @@ public interface FederationMBean { /** * Get the total capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getTotalCapacityBigInt()} instead. * @return Total capacity of the federated cluster. */ long getTotalCapacity(); /** * Get the used capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getUsedCapacityBigInt()} instead. * @return Used capacity of the federated cluster. */ long getUsedCapacity(); /** * Get the remaining capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getRemainingCapacityBigInt()} instead. * @return Remaining capacity of the federated cluster. */ long getRemainingCapacity(); + /** + * Get the total capacity (big integer) of the federated cluster. + * @return Total capacity of the federated cluster. + */ + BigInteger getTotalCapacityBigInt(); + + /** + * Get the used capacity (big integer) of the federated cluster. + * @return Used capacity of the federated cluster. + */ + BigInteger getUsedCapacityBigInt(); + + /** + * Get the remaining capacity (big integer) of the federated cluster. + * @return Remaining capacity of the federated cluster. + */ + BigInteger getRemainingCapacityBigInt(); + /** * Get the total remote storage capacity mounted in the federated cluster. * @return Remote capacity of the federated cluster. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index cc5bf07408246..ec41a32716607 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.reflect.Method; +import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -380,14 +381,29 @@ public long getRemainingCapacity() { return getNameserviceAggregatedLong(MembershipStats::getAvailableSpace); } + @Override + public long getUsedCapacity() { + return getTotalCapacity() - getRemainingCapacity(); + } + + @Override + public BigInteger getTotalCapacityBigInt() { + return getNameserviceAggregatedBigInt(MembershipStats::getTotalSpace); + } + + @Override + public BigInteger getRemainingCapacityBigInt() { + return getNameserviceAggregatedBigInt(MembershipStats::getAvailableSpace); + } + @Override public long getProvidedSpace() { return getNameserviceAggregatedLong(MembershipStats::getProvidedSpace); } @Override - public long getUsedCapacity() { - return getTotalCapacity() - getRemainingCapacity(); + public BigInteger getUsedCapacityBigInt() { + return getTotalCapacityBigInt().subtract(getRemainingCapacityBigInt()); } @Override @@ -783,6 +799,22 @@ private long getNameserviceAggregatedLong(ToLongFunction f) { } } + private BigInteger getNameserviceAggregatedBigInt( + ToLongFunction f) { + try { + List states = getActiveNamenodeRegistrations(); + BigInteger sum = BigInteger.valueOf(0); + for (MembershipState state : states) { + long lvalue = f.applyAsLong(state.getStats()); + sum = sum.add(BigInteger.valueOf(lvalue)); + } + return sum; + } catch (IOException e) { + LOG.error("Unable to extract metrics: {}", e.getMessage()); + return new BigInteger("0"); + } + } + /** * Fetches the most active namenode memberships for all known nameservices. * The fetched membership may not or may not be active. Excludes expired diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index 87d0e718bdcd3..eca395ff4b2a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -113,9 +113,9 @@

Non Heap Memory used {used|fmt_bytes} of {committed|fmt_bytes} Commited Non Heap Memory. Max Non Heap Memory is {@eq key=max value="-1" type="number"}<unbounded>{:else}{max|fmt_bytes}{/eq}.

{/mem.NonHeapMemoryUsage} - - - + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java index 4759d05f820dc..b01e22006d776 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java @@ -259,4 +259,15 @@ private MembershipState createRegistration(String ns, String nn, assertTrue(response.getResult()); return record; } + + // refresh namenode registration for new attributes + public boolean refreshNamenodeRegistration(NamenodeHeartbeatRequest request) + throws IOException { + boolean result = membershipStore.namenodeHeartbeat(request).getResult(); + membershipStore.loadCache(true); + MembershipNamenodeResolver resolver = + (MembershipNamenodeResolver) router.getNamenodeResolver(); + resolver.loadCache(true); + return result; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 2c7edaa68e195..25473f8df9233 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -19,11 +19,13 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getBean; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; import java.io.IOException; +import java.math.BigInteger; import java.util.Iterator; import java.util.List; @@ -31,6 +33,7 @@ import org.apache.commons.collections.ListUtils; import org.apache.hadoop.hdfs.server.federation.router.Router; +import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; @@ -58,6 +61,7 @@ public void testClusterStatsJMX() FederationMBean federationBean = getBean(FEDERATION_BEAN, FederationMBean.class); validateClusterStatsFederationBean(federationBean); + testCapacity(federationBean); RouterMBean routerBean = getBean(ROUTER_BEAN, RouterMBean.class); validateClusterStatsRouterBean(routerBean); } @@ -348,4 +352,34 @@ private void validateClusterStatsRouterBean(RouterMBean bean) { assertTrue(bean.getHostAndPort().length() > 0); assertFalse(bean.isSecurityEnabled()); } + + private void testCapacity(FederationMBean bean) throws IOException { + List memberships = getActiveMemberships(); + assertTrue(memberships.size() > 1); + + BigInteger availableCapacity = BigInteger.valueOf(0); + BigInteger totalCapacity = BigInteger.valueOf(0); + BigInteger unitCapacity = BigInteger.valueOf(Long.MAX_VALUE); + for (MembershipState mock : memberships) { + MembershipStats stats = mock.getStats(); + stats.setTotalSpace(Long.MAX_VALUE); + stats.setAvailableSpace(Long.MAX_VALUE); + // reset stats to make the new value persistent + mock.setStats(stats); + // write back the new namenode information to state store + assertTrue(refreshNamenodeRegistration( + NamenodeHeartbeatRequest.newInstance(mock))); + totalCapacity = totalCapacity.add(unitCapacity); + availableCapacity = availableCapacity.add(unitCapacity); + } + + // for local cache update + assertEquals(totalCapacity, bean.getTotalCapacityBigInt()); + // not equal since overflow happened. + assertNotEquals(totalCapacity, BigInteger.valueOf(bean.getTotalCapacity())); + assertEquals(availableCapacity, bean.getRemainingCapacityBigInt()); + // not equal since overflow happened. + assertNotEquals(availableCapacity, + BigInteger.valueOf(bean.getRemainingCapacity())); + } } From 7c561e2a02526280a2fe7b5a873e80b78cf4900c Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 4 May 2021 00:25:29 +0900 Subject: [PATCH 35/48] HDFS-15952. TestRouterRpcMultiDestination#testProxyGetTransactionID and testProxyVersionRequest are flaky. (#2970) --- .../federation/router/TestRouterRpc.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index c3e7f10c95a00..4f772cc4dc89c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -185,6 +186,7 @@ public int compare( private NamenodeProtocol routerNamenodeProtocol; /** NameNodeProtocol interface to the Namenode. */ private NamenodeProtocol nnNamenodeProtocol; + private NamenodeProtocol nnNamenodeProtocol1; /** Filesystem interface to the Router. */ private FileSystem routerFS; @@ -366,6 +368,11 @@ protected void setNamenode(NamenodeContext nn) NamenodeContext nn0 = cluster.getNamenode(ns0, null); this.nnNamenodeProtocol = NameNodeProxies.createProxy(nn0.getConf(), nn0.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); + // Namenode from the other namespace + String ns1 = cluster.getNameservices().get(1); + NamenodeContext nn1 = cluster.getNamenode(ns1, null); + this.nnNamenodeProtocol1 = NameNodeProxies.createProxy(nn1.getConf(), + nn1.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); } protected String getNs() { @@ -1302,11 +1309,14 @@ public void testProxyVersionRequest() throws Exception { // Check with default namespace specified. NamespaceInfo rVersion = routerNamenodeProtocol.versionRequest(); NamespaceInfo nnVersion = nnNamenodeProtocol.versionRequest(); + NamespaceInfo nnVersion1 = nnNamenodeProtocol1.versionRequest(); compareVersion(rVersion, nnVersion); // Check with default namespace unspecified. resolver.setDisableNamespace(true); - rVersion = routerNamenodeProtocol.versionRequest(); - compareVersion(rVersion, nnVersion); + // Verify the NamespaceInfo is of nn0 or nn1 + boolean isNN0 = + rVersion.getBlockPoolID().equals(nnVersion.getBlockPoolID()); + compareVersion(rVersion, isNN0 ? nnVersion : nnVersion1); } finally { resolver.setDisableNamespace(false); } @@ -1375,11 +1385,13 @@ public void testProxyGetTransactionID() throws IOException { // Check with default namespace specified. long routerTransactionID = routerNamenodeProtocol.getTransactionID(); long nnTransactionID = nnNamenodeProtocol.getTransactionID(); + long nnTransactionID1 = nnNamenodeProtocol1.getTransactionID(); assertEquals(nnTransactionID, routerTransactionID); // Check with default namespace unspecified. resolver.setDisableNamespace(true); + // Verify the transaction ID is of nn0 or nn1 routerTransactionID = routerNamenodeProtocol.getTransactionID(); - assertEquals(nnTransactionID, routerTransactionID); + assertThat(routerTransactionID).isIn(nnTransactionID, nnTransactionID1); } finally { resolver.setDisableNamespace(false); } From 62bcc79a2c5bc5a6808edee142100672eebb5035 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 3 May 2021 21:40:26 +0530 Subject: [PATCH 36/48] HADOOP-17676. Restrict imports from org.apache.curator.shaded (#2969) Signed-off-by: Akira Ajisaka --- pom.xml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pom.xml b/pom.xml index 32090f86205d3..5056869cb2673 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x 2.4 3.0.2 3.0.0-M1 + 1.1.0 3.0.1 1.5 1.5 @@ -160,6 +161,33 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-thirdparty shaded instead of curator shaded + + org.apache.curator.shaded.** + + + + + + org.apache.maven.plugins From e571025f5b371ade25d1457f0186ba656bb71c5f Mon Sep 17 00:00:00 2001 From: kishendas Date: Tue, 4 May 2021 01:20:56 -0700 Subject: [PATCH 37/48] HADOOP-17657: implement StreamCapabilities in SequenceFile.Writer and fall back to flush, if hflush is not supported (#2949) Co-authored-by: Kishen Das Reviewed-by: Steve Loughran --- .../org/apache/hadoop/io/SequenceFile.java | 19 +++++++++++++- .../apache/hadoop/io/TestSequenceFile.java | 26 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index 3f4649f04dc9a..0581fb3f577a0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -27,6 +27,7 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.Options; import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.io.compress.CodecPool; import org.apache.hadoop.io.compress.CompressionCodec; @@ -834,7 +835,8 @@ public String toString() { } /** Write key/value pairs to a sequence-format file. */ - public static class Writer implements java.io.Closeable, Syncable { + public static class Writer implements java.io.Closeable, Syncable, + Flushable, StreamCapabilities { private Configuration conf; FSDataOutputStream out; boolean ownOutputStream = true; @@ -1367,6 +1369,21 @@ public void hflush() throws IOException { out.hflush(); } } + + @Override + public void flush() throws IOException { + if (out != null) { + out.flush(); + } + } + + @Override + public boolean hasCapability(String capability) { + if (out !=null && capability != null) { + return out.hasCapability(capability); + } + return false; + } /** Returns the configuration of this file. */ Configuration getConf() { return conf; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java index cf64bbc0f9457..8b604d9c7a44b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java @@ -30,6 +30,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.conf.*; +import org.assertj.core.api.Assertions; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -730,6 +731,31 @@ public void testSerializationAvailability() throws IOException { } } + @Test + public void testSequenceFileWriter() throws Exception { + Configuration conf = new Configuration(); + // This test only works with Raw File System and not Local File System + FileSystem fs = FileSystem.getLocal(conf).getRaw(); + Path p = new Path(GenericTestUtils + .getTempPath("testSequenceFileWriter.seq")); + try(SequenceFile.Writer writer = SequenceFile.createWriter( + fs, conf, p, LongWritable.class, Text.class)) { + Assertions.assertThat(writer.hasCapability + (StreamCapabilities.HSYNC)).isEqualTo(true); + Assertions.assertThat(writer.hasCapability( + StreamCapabilities.HFLUSH)).isEqualTo(true); + LongWritable key = new LongWritable(); + key.set(1); + Text value = new Text(); + value.set("somevalue"); + writer.append(key, value); + writer.flush(); + writer.hflush(); + writer.hsync(); + Assertions.assertThat(fs.getFileStatus(p).getLen()).isGreaterThan(0); + } + } + /** For debugging and testing. */ public static void main(String[] args) throws Exception { int count = 1024 * 1024; From 041488e8f533a27e9497e86bb779c0b82c2e1d71 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Tue, 4 May 2021 14:01:23 +0530 Subject: [PATCH 38/48] HDFS-15982. Deleted data using HTTP API should be saved to the trash (#2927) Reviewed-by: Siyao Meng --- .../web/resources/DeleteSkipTrashParam.java | 50 +++++++++++++++ .../fs/http/client/HttpFSFileSystem.java | 1 + .../hadoop/fs/http/server/FSOperations.java | 28 ++++++++- .../http/server/HttpFSParametersProvider.java | 22 ++++++- .../hadoop/fs/http/server/HttpFSServer.java | 9 ++- .../fs/http/server/TestHttpFSServer.java | 62 +++++++++++++++++++ .../src/main/webapps/router/explorer.html | 23 ++++++- .../src/main/webapps/router/explorer.js | 50 +++++++++++---- .../web/resources/NamenodeWebHdfsMethods.java | 61 +++++++++++++----- .../src/main/webapps/hdfs/explorer.html | 23 ++++++- .../src/main/webapps/hdfs/explorer.js | 46 ++++++++++---- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 6 +- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 35 ++++++++++- 13 files changed, 363 insertions(+), 53 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java new file mode 100644 index 0000000000000..5ca9d69d7c870 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.web.resources; + +/** + * SkipTrash param to be used by DELETE query. + */ +public class DeleteSkipTrashParam extends BooleanParam { + + public static final String NAME = "skiptrash"; + public static final String DEFAULT = FALSE; + + private static final Domain DOMAIN = new Domain(NAME); + + /** + * Constructor. + * @param value the parameter value. + */ + public DeleteSkipTrashParam(final Boolean value) { + super(DOMAIN, value); + } + + /** + * Constructor. + * @param str a string representation of the parameter value. + */ + public DeleteSkipTrashParam(final String str) { + this(DOMAIN.parse(str)); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 2866044badcd2..3cf7dad1dad31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -122,6 +122,7 @@ public class HttpFSFileSystem extends FileSystem public static final String ACLSPEC_PARAM = "aclspec"; public static final String DESTINATION_PARAM = "destination"; public static final String RECURSIVE_PARAM = "recursive"; + public static final String SKIP_TRASH_PARAM = "skiptrash"; public static final String SOURCES_PARAM = "sources"; public static final String OWNER_PARAM = "owner"; public static final String GROUP_PARAM = "group"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index e272cdc71b686..a092db09eed08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.XAttrCodec; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; @@ -53,6 +54,8 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.apache.hadoop.fs.permission.FsCreateModes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; @@ -75,6 +78,8 @@ @InterfaceAudience.Private public final class FSOperations { + private static final Logger LOG = LoggerFactory.getLogger(FSOperations.class); + private static int bufferSize = 4096; private FSOperations() { @@ -717,18 +722,22 @@ public static long copyBytes(InputStream in, OutputStream out, long count) */ @InterfaceAudience.Private public static class FSDelete implements FileSystemAccess.FileSystemExecutor { - private Path path; - private boolean recursive; + private final Path path; + private final boolean recursive; + private final boolean skipTrash; /** * Creates a Delete executor. * * @param path path to delete. * @param recursive if the delete should be recursive or not. + * @param skipTrash if the file must be deleted and not kept in trash + * regardless of fs.trash.interval config value. */ - public FSDelete(String path, boolean recursive) { + public FSDelete(String path, boolean recursive, boolean skipTrash) { this.path = new Path(path); this.recursive = recursive; + this.skipTrash = skipTrash; } /** @@ -743,6 +752,19 @@ public FSDelete(String path, boolean recursive) { */ @Override public JSONObject execute(FileSystem fs) throws IOException { + if (!skipTrash) { + boolean movedToTrash = Trash.moveToAppropriateTrash(fs, path, + fs.getConf()); + if (movedToTrash) { + HttpFSServerWebApp.getMetrics().incrOpsDelete(); + return toJSON( + StringUtils.toLowerCase(HttpFSFileSystem.DELETE_JSON), true); + } + // Same is the behavior with Delete shell command. + // If moveToAppropriateTrash() returns false, file deletion + // is attempted rather than throwing Error. + LOG.debug("Could not move {} to Trash, attempting removal", path); + } boolean deleted = fs.delete(path, recursive); HttpFSServerWebApp.get().getMetrics().incrOpsDelete(); return toJSON( diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index f6c84dcae4e07..b810ec6690372 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -81,7 +81,8 @@ public class HttpFSParametersProvider extends ParametersProvider { new Class[]{ReplicationParam.class}); PARAMS_DEF.put(Operation.SETTIMES, new Class[]{ModifiedTimeParam.class, AccessTimeParam.class}); - PARAMS_DEF.put(Operation.DELETE, new Class[]{RecursiveParam.class}); + PARAMS_DEF.put(Operation.DELETE, new Class[]{RecursiveParam.class, + DeleteSkipTrashParam.class}); PARAMS_DEF.put(Operation.SETACL, new Class[]{AclPermissionParam.class}); PARAMS_DEF.put(Operation.REMOVEACL, new Class[]{}); PARAMS_DEF.put(Operation.MODIFYACLENTRIES, @@ -242,6 +243,25 @@ public RecursiveParam() { } } + /** + * Class for delete's skipTrash parameter. + */ + @InterfaceAudience.Private + public static class DeleteSkipTrashParam extends BooleanParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SKIP_TRASH_PARAM; + + /** + * Constructor. + */ + public DeleteSkipTrashParam() { + super(NAME, false); + } + } + /** * Class for filter parameter. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index d0d76d6289912..e3254d5bd9eff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AclPermissionParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DeleteSkipTrashParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ECPolicyParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; @@ -548,9 +549,13 @@ public Response delete(@PathParam("path") String path, case DELETE: { Boolean recursive = params.get(RecursiveParam.NAME, RecursiveParam.class); - AUDIT_LOG.info("[{}] recursive [{}]", path, recursive); + Boolean skipTrashParam = params.get(DeleteSkipTrashParam.NAME, + DeleteSkipTrashParam.class); + boolean skipTrash = skipTrashParam != null && skipTrashParam; + AUDIT_LOG.info("[{}] recursive [{}] skipTrash [{}]", path, recursive, + skipTrash); FSOperations.FSDelete command = - new FSOperations.FSDelete(path, recursive); + new FSOperations.FSDelete(path, recursive, skipTrash); JSONObject json = fsExecute(user, command); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java index 6aa8aa346ef9b..cddf95e22129f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.http.server; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -537,6 +538,36 @@ private void createWithHttp(String filename, String perms, Assert.assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode()); } + private void deleteWithHttp(String filename, String perms, + String unmaskedPerms, Boolean skipTrash) throws Exception { + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + // Remove leading / from filename + if (filename.charAt(0) == '/') { + filename = filename.substring(1); + } + String pathOps; + if (perms == null) { + pathOps = MessageFormat.format("/webhdfs/v1/{0}?user.name={1}&op=DELETE", + filename, user); + } else { + pathOps = MessageFormat.format( + "/webhdfs/v1/{0}?user.name={1}&permission={2}&op=DELETE", + filename, user, perms); + } + if (unmaskedPerms != null) { + pathOps = pathOps + "&unmaskedpermission=" + unmaskedPerms; + } + if (skipTrash != null) { + pathOps = pathOps + "&skiptrash=" + skipTrash; + } + URL url = new URL(TestJettyHelper.getJettyURL(), pathOps); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.addRequestProperty("Content-Type", "application/octet-stream"); + conn.setRequestMethod("DELETE"); + conn.connect(); + Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + } + /** * Talks to the http interface to create a directory. * @@ -782,6 +813,37 @@ public void testPerms() throws Exception { Assert.assertTrue("321".equals(getPerms(statusJson))); } + /** + * Validate create and delete calls. + */ + @Test + @TestDir + @TestJetty + @TestHdfs + public void testCreateDelete() throws Exception { + final String dir1 = "/testCreateDelete1"; + final String path1 = dir1 + "/file1"; + final String dir2 = "/testCreateDelete2"; + final String path2 = dir2 + "/file2"; + + createHttpFSServer(false, false); + final Configuration conf = HttpFSServerWebApp.get() + .get(FileSystemAccess.class).getFileSystemConfiguration(); + conf.setLong(FS_TRASH_INTERVAL_KEY, 5); + writeConf(conf, "hdfs-site.xml"); + + FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); + fs.mkdirs(new Path(dir1)); + + createWithHttp(path1, null); + deleteWithHttp(path1, null, null, null); + + fs.mkdirs(new Path(dir2)); + + createWithHttp(path2, null); + deleteWithHttp(path2, null, null, true); + } + /** * Validate XAttr get/set/remove calls. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html index 80b38e7165f78..3150d87bdce62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html @@ -165,11 +165,32 @@
+ data-complete-text="Deleting..." data-toggle="modal" data-target="#delete-trash-modal">Delete +
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js index cb16eac7b1221..0895eb967a6f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.js @@ -82,23 +82,47 @@ function delete_path(inode_name, absolute_file_path) { $('#delete-modal-title').text("Delete - " + inode_name); $('#delete-prompt').text("Are you sure you want to delete " + inode_name - + " ?"); - - $('#delete-button').click(function() { + + " ?"); + $('#delete-trash-modal-title').text("Skip Trash - " + inode_name); + $('#delete-trash-prompt').text("Skipping Trash might delete file forever." + + " Do you want to skip-trash " + inode_name + + " ? (default behaviour - No)"); + + $('#skip-trash-button').click(function () { + // DELETE /webhdfs/v1/?op=DELETE&recursive=&skiptrash=true + var url = '/webhdfs/v1' + encode_path(absolute_file_path) + + '?op=DELETE' + '&recursive=true&skiptrash=true'; + $.ajax(url, + { + type: 'DELETE' + }).done(function (data) { + browse_directory(current_directory); + }).fail(network_error_handler(url) + ).always(function () { + $('#delete-modal').modal('hide'); + $('#delete-button').button('reset'); + $('#delete-trash-modal').modal('hide'); + $('#skip-trash-button').button('reset'); + }); + }) + $('#trash-button').click(function () { // DELETE /webhdfs/v1/?op=DELETE&recursive= var url = '/webhdfs/v1' + encode_path(absolute_file_path) + - '?op=DELETE' + '&recursive=true'; - + '?op=DELETE' + '&recursive=true'; $.ajax(url, - { type: 'DELETE' - }).done(function(data) { - browse_directory(current_directory); - }).fail(network_error_handler(url) - ).always(function() { - $('#delete-modal').modal('hide'); - $('#delete-button').button('reset'); - }); + { + type: 'DELETE' + }).done(function (data) { + browse_directory(current_directory); + }).fail(network_error_handler(url) + ).always(function () { + $('#delete-modal').modal('hide'); + $('#delete-button').button('reset'); + $('#delete-trash-modal').modal('hide'); + $('#trash-button').button('reset'); + }); }) + $('#delete-modal').modal(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index e7bf32fcc05f0..2703349db8704 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -55,8 +55,10 @@ import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,6 +121,9 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import com.sun.jersey.spi.container.ResourceFilters; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; + /** Web-hdfs NameNode implementation. */ @Path("") @ResourceFilters(ParamFilter.class) @@ -1503,10 +1508,13 @@ public Response deleteRoot( @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) final RecursiveParam recursive, @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) - final SnapshotNameParam snapshotName + final SnapshotNameParam snapshotName, + @QueryParam(DeleteSkipTrashParam.NAME) + @DefaultValue(DeleteSkipTrashParam.DEFAULT) + final DeleteSkipTrashParam skiptrash ) throws IOException, InterruptedException { return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive, - snapshotName); + snapshotName, skiptrash); } /** Handle HTTP DELETE request. */ @@ -1527,34 +1535,53 @@ public Response delete( @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) final RecursiveParam recursive, @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) - final SnapshotNameParam snapshotName + final SnapshotNameParam snapshotName, + @QueryParam(DeleteSkipTrashParam.NAME) + @DefaultValue(DeleteSkipTrashParam.DEFAULT) + final DeleteSkipTrashParam skiptrash ) throws IOException, InterruptedException { - init(ugi, delegation, username, doAsUser, path, op, recursive, snapshotName); + init(ugi, delegation, username, doAsUser, path, op, recursive, + snapshotName, skiptrash); - return doAs(ugi, new PrivilegedExceptionAction() { - @Override - public Response run() throws IOException { - return delete(ugi, delegation, username, doAsUser, - path.getAbsolutePath(), op, recursive, snapshotName); - } - }); + return doAs(ugi, () -> delete( + path.getAbsolutePath(), op, recursive, snapshotName, skiptrash)); } protected Response delete( - final UserGroupInformation ugi, - final DelegationParam delegation, - final UserParam username, - final DoAsParam doAsUser, final String fullpath, final DeleteOpParam op, final RecursiveParam recursive, - final SnapshotNameParam snapshotName - ) throws IOException { + final SnapshotNameParam snapshotName, + final DeleteSkipTrashParam skipTrash) throws IOException { final ClientProtocol cp = getRpcClientProtocol(); switch(op.getValue()) { case DELETE: { + Configuration conf = + (Configuration) context.getAttribute(JspHelper.CURRENT_CONF); + long trashInterval = + conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); + if (trashInterval > 0 && !skipTrash.getValue()) { + LOG.info("{} is {} , trying to archive {} instead of removing", + FS_TRASH_INTERVAL_KEY, trashInterval, fullpath); + org.apache.hadoop.fs.Path path = + new org.apache.hadoop.fs.Path(fullpath); + Configuration clonedConf = new Configuration(conf); + // To avoid caching FS objects and prevent OOM issues + clonedConf.set("fs.hdfs.impl.disable.cache", "true"); + FileSystem fs = FileSystem.get(clonedConf); + boolean movedToTrash = Trash.moveToAppropriateTrash(fs, path, + clonedConf); + if (movedToTrash) { + final String js = JsonUtil.toJsonString("boolean", true); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + // Same is the behavior with Delete shell command. + // If moveToAppropriateTrash() returns false, file deletion + // is attempted rather than throwing Error. + LOG.debug("Could not move {} to Trash, attempting removal", fullpath); + } final boolean b = cp.delete(fullpath, recursive.getValue()); final String js = JsonUtil.toJsonString("boolean", b); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html index 3f0509a229700..eb7ca3504891a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html @@ -166,11 +166,32 @@
+ data-complete-text="Deleting..." data-toggle="modal" data-target="#delete-trash-modal">Delete +
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js index ea8b0accbde6c..d9371bb9288e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js @@ -83,22 +83,46 @@ $('#delete-modal-title').text("Delete - " + inode_name); $('#delete-prompt').text("Are you sure you want to delete " + inode_name + " ?"); + $('#delete-trash-modal-title').text("Skip Trash - " + inode_name); + $('#delete-trash-prompt').text("Skipping Trash might delete file forever." + + " Do you want to skip-trash " + inode_name + + " ? (default behaviour - No)"); - $('#delete-button').click(function() { + $('#skip-trash-button').click(function () { + // DELETE /webhdfs/v1/?op=DELETE&recursive=&skiptrash=true + var url = '/webhdfs/v1' + encode_path(absolute_file_path) + + '?op=DELETE' + '&recursive=true&skiptrash=true'; + $.ajax(url, + { + type: 'DELETE' + }).done(function (data) { + browse_directory(current_directory); + }).fail(network_error_handler(url) + ).always(function () { + $('#delete-modal').modal('hide'); + $('#delete-button').button('reset'); + $('#delete-trash-modal').modal('hide'); + $('#skip-trash-button').button('reset'); + }); + }) + $('#trash-button').click(function () { // DELETE /webhdfs/v1/?op=DELETE&recursive= var url = '/webhdfs/v1' + encode_path(absolute_file_path) + - '?op=DELETE' + '&recursive=true'; - + '?op=DELETE' + '&recursive=true'; $.ajax(url, - { type: 'DELETE' - }).done(function(data) { - browse_directory(current_directory); - }).fail(network_error_handler(url) - ).always(function() { - $('#delete-modal').modal('hide'); - $('#delete-button').button('reset'); - }); + { + type: 'DELETE' + }).done(function (data) { + browse_directory(current_directory); + }).fail(network_error_handler(url) + ).always(function () { + $('#delete-modal').modal('hide'); + $('#delete-button').button('reset'); + $('#delete-trash-modal').modal('hide'); + $('#trash-button').button('reset'); + }); }) + $('#delete-modal').modal(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 3fb68d40e929b..07891ae64b3e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -462,7 +462,11 @@ See also: [`destination`](#Destination), [FileSystem](../../api/org/apache/hadoo * Submit a HTTP DELETE request. curl -i -X DELETE "http://:/webhdfs/v1/?op=DELETE - [&recursive=]" + [&recursive=][&skiptrash=]" + + Default values of queryparams if not provided: + 1. recursive: false + 2. skiptrash: false The client receives a response with a [`boolean` JSON object](#Boolean_JSON_Schema): diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 68087c4c4ed6e..698c323e9a970 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.web; +import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; @@ -58,6 +60,8 @@ import java.util.NoSuchElementException; import java.util.Random; +import org.apache.hadoop.hdfs.web.resources.DeleteSkipTrashParam; +import org.apache.hadoop.hdfs.web.resources.RecursiveParam; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.QuotaUsage; @@ -1559,8 +1563,12 @@ private void checkResponseContainsLocation(URL url, String TYPE) HttpURLConnection.HTTP_OK, conn.getResponseCode()); JSONObject responseJson = new JSONObject(response); - Assert.assertTrue("Response didn't give us a location. " + response, - responseJson.has("Location")); + if (!TYPE.equals("DELETE")) { + Assert.assertTrue("Response didn't give us a location. " + response, + responseJson.has("Location")); + } else { + Assert.assertTrue(responseJson.getBoolean("boolean")); + } //Test that the DN allows CORS on Create if(TYPE.equals("CREATE")) { @@ -1572,14 +1580,15 @@ private void checkResponseContainsLocation(URL url, String TYPE) } } - @Test /** * Test that when "&noredirect=true" is added to operations CREATE, APPEND, * OPEN, and GETFILECHECKSUM the response (which is usually a 307 temporary * redirect) is a 200 with JSON that contains the redirected location */ + @Test public void testWebHdfsNoRedirect() throws Exception { final Configuration conf = WebHdfsTestUtil.createConf(); + conf.setLong(FS_TRASH_INTERVAL_KEY, 5); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); LOG.info("Started cluster"); InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); @@ -1618,6 +1627,26 @@ public void testWebHdfsNoRedirect() throws Exception { + Param.toSortedString("&", new NoRedirectParam(true))); LOG.info("Sending append request " + url); checkResponseContainsLocation(url, "POST"); + + // setup some permission to allow moving file to .Trash location + cluster.getFileSystem().setPermission(new Path("/testWebHdfsNoRedirect"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + Path userDir = new Path(FileSystem.USER_HOME_PREFIX); + Path trashDir = new Path(FileSystem.USER_HOME_PREFIX, DEFAULT_HADOOP_HTTP_STATIC_USER); + Path trashPath = new Path(FileSystem.USER_HOME_PREFIX, + new Path(DEFAULT_HADOOP_HTTP_STATIC_USER, FileSystem.TRASH_PREFIX)); + cluster.getFileSystem().mkdirs(userDir, FsPermission.getDirDefault()); + cluster.getFileSystem().mkdir(trashDir, FsPermission.getDirDefault()); + cluster.getFileSystem().mkdir(trashPath, FsPermission.getDirDefault()); + cluster.getFileSystem().setOwner(trashPath, DEFAULT_HADOOP_HTTP_STATIC_USER, HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT); + cluster.getFileSystem().setPermission(new Path("/"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + + url = new URL("http", addr.getHostString(), addr.getPort(), + WebHdfsFileSystem.PATH_PREFIX + "/testWebHdfsNoRedirect" + "?op=DELETE" + + Param.toSortedString("&", new RecursiveParam(true)) + + Param.toSortedString("&", new DeleteSkipTrashParam(false))); + LOG.info("Sending append request " + url); + checkResponseContainsLocation(url, "DELETE"); } @Test From 68425eb469560d08b779e240dd74720328556b22 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 4 May 2021 10:49:52 +0100 Subject: [PATCH 39/48] HADOOP-16742. NullPointerException in S3A MultiObjectDeleteSupport Contributed by Tor Arvid Lund. Change-Id: Iadfe9b2f355cf373031075bfbe681705a2c65bdc --- .../org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java index 0a8338934c9ef..d2b3899a69104 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java @@ -106,7 +106,7 @@ public static IOException translateDeleteException( error.getMessage()); LOG.warn(item); result.append(item); - if (exitCode.isEmpty() || ACCESS_DENIED.equals(code)) { + if (exitCode == null || exitCode.isEmpty() || ACCESS_DENIED.equals(code)) { exitCode = code; } } From 0d78d73973cf8643c4120678ebeea9cde473a2c4 Mon Sep 17 00:00:00 2001 From: Istvan Fajth Date: Tue, 4 May 2021 12:33:59 +0200 Subject: [PATCH 40/48] HADOOP-17675. LdapGroupsMapping$LdapSslSocketFactory ClassNotFoundException (#2965) --- .../hadoop/security/LdapGroupsMapping.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index 10483a2c56571..17934add81413 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -643,7 +643,29 @@ private DirContext getDirContext() throws NamingException { env.put("com.sun.jndi.ldap.read.timeout", conf.get(READ_TIMEOUT, String.valueOf(READ_TIMEOUT_DEFAULT))); - ctx = new InitialDirContext(env); + // See HADOOP-17675 for details TLDR: + // From a native thread the thread's context classloader is null. + // jndi internally in the InitialDirContext specifies the context + // classloader for Class.forName, and as it is null, jndi will use the + // bootstrap classloader in this case to laod the socket factory + // implementation. + // BUT + // Bootstrap classloader does not have it in its classpath, so throws a + // ClassNotFoundException. + // This affects Impala for example when it uses LdapGroupsMapping. + ClassLoader currentContextLoader = + Thread.currentThread().getContextClassLoader(); + if (currentContextLoader == null) { + try { + Thread.currentThread().setContextClassLoader( + this.getClass().getClassLoader()); + ctx = new InitialDirContext(env); + } finally { + Thread.currentThread().setContextClassLoader(null); + } + } else { + ctx = new InitialDirContext(env); + } } return ctx; } From 76c69c39d97c1fdeee61dc5c3db52fe455be8ddc Mon Sep 17 00:00:00 2001 From: zhuqi Date: Wed, 5 May 2021 23:32:13 +0800 Subject: [PATCH 41/48] HDFS-15934. Make DirectoryScanner reconcile blocks batch size and interval between batch configurable. Contributed by Qi Zhu. (#2833) Signed-off-by: Ayush Saxena --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ++++ .../server/datanode/DirectoryScanner.java | 44 +++++++++++++++++-- .../src/main/resources/hdfs-default.xml | 12 +++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 51fc564a97584..77355be0c2664 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -836,6 +836,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT = 21600; public static final String DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY = "dfs.datanode.directoryscan.threads"; public static final int DFS_DATANODE_DIRECTORYSCAN_THREADS_DEFAULT = 1; + public static final String DFS_DATANODE_RECONCILE_BLOCKS_BATCH_SIZE = + "dfs.datanode.reconcile.blocks.batch.size"; + public static final int + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_SIZE_DEFAULT = 1000; + public static final String DFS_DATANODE_RECONCILE_BLOCKS_BATCH_INTERVAL + = "dfs.datanode.reconcile.blocks.batch.interval"; + public static final long + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_INTERVAL_DEFAULT = 2000; public static final String DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY = "dfs.datanode.disk.check.min.gap"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index a3bceec9a96a7..7a8569d39008e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -65,7 +65,8 @@ public class DirectoryScanner implements Runnable { LoggerFactory.getLogger(DirectoryScanner.class); private static final int DEFAULT_MAP_SIZE = 32768; - private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000; + private final int reconcileBlocksBatchSize; + private final long reconcileBlocksBatchInterval; private final FsDatasetSpi dataset; private final ExecutorService reportCompileThreadPool; private final ScheduledExecutorService masterThread; @@ -315,6 +316,41 @@ public DirectoryScanner(FsDatasetSpi dataset, Configuration conf) { masterThread = new ScheduledThreadPoolExecutor(1, new Daemon.DaemonFactory()); + + int reconcileBatchSize = + conf.getInt(DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_SIZE, + DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_SIZE_DEFAULT); + + if (reconcileBatchSize <= 0) { + LOG.warn("Invalid value configured for " + + "dfs.datanode.reconcile.blocks.batch.size, " + + "should be greater than 0, Using default."); + reconcileBatchSize = + DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_SIZE_DEFAULT; + } + + reconcileBlocksBatchSize = reconcileBatchSize; + + long reconcileBatchInterval = + conf.getTimeDuration(DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_INTERVAL, + DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + + if (reconcileBatchInterval <= 0) { + LOG.warn("Invalid value configured for " + + "dfs.datanode.reconcile.blocks.batch.interval, " + + "should be greater than 0, Using default."); + reconcileBatchInterval = + DFSConfigKeys. + DFS_DATANODE_RECONCILE_BLOCKS_BATCH_INTERVAL_DEFAULT; + } + + reconcileBlocksBatchInterval = reconcileBatchInterval; } /** @@ -428,16 +464,16 @@ public void reconcile() throws IOException { LOG.debug("reconcile start DirectoryScanning"); scan(); - // HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too + // HDFS-14476: run checkAndUpdate with batch to avoid holding the lock too // long int loopCount = 0; synchronized (diffs) { for (final Map.Entry entry : diffs.getEntries()) { dataset.checkAndUpdate(entry.getKey(), entry.getValue()); - if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) { + if (loopCount % reconcileBlocksBatchSize == 0) { try { - Thread.sleep(2000); + Thread.sleep(reconcileBlocksBatchInterval); } catch (InterruptedException e) { // do nothing } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a30889bdd0042..6e8e36e1cf0dc 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -873,6 +873,18 @@ + + dfs.datanode.reconcile.blocks.batch.size + 1000 + Setting this to define reconcile batch size. + + + + dfs.datanode.reconcile.blocks.batch.interval + 2000 + Setting this to define interval between batches. + + dfs.heartbeat.interval 3s From b93e448f9aa66689f1ce5059f6cdce8add130457 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 6 May 2021 01:22:02 +0530 Subject: [PATCH 42/48] HADOOP-11616. Remove workaround for Curator's ChildReaper requiring Guava 15+ (#2973) Reviewed-by: Wei-Chiu Chuang Signed-off-by: Akira Ajisaka --- .../hadoop/util/curator/ChildReaper.java | 234 ------------------ .../hadoop/util/curator/TestChildReaper.java | 209 ---------------- 2 files changed, 443 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestChildReaper.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java deleted file mode 100644 index e125dbfbd0abb..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.util.curator; - -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; -import org.apache.curator.framework.recipes.locks.Reaper; -import org.apache.curator.utils.CloseableUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.utils.CloseableScheduledExecutorService; -import org.apache.curator.utils.ThreadUtils; -import org.apache.curator.utils.ZKPaths; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.Closeable; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.curator.utils.PathUtils; - -/** - * This is a copy of Curator 2.7.1's ChildReaper class, modified to work with - * Guava 11.0.2. The problem is the 'paths' Collection, which calls Guava's - * Sets.newConcurrentHashSet(), which was added in Guava 15.0. - *

- * Utility to reap empty child nodes of a parent node. Periodically calls getChildren on - * the node and adds empty nodes to an internally managed {@link Reaper} - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class ChildReaper implements Closeable -{ - private final Logger log = LoggerFactory.getLogger(getClass()); - private final Reaper reaper; - private final AtomicReference state = new AtomicReference(State.LATENT); - private final CuratorFramework client; - private final Collection paths = newConcurrentHashSet(); - private final Reaper.Mode mode; - private final CloseableScheduledExecutorService executor; - private final int reapingThresholdMs; - - private volatile Future task; - - // This is copied from Curator's Reaper class - static final int DEFAULT_REAPING_THRESHOLD_MS = (int)TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES); - - // This is copied from Guava - /** - * Creates a thread-safe set backed by a hash map. The set is backed by a - * {@link ConcurrentHashMap} instance, and thus carries the same concurrency - * guarantees. - * - *

Unlike {@code HashSet}, this class does NOT allow {@code null} to be - * used as an element. The set is serializable. - * - * @return a new, empty thread-safe {@code Set} - * @since 15.0 - */ - public static Set newConcurrentHashSet() { - return Collections.newSetFromMap(new ConcurrentHashMap()); - } - - private enum State - { - LATENT, - STARTED, - CLOSED - } - - /** - * @param client the client - * @param path path to reap children from - * @param mode reaping mode - */ - public ChildReaper(CuratorFramework client, String path, Reaper.Mode mode) - { - this(client, path, mode, newExecutorService(), DEFAULT_REAPING_THRESHOLD_MS, null); - } - - /** - * @param client the client - * @param path path to reap children from - * @param reapingThresholdMs threshold in milliseconds that determines that a path can be deleted - * @param mode reaping mode - */ - public ChildReaper(CuratorFramework client, String path, Reaper.Mode mode, int reapingThresholdMs) - { - this(client, path, mode, newExecutorService(), reapingThresholdMs, null); - } - - /** - * @param client the client - * @param path path to reap children from - * @param executor executor to use for background tasks - * @param reapingThresholdMs threshold in milliseconds that determines that a path can be deleted - * @param mode reaping mode - */ - public ChildReaper(CuratorFramework client, String path, Reaper.Mode mode, ScheduledExecutorService executor, int reapingThresholdMs) - { - this(client, path, mode, executor, reapingThresholdMs, null); - } - - /** - * @param client the client - * @param path path to reap children from - * @param executor executor to use for background tasks - * @param reapingThresholdMs threshold in milliseconds that determines that a path can be deleted - * @param mode reaping mode - * @param leaderPath if not null, uses a leader selection so that only 1 reaper is active in the cluster - */ - public ChildReaper(CuratorFramework client, String path, Reaper.Mode mode, ScheduledExecutorService executor, int reapingThresholdMs, String leaderPath) - { - this.client = client; - this.mode = mode; - this.executor = new CloseableScheduledExecutorService(executor); - this.reapingThresholdMs = reapingThresholdMs; - this.reaper = new Reaper(client, executor, reapingThresholdMs, leaderPath); - addPath(path); - } - - /** - * The reaper must be started - * - * @throws Exception errors - */ - public void start() throws Exception - { - Preconditions.checkState(state.compareAndSet(State.LATENT, State.STARTED), "Cannot be started more than once"); - - task = executor.scheduleWithFixedDelay - ( - new Runnable() - { - @Override - public void run() - { - doWork(); - } - }, - reapingThresholdMs, - reapingThresholdMs, - TimeUnit.MILLISECONDS - ); - - reaper.start(); - } - - @Override - public void close() throws IOException - { - if ( state.compareAndSet(State.STARTED, State.CLOSED) ) - { - CloseableUtils.closeQuietly(reaper); - task.cancel(true); - } - } - - /** - * Add a path to reap children from - * - * @param path the path - * @return this for chaining - */ - public ChildReaper addPath(String path) - { - paths.add(PathUtils.validatePath(path)); - return this; - } - - /** - * Remove a path from reaping - * - * @param path the path - * @return true if the path existed and was removed - */ - public boolean removePath(String path) - { - return paths.remove(PathUtils.validatePath(path)); - } - - private static ScheduledExecutorService newExecutorService() - { - return ThreadUtils.newFixedThreadScheduledPool(2, "ChildReaper"); - } - - private void doWork() - { - for ( String path : paths ) - { - try - { - List children = client.getChildren().forPath(path); - for ( String name : children ) - { - String thisPath = ZKPaths.makePath(path, name); - Stat stat = client.checkExists().forPath(thisPath); - if ( (stat != null) && (stat.getNumChildren() == 0) ) - { - reaper.addPath(thisPath, mode); - } - } - } - catch ( Exception e ) - { - log.error("Could not get children for path: " + path, e); - } - } - } -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestChildReaper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestChildReaper.java deleted file mode 100644 index 960471841948c..0000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestChildReaper.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.util.curator; - -import org.apache.curator.framework.recipes.locks.Reaper; -import org.apache.curator.test.TestingServer; -import org.apache.curator.utils.CloseableUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryOneTime; -import org.apache.curator.test.Timing; -import org.apache.zookeeper.data.Stat; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.net.BindException; -import java.util.Random; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * This is a copy of Curator 2.7.1's TestChildReaper class, with minor - * modifications to make it work with JUnit (some setup code taken from - * Curator's BaseClassForTests). This is to ensure that the ChildReaper - * class we modified is still correct. - */ -public class TestChildReaper -{ - protected TestingServer server; - - @Before - public void setup() throws Exception { - while(this.server == null) { - try { - this.server = new TestingServer(); - } catch (BindException var2) { - System.err.println("Getting bind exception - retrying to allocate server"); - this.server = null; - } - } - } - - @After - public void teardown() throws Exception { - this.server.close(); - this.server = null; - } - - @Test - public void testSomeNodes() throws Exception - { - - Timing timing = new Timing(); - ChildReaper reaper = null; - CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); - try - { - client.start(); - - Random r = new Random(); - int nonEmptyNodes = 0; - for ( int i = 0; i < 10; ++i ) - { - client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i)); - if ( r.nextBoolean() ) - { - client.create().forPath("/test/" + Integer.toString(i) + "/foo"); - ++nonEmptyNodes; - } - } - - reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1); - reaper.start(); - - timing.forWaiting().sleepABit(); - - Stat stat = client.checkExists().forPath("/test"); - assertThat(stat.getNumChildren()).isEqualTo(nonEmptyNodes); - } - finally - { - CloseableUtils.closeQuietly(reaper); - CloseableUtils.closeQuietly(client); - } - } - - @Test - public void testSimple() throws Exception - { - Timing timing = new Timing(); - ChildReaper reaper = null; - CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); - try - { - client.start(); - - for ( int i = 0; i < 10; ++i ) - { - client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i)); - } - - reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1); - reaper.start(); - - timing.forWaiting().sleepABit(); - - Stat stat = client.checkExists().forPath("/test"); - assertThat(stat.getNumChildren()).isZero(); - } - finally - { - CloseableUtils.closeQuietly(reaper); - CloseableUtils.closeQuietly(client); - } - } - - @Test - public void testMultiPath() throws Exception - { - Timing timing = new Timing(); - ChildReaper reaper = null; - CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); - try - { - client.start(); - - for ( int i = 0; i < 10; ++i ) - { - client.create().creatingParentsIfNeeded().forPath("/test1/" + Integer.toString(i)); - client.create().creatingParentsIfNeeded().forPath("/test2/" + Integer.toString(i)); - client.create().creatingParentsIfNeeded().forPath("/test3/" + Integer.toString(i)); - } - - reaper = new ChildReaper(client, "/test2", Reaper.Mode.REAP_UNTIL_DELETE, 1); - reaper.start(); - reaper.addPath("/test1"); - - timing.forWaiting().sleepABit(); - - Stat stat = client.checkExists().forPath("/test1"); - assertThat(stat.getNumChildren()).isZero(); - stat = client.checkExists().forPath("/test2"); - assertThat(stat.getNumChildren()).isZero(); - stat = client.checkExists().forPath("/test3"); - assertThat(stat.getNumChildren()).isEqualTo(10); - } - finally - { - CloseableUtils.closeQuietly(reaper); - CloseableUtils.closeQuietly(client); - } - } - - @Test - public void testNamespace() throws Exception - { - Timing timing = new Timing(); - ChildReaper reaper = null; - CuratorFramework client = CuratorFrameworkFactory.builder() - .connectString(server.getConnectString()) - .sessionTimeoutMs(timing.session()) - .connectionTimeoutMs(timing.connection()) - .retryPolicy(new RetryOneTime(1)) - .namespace("foo") - .build(); - try - { - client.start(); - - for ( int i = 0; i < 10; ++i ) - { - client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i)); - } - - reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1); - reaper.start(); - - timing.forWaiting().sleepABit(); - - Stat stat = client.checkExists().forPath("/test"); - assertThat(stat.getNumChildren()).isZero(); - - stat = client.usingNamespace(null).checkExists().forPath("/foo/test"); - assertThat(stat).isNotNull(); - assertThat(stat.getNumChildren()).isZero(); - } - finally - { - CloseableUtils.closeQuietly(reaper); - CloseableUtils.closeQuietly(client); - } - } -} From 7cb7b8adc137239682e6189d1663b6b868757938 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 6 May 2021 23:37:45 +0900 Subject: [PATCH 43/48] YARN-10756. Remove additional junit 4.11 dependency from javadoc. (#2960) Reviewed-by: Viraj Jasani Reviewed-by: Bilwa S T Signed-off-by: Takanobu Asanuma --- .../pom.xml | 13 ------------- .../pom.xml | 13 ------------- .../pom.xml | 13 ------------- .../hadoop-yarn-server-timelineservice/pom.xml | 13 ------------- 4 files changed, 52 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml index c51b5a20baecb..ccd1b13d1572a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml @@ -142,19 +142,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - - junit - junit - 4.11 - - - - org.apache.maven.plugins maven-dependency-plugin diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 12b0112680320..d8cf6ded0001e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -406,19 +406,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - - junit - junit - 4.11 - - - - org.apache.maven.plugins diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml index 355a2697496dc..6207a2d024eb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml @@ -204,19 +204,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - - junit - junit - 4.11 - - - - org.apache.maven.plugins maven-dependency-plugin diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index cb4f09766b881..65af3afadda62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -157,19 +157,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - - junit - junit - 4.11 - - - - From f40e3eb0590f85bb42d2471992bf5d524628fdd6 Mon Sep 17 00:00:00 2001 From: hchaverr Date: Thu, 6 May 2021 16:40:45 -0700 Subject: [PATCH 44/48] HADOOP-17680. Allow ProtobufRpcEngine to be extensible (#2905) Contributed by Hector Chaverri. --- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 30 +++++++++++++++---- .../apache/hadoop/ipc/ProtobufRpcEngine2.java | 30 +++++++++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index a1500d52a7444..882cc141d8913 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -122,7 +122,7 @@ public ProtocolProxy getProtocolMetaInfoProxy( factory)), false); } - private static class Invoker implements RpcInvocationHandler { + protected static class Invoker implements RpcInvocationHandler { private final Map returnTypes = new ConcurrentHashMap(); private boolean isClosed = false; @@ -133,7 +133,7 @@ private static class Invoker implements RpcInvocationHandler { private AtomicBoolean fallbackToSimpleAuth; private AlignmentContext alignmentContext; - private Invoker(Class protocol, InetSocketAddress addr, + protected Invoker(Class protocol, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy, AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext) @@ -148,7 +148,7 @@ private Invoker(Class protocol, InetSocketAddress addr, /** * This constructor takes a connectionId, instead of creating a new one. */ - private Invoker(Class protocol, Client.ConnectionId connId, + protected Invoker(Class protocol, Client.ConnectionId connId, Configuration conf, SocketFactory factory) { this.remoteId = connId; this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class); @@ -225,8 +225,6 @@ public Message invoke(Object proxy, final Method method, Object[] args) traceScope = tracer.newScope(RpcClientUtil.methodToTraceString(method)); } - RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method); - if (LOG.isTraceEnabled()) { LOG.trace(Thread.currentThread().getId() + ": Call -> " + remoteId + ": " + method.getName() + @@ -238,7 +236,7 @@ public Message invoke(Object proxy, final Method method, Object[] args) final RpcWritable.Buffer val; try { val = (RpcWritable.Buffer) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER, - new RpcProtobufRequest(rpcRequestHeader, theRequest), remoteId, + constructRpcRequest(method, theRequest), remoteId, fallbackToSimpleAuth, alignmentContext); } catch (Throwable e) { @@ -283,6 +281,11 @@ public boolean isDone() { } } + protected Writable constructRpcRequest(Method method, Message theRequest) { + RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method); + return new RpcProtobufRequest(rpcRequestHeader, theRequest); + } + private Message getReturnMessage(final Method method, final RpcWritable.Buffer buf) throws ServiceException { Message prototype = null; @@ -332,6 +335,14 @@ private Message getReturnProtoType(Method method) throws Exception { public ConnectionId getConnectionId() { return remoteId; } + + protected long getClientProtocolVersion() { + return clientProtocolVersion; + } + + protected String getProtocolName() { + return protocolName; + } } @VisibleForTesting @@ -518,6 +529,13 @@ public Writable call(RPC.Server server, String connectionProtocolName, String declaringClassProtoName = rpcRequest.getDeclaringClassProtocolName(); long clientVersion = rpcRequest.getClientProtocolVersion(); + return call(server, connectionProtocolName, request, receiveTime, + methodName, declaringClassProtoName, clientVersion); + } + + protected Writable call(RPC.Server server, String connectionProtocolName, + RpcWritable.Buffer request, long receiveTime, String methodName, + String declaringClassProtoName, long clientVersion) throws Exception { if (server.verbose) LOG.info("Call: connectionProtocolName=" + connectionProtocolName + ", method=" + methodName); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java index 310f44eebe213..2f5d56437d034 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java @@ -116,7 +116,7 @@ public ProtocolProxy getProtocolMetaInfoProxy( factory)), false); } - private static final class Invoker implements RpcInvocationHandler { + protected static class Invoker implements RpcInvocationHandler { private final Map returnTypes = new ConcurrentHashMap(); private boolean isClosed = false; @@ -127,7 +127,7 @@ private static final class Invoker implements RpcInvocationHandler { private AtomicBoolean fallbackToSimpleAuth; private AlignmentContext alignmentContext; - private Invoker(Class protocol, InetSocketAddress addr, + protected Invoker(Class protocol, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy, AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext) @@ -142,7 +142,7 @@ private Invoker(Class protocol, InetSocketAddress addr, /** * This constructor takes a connectionId, instead of creating a new one. */ - private Invoker(Class protocol, Client.ConnectionId connId, + protected Invoker(Class protocol, Client.ConnectionId connId, Configuration conf, SocketFactory factory) { this.remoteId = connId; this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class); @@ -219,8 +219,6 @@ public Message invoke(Object proxy, final Method method, Object[] args) traceScope = tracer.newScope(RpcClientUtil.methodToTraceString(method)); } - RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method); - if (LOG.isTraceEnabled()) { LOG.trace(Thread.currentThread().getId() + ": Call -> " + remoteId + ": " + method.getName() + @@ -232,7 +230,7 @@ public Message invoke(Object proxy, final Method method, Object[] args) final RpcWritable.Buffer val; try { val = (RpcWritable.Buffer) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER, - new RpcProtobufRequest(rpcRequestHeader, theRequest), remoteId, + constructRpcRequest(method, theRequest), remoteId, fallbackToSimpleAuth, alignmentContext); } catch (Throwable e) { @@ -279,6 +277,11 @@ public boolean isDone() { } } + protected Writable constructRpcRequest(Method method, Message theRequest) { + RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method); + return new RpcProtobufRequest(rpcRequestHeader, theRequest); + } + private Message getReturnMessage(final Method method, final RpcWritable.Buffer buf) throws ServiceException { Message prototype = null; @@ -328,6 +331,14 @@ private Message getReturnProtoType(Method method) throws Exception { public ConnectionId getConnectionId() { return remoteId; } + + protected long getClientProtocolVersion() { + return clientProtocolVersion; + } + + protected String getProtocolName() { + return protocolName; + } } @VisibleForTesting @@ -509,6 +520,13 @@ public Writable call(RPC.Server server, String connectionProtocolName, String declaringClassProtoName = rpcRequest.getDeclaringClassProtocolName(); long clientVersion = rpcRequest.getClientProtocolVersion(); + return call(server, connectionProtocolName, request, receiveTime, + methodName, declaringClassProtoName, clientVersion); + } + + protected Writable call(RPC.Server server, String connectionProtocolName, + RpcWritable.Buffer request, long receiveTime, String methodName, + String declaringClassProtoName, long clientVersion) throws Exception { if (server.verbose) { LOG.info("Call: connectionProtocolName=" + connectionProtocolName + ", method=" + methodName); From 207210263a27784bf3b61771d2f8364bda7bbb50 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Fri, 7 May 2021 13:52:17 +0900 Subject: [PATCH 45/48] HADOOP-17375. Fix the error of TestDynamometerInfra. (#2471) Signed-off-by: Akira Ajisaka --- .../tools/dynamometer/AllowAllImpersonationProvider.java | 5 +++++ .../hadoop/tools/dynamometer/TestDynamometerInfra.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AllowAllImpersonationProvider.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AllowAllImpersonationProvider.java index 6800bbe0ef814..14f52ac2b28a2 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AllowAllImpersonationProvider.java +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AllowAllImpersonationProvider.java @@ -38,4 +38,9 @@ public void authorize(UserGroupInformation user, InetAddress remoteAddress) { // Do nothing } + // Although this API was removed from the interface by HADOOP-17367, we need + // to keep it here because TestDynamometerInfra uses an old hadoop binary. + public void authorize(UserGroupInformation user, String remoteAddress) { + // Do nothing + } } diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java index 174d9cc039e03..092a54b78aaad 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java @@ -122,7 +122,7 @@ public class TestDynamometerInfra { private static final String HADOOP_BIN_PATH_KEY = "dyno.hadoop.bin.path"; private static final String HADOOP_BIN_VERSION_KEY = "dyno.hadoop.bin.version"; - private static final String HADOOP_BIN_VERSION_DEFAULT = "3.1.3"; + private static final String HADOOP_BIN_VERSION_DEFAULT = "3.1.4"; private static final String FSIMAGE_FILENAME = "fsimage_0000000000000061740"; private static final String VERSION_FILENAME = "VERSION"; From 9aa651f0b4daec1332afc2a527e5525f118c244b Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Sat, 8 May 2021 00:24:06 +0900 Subject: [PATCH 46/48] HDFS-16001. TestOfflineEditsViewer.testStored() fails reading negative value of FSEditLogOpCodes. (#2980) Reviewed-by: Xiaoqiao He --- .../src/test/resources/editsStored | Bin 7933 -> 7933 bytes .../src/test/resources/editsStored.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index 63c3a8f5c61bea4bfa9d3091c1659ac75e25f9b2..6a0891d2dc7fed0e41e576ef3b67c483fe8d5d68 100644 GIT binary patch delta 13 Vcmexs``4D`|NsAcH?sVc0{}Xs2mb&7 delta 13 Vcmexs``4D`|Ns97HnRMb0{}X*2m$~A diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 4e91a59e39ac8..afadc5d884565 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ - -64 + -67 OP_START_LOG_SEGMENT From 1b6994287a86a9a96907e641664ce75e522d0be2 Mon Sep 17 00:00:00 2001 From: Jinglun Date: Sat, 8 May 2021 19:50:17 +0800 Subject: [PATCH 47/48] HDFS-15923. RBF: Authentication failed when rename accross sub clusters (#2819). Contributed by zhuobin zheng. Reviewed-by: Jinglun Reviewed-by: Inigo Goiri --- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 6 + .../router/RouterFederationRename.java | 47 ++-- .../router/TestRouterFederationRename.java | 2 +- ...stRouterFederationRenameInKerberosEnv.java | 260 ++++++++++++++++++ 4 files changed, 298 insertions(+), 17 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameInKerberosEnv.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 8ba016a3f39ee..e17602d1f6466 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -101,6 +101,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-distcp test + + org.apache.zookeeper + zookeeper + test + test-jar + com.fasterxml.jackson.core jackson-annotations diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFederationRename.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFederationRename.java index 8074fdd2f64a7..d2bf989f1f46e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFederationRename.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFederationRename.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.fedbalance.DistCpProcedure; import org.apache.hadoop.tools.fedbalance.FedBalanceConfigs; import org.apache.hadoop.tools.fedbalance.FedBalanceContext; @@ -31,6 +32,8 @@ import org.apache.hadoop.tools.fedbalance.procedure.BalanceProcedureScheduler; import java.io.IOException; +import java.io.InterruptedIOException; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -96,23 +99,35 @@ boolean routerFedRename(final String src, final String dst, } RemoteLocation srcLoc = srcLocations.get(0); RemoteLocation dstLoc = dstLocations.get(0); - // Build and submit router federation rename job. - BalanceJob job = buildRouterRenameJob(srcLoc.getNameserviceId(), - dstLoc.getNameserviceId(), srcLoc.getDest(), dstLoc.getDest()); - BalanceProcedureScheduler scheduler = rpcServer.getFedRenameScheduler(); - countIncrement(); + + UserGroupInformation routerUser = UserGroupInformation.getLoginUser(); + try { - scheduler.submit(job); - LOG.info("Rename {} to {} from namespace {} to {}. JobId={}.", src, dst, - srcLoc.getNameserviceId(), dstLoc.getNameserviceId(), job.getId()); - scheduler.waitUntilDone(job); - if (job.getError() != null) { - throw new IOException("Rename of " + src + " to " + dst + " failed.", - job.getError()); - } - return true; - } finally { - countDecrement(); + // as router user with saveJournal and task submission privileges + return routerUser.doAs((PrivilegedExceptionAction) () -> { + // Build and submit router federation rename job. + BalanceJob job = buildRouterRenameJob(srcLoc.getNameserviceId(), + dstLoc.getNameserviceId(), srcLoc.getDest(), dstLoc.getDest()); + BalanceProcedureScheduler scheduler = rpcServer.getFedRenameScheduler(); + countIncrement(); + try { + scheduler.submit(job); + LOG.info("Rename {} to {} from namespace {} to {}. JobId={}.", src, + dst, srcLoc.getNameserviceId(), dstLoc.getNameserviceId(), + job.getId()); + scheduler.waitUntilDone(job); + if (job.getError() != null) { + throw new IOException("Rename of " + src + " to " + dst + + " failed.", job.getError()); + } + return true; + } finally { + countDecrement(); + } + }); + } catch (InterruptedException e) { + LOG.warn("Fed balance job is interrupted.", e); + throw new InterruptedIOException(e.getMessage()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRename.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRename.java index c47098f3372cd..a9a17b3ef5025 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRename.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRename.java @@ -409,7 +409,7 @@ public void testRbfRenameWithMultiDestination() throws Exception { getRouterFileSystem().delete(new Path(renamedDir), true); } - @Test(timeout = 10000) + @Test(timeout = 20000) public void testCounter() throws Exception { final RouterRpcServer rpcServer = router.getRouter().getRpcServer(); List nss = cluster.getNameservices(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameInKerberosEnv.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameInKerberosEnv.java new file mode 100644 index 0000000000000..369508f3b22f7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameInKerberosEnv.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ha.ClientBaseWithFixes; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.tools.fedbalance.DistCpProcedure; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FEDERATION_RENAME_BANDWIDTH; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FEDERATION_RENAME_MAP; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; +import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZK_AUTH_TYPE; +import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZK_CONNECTION_STRING; +import static org.apache.hadoop.test.GenericTestUtils.getMethodName; +import static org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.SCHEDULER_JOURNAL_URI; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Basic tests of router federation rename. Rename across namespaces. + */ +public class TestRouterFederationRenameInKerberosEnv + extends ClientBaseWithFixes { + + private static final int NUM_SUBCLUSTERS = 2; + private static final int NUM_DNS = 6; + + private static String clientPrincipal = "client@EXAMPLE.COM"; + private static String serverPrincipal = System.getenv().get("USERNAME") + + "/localhost@EXAMPLE.COM"; + private static String keytab = new File( + System.getProperty("test.dir", "target"), + UUID.randomUUID().toString()) + .getAbsolutePath(); + + private static Configuration baseConf = new Configuration(false); + + private static MiniKdc kdc; + + /** Federated HDFS cluster. */ + private MiniRouterDFSCluster cluster; + + /** Random Router for this federated cluster. */ + private RouterContext router; + + @BeforeClass + public static void globalSetUp() throws Exception { + // init KDC + File workDir = new File(System.getProperty("test.dir", "target")); + kdc = new MiniKdc(MiniKdc.createConf(), workDir); + kdc.start(); + kdc.createPrincipal(new File(keytab), clientPrincipal, serverPrincipal); + + + baseConf.setBoolean(DFSConfigKeys.HADOOP_CALLER_CONTEXT_ENABLED_KEY, + true); + + SecurityUtil.setAuthenticationMethod(KERBEROS, baseConf); + baseConf.set(RBFConfigKeys.DFS_ROUTER_KERBEROS_PRINCIPAL_KEY, + serverPrincipal); + baseConf.set(RBFConfigKeys.DFS_ROUTER_KEYTAB_FILE_KEY, keytab); + baseConf.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, + serverPrincipal); + baseConf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, keytab); + baseConf.set(DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, + serverPrincipal); + baseConf.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, keytab); + baseConf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, + true); + + baseConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, + DFS_DATA_TRANSFER_PROTECTION_DEFAULT); + baseConf.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true); + + DistCpProcedure.enableForTest(); + } + + @AfterClass + public static void globalTearDown() { + kdc.stop(); + DistCpProcedure.disableForTest(); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + cluster.shutdown(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + cluster = new MiniRouterDFSCluster(false, NUM_SUBCLUSTERS); + cluster.setNumDatanodesPerNameservice(NUM_DNS); + cluster.addNamenodeOverrides(baseConf); + cluster.setIndependentDNs(); + // Start NNs and DNs and wait until ready. + cluster.startCluster(); + + // Start routers, enable router federation rename. + String journal = "hdfs://" + cluster.getCluster().getNameNode(1) + .getClientNamenodeAddress() + "/journal"; + Configuration routerConf = new RouterConfigBuilder() + .metrics() + .rpc() + .routerRenameOption() + .set(SCHEDULER_JOURNAL_URI, journal) + .set(DFS_ROUTER_FEDERATION_RENAME_MAP, "1") + .set(DFS_ROUTER_FEDERATION_RENAME_BANDWIDTH, "1") + .set(ZK_DTSM_ZK_CONNECTION_STRING, hostPort) + .set(ZK_DTSM_ZK_AUTH_TYPE, "none") + .set(RM_PRINCIPAL, serverPrincipal) + .build(); + // We decrease the DN cache times to make the test faster. + routerConf.setTimeDuration( + RBFConfigKeys.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS); + cluster.addRouterOverrides(baseConf); + cluster.addRouterOverrides(routerConf); + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + // We decrease the DN heartbeat expire interval to make them dead faster + cluster.getCluster().getNamesystem(0).getBlockManager() + .getDatanodeManager().setHeartbeatInterval(1); + cluster.getCluster().getNamesystem(1).getBlockManager() + .getDatanodeManager().setHeartbeatInterval(1); + cluster.getCluster().getNamesystem(0).getBlockManager() + .getDatanodeManager().setHeartbeatExpireInterval(3000); + cluster.getCluster().getNamesystem(1).getBlockManager() + .getDatanodeManager().setHeartbeatExpireInterval(3000); + + // Create mock locations + cluster.installMockLocations(); + + // Create test fixtures on NN + cluster.createTestDirectoriesNamenode(); + + // Random router for this test + RouterContext rndRouter = cluster.getRandomRouter(); + setRouter(rndRouter); + } + + protected void createDir(FileSystem fs, String dir) throws IOException { + fs.mkdirs(new Path(dir)); + String file = dir + "/file"; + createFile(fs, file, 32); + verifyFileExists(fs, dir); + verifyFileExists(fs, file); + } + + protected void testRenameDir(RouterContext testRouter, String path, + String renamedPath, boolean exceptionExpected, Callable call) + throws IOException { + createDir(testRouter.getFileSystem(), path); + // rename + boolean exceptionThrown = false; + try { + call.call(); + assertFalse(verifyFileExists(testRouter.getFileSystem(), path)); + assertTrue(verifyFileExists(testRouter.getFileSystem(), + renamedPath + "/file")); + } catch (Exception ex) { + exceptionThrown = true; + assertTrue(verifyFileExists(testRouter.getFileSystem(), + path + "/file")); + assertFalse(verifyFileExists(testRouter.getFileSystem(), renamedPath)); + } finally { + FileContext fileContext = testRouter.getFileContext(); + fileContext.delete(new Path(path), true); + fileContext.delete(new Path(renamedPath), true); + } + if (exceptionExpected) { + // Error was expected. + assertTrue(exceptionThrown); + } else { + // No error was expected. + assertFalse(exceptionThrown); + } + } + + protected void setRouter(RouterContext r) throws IOException { + this.router = r; + } + + @Test + public void testClientRename() throws IOException { + String ns0 = cluster.getNameservices().get(0); + String ns1 = cluster.getNameservices().get(1); + // Test successfully rename a dir to a destination that is in a different + // namespace. + String dir = + cluster.getFederatedTestDirectoryForNS(ns0) + "/" + getMethodName(); + String renamedDir = + cluster.getFederatedTestDirectoryForNS(ns1) + "/" + getMethodName(); + testRenameDir(router, dir, renamedDir, false, () -> { + UserGroupInformation ugi = UserGroupInformation + .loginUserFromKeytabAndReturnUGI(clientPrincipal, keytab); + ugi.doAs((PrivilegedExceptionAction) () -> { + DFSClient client = router.getClient(); + ClientProtocol clientProtocol = client.getNamenode(); + clientProtocol.rename(dir, renamedDir); + return null; + }); + return null; + }); + + } + +} \ No newline at end of file From 7f93349ee74da5f35276b7535781714501ab2457 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sun, 9 May 2021 03:40:08 +0530 Subject: [PATCH 48/48] HADOOP-17644. Add back the exceptions removed by HADOOP-17432 for compatibility. Contributed by Quan Li. --- .../hadoop/security/authentication/util/KerberosUtil.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java index 95f0106a24424..fc6f957b9622e 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java @@ -79,11 +79,16 @@ private static Oid getNumericOidInstance(String oidName) { * * @return Oid instance * @param oidName The oid Name + * @throws ClassNotFoundException for backward compatibility. + * @throws GSSException for backward compatibility. * @throws NoSuchFieldException if the input is not supported. + * @throws IllegalAccessException for backward compatibility. + * */ @Deprecated public static Oid getOidInstance(String oidName) - throws NoSuchFieldException { + throws ClassNotFoundException, GSSException, NoSuchFieldException, + IllegalAccessException { switch (oidName) { case "GSS_SPNEGO_MECH_OID": return GSS_SPNEGO_MECH_OID;

Total capacity{TotalCapacity|fmt_bytes}
Used capacity{UsedCapacity|fmt_bytes}
Remaining capacity{RemainingCapacity|fmt_bytes}
Total capacity{TotalCapacityBigInt|fmt_bytes}
Used capacity{UsedCapacityBigInt|fmt_bytes}
Remaining capacity{RemainingCapacityBigInt|fmt_bytes}
Nameservices{NumNameservices}
Namenodes{NumNamenodes}