From 187fe453d034f81252f5829071521ed02725ce0d Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Thu, 18 Aug 2022 15:45:50 +0800 Subject: [PATCH 1/4] HDFS-16732. [SBN READ] Avoid get location from observer when the block report is delayed. --- .../hdfs/server/namenode/FSNamesystem.java | 25 +++ .../ha/TestObserverNodeWhenReportDelay.java | 153 ++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 347fec858675d..7dfcd25302618 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -103,6 +103,7 @@ import org.apache.commons.text.CaseUtils; import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; @@ -3470,6 +3471,16 @@ HdfsFileStatus getFileInfo(final String src, boolean resolveLink, logAuditEvent(false, operationName, src); throw e; } + if (needLocation && haEnabled && haContext != null && + haContext.getState().getServiceState() == OBSERVER && + stat instanceof HdfsLocatedFileStatus) { + LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks(); + for (LocatedBlock b : lbs.getLocatedBlocks()) { + if (b.getLocations() == null || b.getLocations().length == 0) { + throw new ObserverRetryOnActiveException("Zero blocklocations for " + src); + } + } + } logAuditEvent(true, operationName, src); return stat; } @@ -4175,6 +4186,20 @@ DirectoryListing getListing(String src, byte[] startAfter, logAuditEvent(false, operationName, src); throw e; } + if (needLocation && haEnabled && haContext != null && + haContext.getState().getServiceState() == OBSERVER) { + for(HdfsFileStatus fs : dl.getPartialListing()) { + if (fs instanceof HdfsLocatedFileStatus) { + LocatedBlocks bs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks(); + for (LocatedBlock b : bs.getLocatedBlocks()) { + if (b.getLocations() == null || b.getLocations().length == 0) { + throw new ObserverRetryOnActiveException( + "Zero blocklocations for " + fs.toString()); + } + } + } + } + } logAuditEvent(true, operationName, src); return dl; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java new file mode 100644 index 0000000000000..573e8e465b661 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; +import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState; +import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.OBSERVER_PROBE_RETRY_PERIOD_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestObserverNodeWhenReportDelay { + public static final Logger LOG = + LoggerFactory.getLogger(TestObserverNodeWhenReportDelay.class.getName()); + + private final static String CONTENT = "0123456789"; + private final static Path TEST_PATH = new Path("/TestObserverNode"); + + private static Configuration conf; + private static MiniQJMHACluster qjmhaCluster; + private static MiniDFSCluster dfsCluster; + private static DistributedFileSystem dfs; + + @BeforeClass + public static void startUpCluster() throws Exception { + conf = new Configuration(); + conf.setBoolean(DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY, true); + conf.setInt(DFS_BLOCK_SIZE_KEY, 5); + conf.setInt(DFS_BYTES_PER_CHECKSUM_KEY, 5); + conf.setInt(DFS_REPLICATION_KEY, 1); + conf.setInt(OBSERVER_PROBE_RETRY_PERIOD_KEY, -1); + qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1, 2, true); + dfsCluster = qjmhaCluster.getDfsCluster(); + } + + @AfterClass + public static void shutDownCluster() throws IOException { + if (qjmhaCluster != null) { + qjmhaCluster.shutdown(); + } + } + + @Before + public void setUp() throws Exception { + dfs = HATestUtil.configureObserverReadFs(dfsCluster, conf, ObserverReadProxyProvider.class, + true); + } + + @After + public void cleanUp() throws IOException { + dfs.delete(TEST_PATH, true); + assertEquals("NN[0] should be active", HAServiceState.ACTIVE, + getServiceState(dfsCluster.getNameNode(0))); + assertEquals("NN[1] should be standby", HAServiceState.STANDBY, + getServiceState(dfsCluster.getNameNode(1))); + assertEquals("NN[2] should be observer", HAServiceState.OBSERVER, + getServiceState(dfsCluster.getNameNode(2))); + } + + @Test + public void testReadWriteWhenReportDelay() throws Exception { + // 1 Disable block report to observer. + for (DataNode dn : dfsCluster.getDataNodes()) { + Configuration conf = new Configuration(dn.getConf()); + conf.set("dfs.ha.namenodes.ns1", "nn0,nn1"); + dn.refreshNamenodes(conf); + } + + // 2 Create File + FSDataOutputStream out = dfs.create(TEST_PATH); + out.write(CONTENT.getBytes()); + out.close(); + dfsCluster.rollEditLogAndTail(0); + + // 3 Read file + // The block report of observer is delayed, so failover to active + byte[] buf = new byte[CONTENT.length()]; + FSDataInputStream in = dfs.open(TEST_PATH); + in.readFully(buf); + in.close(); + Assert.assertEquals(CONTENT, new String(buf)); + assertSentTo(0); + + // 4 List path with location + // the block report of observer is delayed, so failover to active. + DirectoryListing listing = dfs.getClient().listPaths("/", new byte[0], true); + assertSentTo(0); + Assert.assertEquals(1, listing.getPartialListing().length); + LocatedBlocks lbs =((HdfsLocatedFileStatus) listing.getPartialListing()[0]).getLocatedBlocks(); + List blocks = lbs.getLocatedBlocks(); + Assert.assertEquals(2, blocks.size()); + Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(0).getLocations())); + Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(1).getLocations())); + + // 5 Get located file info with location + // the block report of observer is delayed, so failover to active. + HdfsLocatedFileStatus status = dfs.getClient().getLocatedFileInfo(TEST_PATH.toString(), false); + assertSentTo(0); + blocks = status.getLocatedBlocks().getLocatedBlocks(); + Assert.assertEquals(2, blocks.size()); + Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(0).getLocations())); + Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(1).getLocations())); + } + + + private void assertSentTo(int nnIdx) throws IOException { + assertTrue("Request was not sent to the expected namenode " + nnIdx, + HATestUtil.isSentToAnyOfNameNodes(dfs, dfsCluster, nnIdx)); + } +} From c54f61799d318ee7b22e0c159576c054dbdd66e8 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Fri, 19 Aug 2022 12:24:09 +0800 Subject: [PATCH 2/4] fix code style --- .../server/namenode/ha/TestObserverNodeWhenReportDelay.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java index 573e8e465b661..594348ca045e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java @@ -104,9 +104,9 @@ public void cleanUp() throws IOException { public void testReadWriteWhenReportDelay() throws Exception { // 1 Disable block report to observer. for (DataNode dn : dfsCluster.getDataNodes()) { - Configuration conf = new Configuration(dn.getConf()); - conf.set("dfs.ha.namenodes.ns1", "nn0,nn1"); - dn.refreshNamenodes(conf); + Configuration overrideConf = new Configuration(dn.getConf()); + overrideConf.set("dfs.ha.namenodes.ns1", "nn0,nn1"); + dn.refreshNamenodes(overrideConf); } // 2 Create File From 140b9e2fb3d9b12f81ae575a0119c5f9f156af69 Mon Sep 17 00:00:00 2001 From: zhengchenyu001 Date: Sat, 20 Aug 2022 16:32:18 +0800 Subject: [PATCH 3/4] update code structure --- .../hdfs/server/namenode/FSNamesystem.java | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7dfcd25302618..6634745a09b4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2203,14 +2203,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String srcArg, } } } - } else if (haEnabled && haContext != null && - haContext.getState().getServiceState() == OBSERVER) { - for (LocatedBlock b : res.blocks.getLocatedBlocks()) { - if (b.getLocations() == null || b.getLocations().length == 0) { - throw new ObserverRetryOnActiveException("Zero blocklocations " - + "for " + srcArg); - } - } + } else if (isObserver()) { + checkBlockLocationsWhenObserver(res.blocks, srcArg); } } finally { readUnlock(operationName, getLockReportInfoSupplier(srcArg)); @@ -3471,15 +3465,9 @@ HdfsFileStatus getFileInfo(final String src, boolean resolveLink, logAuditEvent(false, operationName, src); throw e; } - if (needLocation && haEnabled && haContext != null && - haContext.getState().getServiceState() == OBSERVER && - stat instanceof HdfsLocatedFileStatus) { + if (needLocation && isObserver() && stat instanceof HdfsLocatedFileStatus) { LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks(); - for (LocatedBlock b : lbs.getLocatedBlocks()) { - if (b.getLocations() == null || b.getLocations().length == 0) { - throw new ObserverRetryOnActiveException("Zero blocklocations for " + src); - } - } + checkBlockLocationsWhenObserver(lbs, src); } logAuditEvent(true, operationName, src); return stat; @@ -4186,17 +4174,11 @@ DirectoryListing getListing(String src, byte[] startAfter, logAuditEvent(false, operationName, src); throw e; } - if (needLocation && haEnabled && haContext != null && - haContext.getState().getServiceState() == OBSERVER) { - for(HdfsFileStatus fs : dl.getPartialListing()) { + if (needLocation && isObserver()) { + for (HdfsFileStatus fs : dl.getPartialListing()) { if (fs instanceof HdfsLocatedFileStatus) { - LocatedBlocks bs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks(); - for (LocatedBlock b : bs.getLocatedBlocks()) { - if (b.getLocations() == null || b.getLocations().length == 0) { - throw new ObserverRetryOnActiveException( - "Zero blocklocations for " + fs.toString()); - } - } + LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks(); + checkBlockLocationsWhenObserver(lbs, fs.toString()); } } } @@ -9045,4 +9027,17 @@ public void checkErasureCodingSupported(String operationName) throw new UnsupportedActionException(operationName + " not supported."); } } + + private boolean isObserver() { + return haEnabled && haContext != null && haContext.getState().getServiceState() == OBSERVER; + } + + private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src) + throws ObserverRetryOnActiveException { + for (LocatedBlock b : blocks.getLocatedBlocks()) { + if (b.getLocations() == null || b.getLocations().length == 0) { + throw new ObserverRetryOnActiveException("Zero blocklocations for " + src); + } + } + } } From bae43f8dba103156707a6d077f13e32dc8a5b551 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Thu, 25 Aug 2022 12:56:51 +0800 Subject: [PATCH 4/4] update the unit test --- .../server/namenode/ha/TestObserverNode.java | 6 + .../ha/TestObserverNodeWhenReportDelay.java | 153 ------------------ 2 files changed, 6 insertions(+), 153 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java index 29cae6f13adbc..a910117194507 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java @@ -369,6 +369,12 @@ public void testObserverNodeBlockMissingRetry() throws Exception { dfs.open(testPath); assertSentTo(0); + dfs.getClient().listPaths("/", new byte[0], true); + assertSentTo(0); + + dfs.getClient().getLocatedFileInfo(testPath.toString(), false); + assertSentTo(0); + Mockito.reset(bmSpy); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java deleted file mode 100644 index 594348ca045e5..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNodeWhenReportDelay.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode.ha; - -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; -import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState; -import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.OBSERVER_PROBE_RETRY_PERIOD_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.lang3.ArrayUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestObserverNodeWhenReportDelay { - public static final Logger LOG = - LoggerFactory.getLogger(TestObserverNodeWhenReportDelay.class.getName()); - - private final static String CONTENT = "0123456789"; - private final static Path TEST_PATH = new Path("/TestObserverNode"); - - private static Configuration conf; - private static MiniQJMHACluster qjmhaCluster; - private static MiniDFSCluster dfsCluster; - private static DistributedFileSystem dfs; - - @BeforeClass - public static void startUpCluster() throws Exception { - conf = new Configuration(); - conf.setBoolean(DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY, true); - conf.setInt(DFS_BLOCK_SIZE_KEY, 5); - conf.setInt(DFS_BYTES_PER_CHECKSUM_KEY, 5); - conf.setInt(DFS_REPLICATION_KEY, 1); - conf.setInt(OBSERVER_PROBE_RETRY_PERIOD_KEY, -1); - qjmhaCluster = HATestUtil.setUpObserverCluster(conf, 1, 2, true); - dfsCluster = qjmhaCluster.getDfsCluster(); - } - - @AfterClass - public static void shutDownCluster() throws IOException { - if (qjmhaCluster != null) { - qjmhaCluster.shutdown(); - } - } - - @Before - public void setUp() throws Exception { - dfs = HATestUtil.configureObserverReadFs(dfsCluster, conf, ObserverReadProxyProvider.class, - true); - } - - @After - public void cleanUp() throws IOException { - dfs.delete(TEST_PATH, true); - assertEquals("NN[0] should be active", HAServiceState.ACTIVE, - getServiceState(dfsCluster.getNameNode(0))); - assertEquals("NN[1] should be standby", HAServiceState.STANDBY, - getServiceState(dfsCluster.getNameNode(1))); - assertEquals("NN[2] should be observer", HAServiceState.OBSERVER, - getServiceState(dfsCluster.getNameNode(2))); - } - - @Test - public void testReadWriteWhenReportDelay() throws Exception { - // 1 Disable block report to observer. - for (DataNode dn : dfsCluster.getDataNodes()) { - Configuration overrideConf = new Configuration(dn.getConf()); - overrideConf.set("dfs.ha.namenodes.ns1", "nn0,nn1"); - dn.refreshNamenodes(overrideConf); - } - - // 2 Create File - FSDataOutputStream out = dfs.create(TEST_PATH); - out.write(CONTENT.getBytes()); - out.close(); - dfsCluster.rollEditLogAndTail(0); - - // 3 Read file - // The block report of observer is delayed, so failover to active - byte[] buf = new byte[CONTENT.length()]; - FSDataInputStream in = dfs.open(TEST_PATH); - in.readFully(buf); - in.close(); - Assert.assertEquals(CONTENT, new String(buf)); - assertSentTo(0); - - // 4 List path with location - // the block report of observer is delayed, so failover to active. - DirectoryListing listing = dfs.getClient().listPaths("/", new byte[0], true); - assertSentTo(0); - Assert.assertEquals(1, listing.getPartialListing().length); - LocatedBlocks lbs =((HdfsLocatedFileStatus) listing.getPartialListing()[0]).getLocatedBlocks(); - List blocks = lbs.getLocatedBlocks(); - Assert.assertEquals(2, blocks.size()); - Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(0).getLocations())); - Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(1).getLocations())); - - // 5 Get located file info with location - // the block report of observer is delayed, so failover to active. - HdfsLocatedFileStatus status = dfs.getClient().getLocatedFileInfo(TEST_PATH.toString(), false); - assertSentTo(0); - blocks = status.getLocatedBlocks().getLocatedBlocks(); - Assert.assertEquals(2, blocks.size()); - Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(0).getLocations())); - Assert.assertTrue(ArrayUtils.isNotEmpty(blocks.get(1).getLocations())); - } - - - private void assertSentTo(int nnIdx) throws IOException { - assertTrue("Request was not sent to the expected namenode " + nnIdx, - HATestUtil.isSentToAnyOfNameNodes(dfs, dfsCluster, nnIdx)); - } -}