From ef157fca50a34eccf920d22d570ac9edd26da4c5 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 3 May 2021 15:34:45 +0900 Subject: [PATCH 1/2] HDFS-15952. TestRouterRpcMultiDestination#testProxyGetTransactionID and testProxyVersionRequest are flaky. --- .../federation/router/TestRouterRpc.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index c3e7f10c95a00..b82652265c300 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists; import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -1305,8 +1306,27 @@ public void testProxyVersionRequest() throws Exception { compareVersion(rVersion, nnVersion); // Check with default namespace unspecified. resolver.setDisableNamespace(true); + + // get version from nn1 + final String ns1 = cluster.getNameservices().get(1); + final NamenodeContext nn1 = cluster.getNamenode(ns1, null); + final NamenodeProtocol nn1NamenodeProtocol = NameNodeProxies.createProxy( + nn1.getConf(), nn1.getFileSystem().getUri(), NamenodeProtocol.class) + .getProxy(); + final NamespaceInfo nn1Version = nn1NamenodeProtocol.versionRequest(); + + // Verify the version is of nn0 or nn1 rVersion = routerNamenodeProtocol.versionRequest(); - compareVersion(rVersion, nnVersion); + assertThat(rVersion.getBlockPoolID()).isIn( + nnVersion.getBlockPoolID(), nn1Version.getBlockPoolID()); + assertThat(rVersion.getNamespaceID()).isIn( + nnVersion.getNamespaceID(), nn1Version.getNamespaceID()); + assertThat(rVersion.getClusterID()).isIn( + nnVersion.getClusterID(), nn1Version.getClusterID()); + assertThat(rVersion.getLayoutVersion()).isIn( + nnVersion.getLayoutVersion(), nn1Version.getLayoutVersion()); + assertThat(rVersion.getCTime()).isIn( + nnVersion.getCTime(), nn1Version.getCTime()); } finally { resolver.setDisableNamespace(false); } @@ -1378,8 +1398,18 @@ public void testProxyGetTransactionID() throws IOException { assertEquals(nnTransactionID, routerTransactionID); // Check with default namespace unspecified. resolver.setDisableNamespace(true); + + // get transaction ID from nn1 + final String ns1 = cluster.getNameservices().get(1); + final NamenodeContext nn1 = cluster.getNamenode(ns1, null); + final NamenodeProtocol nn1NamenodeProtocol = NameNodeProxies.createProxy( + nn1.getConf(), nn1.getFileSystem().getUri(), NamenodeProtocol.class) + .getProxy(); + final long nn1TransactionID = nn1NamenodeProtocol.getTransactionID(); + + // Verify the transaction ID is of nn0 or nn1 routerTransactionID = routerNamenodeProtocol.getTransactionID(); - assertEquals(nnTransactionID, routerTransactionID); + assertThat(routerTransactionID).isIn(nnTransactionID, nn1TransactionID); } finally { resolver.setDisableNamespace(false); } From 88beaa54881b764c894d44624531d62877c875e3 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 3 May 2021 16:07:32 +0900 Subject: [PATCH 2/2] Improve test code based on #2860 --- .../federation/router/TestRouterRpc.java | 44 ++++++------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index b82652265c300..4f772cc4dc89c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -186,6 +186,7 @@ public int compare( private NamenodeProtocol routerNamenodeProtocol; /** NameNodeProtocol interface to the Namenode. */ private NamenodeProtocol nnNamenodeProtocol; + private NamenodeProtocol nnNamenodeProtocol1; /** Filesystem interface to the Router. */ private FileSystem routerFS; @@ -367,6 +368,11 @@ protected void setNamenode(NamenodeContext nn) NamenodeContext nn0 = cluster.getNamenode(ns0, null); this.nnNamenodeProtocol = NameNodeProxies.createProxy(nn0.getConf(), nn0.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); + // Namenode from the other namespace + String ns1 = cluster.getNameservices().get(1); + NamenodeContext nn1 = cluster.getNamenode(ns1, null); + this.nnNamenodeProtocol1 = NameNodeProxies.createProxy(nn1.getConf(), + nn1.getFileSystem().getUri(), NamenodeProtocol.class).getProxy(); } protected String getNs() { @@ -1303,30 +1309,14 @@ public void testProxyVersionRequest() throws Exception { // Check with default namespace specified. NamespaceInfo rVersion = routerNamenodeProtocol.versionRequest(); NamespaceInfo nnVersion = nnNamenodeProtocol.versionRequest(); + NamespaceInfo nnVersion1 = nnNamenodeProtocol1.versionRequest(); compareVersion(rVersion, nnVersion); // Check with default namespace unspecified. resolver.setDisableNamespace(true); - - // get version from nn1 - final String ns1 = cluster.getNameservices().get(1); - final NamenodeContext nn1 = cluster.getNamenode(ns1, null); - final NamenodeProtocol nn1NamenodeProtocol = NameNodeProxies.createProxy( - nn1.getConf(), nn1.getFileSystem().getUri(), NamenodeProtocol.class) - .getProxy(); - final NamespaceInfo nn1Version = nn1NamenodeProtocol.versionRequest(); - - // Verify the version is of nn0 or nn1 - rVersion = routerNamenodeProtocol.versionRequest(); - assertThat(rVersion.getBlockPoolID()).isIn( - nnVersion.getBlockPoolID(), nn1Version.getBlockPoolID()); - assertThat(rVersion.getNamespaceID()).isIn( - nnVersion.getNamespaceID(), nn1Version.getNamespaceID()); - assertThat(rVersion.getClusterID()).isIn( - nnVersion.getClusterID(), nn1Version.getClusterID()); - assertThat(rVersion.getLayoutVersion()).isIn( - nnVersion.getLayoutVersion(), nn1Version.getLayoutVersion()); - assertThat(rVersion.getCTime()).isIn( - nnVersion.getCTime(), nn1Version.getCTime()); + // Verify the NamespaceInfo is of nn0 or nn1 + boolean isNN0 = + rVersion.getBlockPoolID().equals(nnVersion.getBlockPoolID()); + compareVersion(rVersion, isNN0 ? nnVersion : nnVersion1); } finally { resolver.setDisableNamespace(false); } @@ -1395,21 +1385,13 @@ public void testProxyGetTransactionID() throws IOException { // Check with default namespace specified. long routerTransactionID = routerNamenodeProtocol.getTransactionID(); long nnTransactionID = nnNamenodeProtocol.getTransactionID(); + long nnTransactionID1 = nnNamenodeProtocol1.getTransactionID(); assertEquals(nnTransactionID, routerTransactionID); // Check with default namespace unspecified. resolver.setDisableNamespace(true); - - // get transaction ID from nn1 - final String ns1 = cluster.getNameservices().get(1); - final NamenodeContext nn1 = cluster.getNamenode(ns1, null); - final NamenodeProtocol nn1NamenodeProtocol = NameNodeProxies.createProxy( - nn1.getConf(), nn1.getFileSystem().getUri(), NamenodeProtocol.class) - .getProxy(); - final long nn1TransactionID = nn1NamenodeProtocol.getTransactionID(); - // Verify the transaction ID is of nn0 or nn1 routerTransactionID = routerNamenodeProtocol.getTransactionID(); - assertThat(routerTransactionID).isIn(nnTransactionID, nn1TransactionID); + assertThat(routerTransactionID).isIn(nnTransactionID, nnTransactionID1); } finally { resolver.setDisableNamespace(false); }