From baffc921dae31bca6891557c2325a63a44f13e23 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 01:37:06 +0900 Subject: [PATCH 1/7] Test: Remove JUnit API from WebServicesTestUtils to fix MAPREDUCE-7428 --- .../yarn/webapp/WebServicesTestUtils.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java index f803c5313ca57..ce93b06e70ade 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java @@ -27,8 +27,7 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; public class WebServicesTestUtils { public static long getXmlLong(Element element, String name) { @@ -121,28 +120,24 @@ public static String getXmlAttrString(Element element, String name) { } public static void checkStringMatch(String print, String expected, String got) { - assertTrue( - got.matches(expected), - print + " doesn't match, got: " + got + " expected: " + expected); + assertThat(got).as(print).matches(expected); } public static void checkStringContains(String print, String expected, String got) { - assertTrue( - got.contains(expected), - print + " doesn't contain expected string, got: " + got + " expected: " + expected); + assertThat(got).as(print).contains(expected); } public static void checkStringEqual(String print, String expected, String got) { - assertEquals(got, expected); + assertThat(got).as(print).isEqualTo(expected); } public static void assertResponseStatusCode(StatusType expected, StatusType actual) { - assertResponseStatusCode(null, expected, actual); + assertThat(expected.getStatusCode()).isEqualTo(actual.getStatusCode()); } public static void assertResponseStatusCode(String errmsg, StatusType expected, StatusType actual) { - assertEquals(expected.getStatusCode(), actual.getStatusCode(), errmsg); + assertThat(expected.getStatusCode()).withFailMessage(errmsg).isEqualTo(actual.getStatusCode()); } } From f7e550d0afc3ef2400c54a0d6c0649b88c1863de Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 01:39:38 +0900 Subject: [PATCH 2/7] Add changes to run tests in hadoop-mapreduce-client-app and hadoop-mapreduce-client-hs --- .../hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java | 1 + .../hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java | 1 + 2 files changed, 2 insertions(+) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java index 49187a016c32d..4d0f66f526122 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java @@ -66,6 +66,7 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; + /** * Test the app master web service Rest API for getting tasks, a specific task, * and task counters. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index 47329cc39f85d..48f6842469a76 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -72,6 +72,7 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; + /** * Test the history server Rest API for getting tasks, a specific task, * and task counters. From 31773beb41813ca118d4697deea1dbea2eb2c90f Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 01:39:46 +0900 Subject: [PATCH 3/7] Revert "Add changes to run tests in hadoop-mapreduce-client-app and hadoop-mapreduce-client-hs" This reverts commit f7e550d0afc3ef2400c54a0d6c0649b88c1863de. --- .../hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java | 1 - .../hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java index 4d0f66f526122..49187a016c32d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java @@ -66,7 +66,6 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; - /** * Test the app master web service Rest API for getting tasks, a specific task, * and task counters. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index 48f6842469a76..47329cc39f85d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -72,7 +72,6 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; - /** * Test the history server Rest API for getting tasks, a specific task, * and task counters. From 28898725d82be387a560df7972a3709b6b225f53 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 01:48:06 +0900 Subject: [PATCH 4/7] Revert "MAPREDUCE-7428. Fix failures related to Junit 4 to Junit 5 upgrade in org.apache.hadoop.mapreduce.v2.app.webapp (#5209)" This reverts commit 85ec7969a7d7fe2168f1fceb8bce1f6388a8d46d. --- .../hadoop-mapreduce-client-app/pom.xml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index ff268cbd049dd..e3b3511c0ce17 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -124,26 +124,6 @@ assertj-core test - - org.junit.platform - junit-platform-launcher - test - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.platform - junit-platform-launcher - test - From 75f96049c2682771ff98065bb87eacfb5141af69 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 08:04:23 +0900 Subject: [PATCH 5/7] Remove JUnit API from TestContainerLogsUtils --- .../yarn/logaggregation/TestContainerLogsUtils.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java index cce8a62ba1e00..d18e0d85e8c78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java @@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * This class contains several utility functions for log aggregation tests. */ @@ -75,13 +73,12 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } - assertTrue(fs.mkdirs(rootLogDirPath)); + fs.mkdirs(rootLogDirPath); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } - assertTrue(fs.mkdirs(appLogsDir)); - + fs.mkdirs(appLogsDir); createContainerLogInLocalDir(appLogsDir, containerToContent, fs, fileName); // upload container logs to remote log dir @@ -95,7 +92,7 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(path) && deleteRemoteLogDir) { fs.delete(path, true); } - assertTrue(fs.mkdirs(path)); + fs.mkdirs(path); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId, appId, containerToContent.keySet(), path); } @@ -111,7 +108,7 @@ private static void createContainerLogInLocalDir(Path appLogsDir, if (fs.exists(containerLogsDir)) { fs.delete(containerLogsDir, true); } - assertTrue(fs.mkdirs(containerLogsDir)); + fs.mkdirs(containerLogsDir); Writer writer = new FileWriter(new File(containerLogsDir.toString(), fileName)); writer.write(content); From e1269ae0dda68fbd12b1c5b51fe5976478e0a885 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 20 Dec 2022 21:21:51 +0900 Subject: [PATCH 6/7] Add check --- .../yarn/logaggregation/TestContainerLogsUtils.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java index d18e0d85e8c78..ebaaaf7bee631 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java @@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import static org.assertj.core.api.Assertions.assertThat; + /** * This class contains several utility functions for log aggregation tests. */ @@ -73,12 +75,12 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } - fs.mkdirs(rootLogDirPath); + assertThat(fs.mkdirs(rootLogDirPath)).isTrue(); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } - fs.mkdirs(appLogsDir); + assertThat(fs.mkdirs(appLogsDir)).isTrue(); createContainerLogInLocalDir(appLogsDir, containerToContent, fs, fileName); // upload container logs to remote log dir @@ -92,7 +94,7 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(path) && deleteRemoteLogDir) { fs.delete(path, true); } - fs.mkdirs(path); + assertThat(fs.mkdirs(path)).isTrue(); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId, appId, containerToContent.keySet(), path); } @@ -108,7 +110,7 @@ private static void createContainerLogInLocalDir(Path appLogsDir, if (fs.exists(containerLogsDir)) { fs.delete(containerLogsDir, true); } - fs.mkdirs(containerLogsDir); + assertThat(fs.mkdirs(containerLogsDir)).isTrue(); Writer writer = new FileWriter(new File(containerLogsDir.toString(), fileName)); writer.write(content); From ff6d5fa24b22cf1e5a8efc58c85084982a35d8a3 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Fri, 23 Dec 2022 18:34:13 +0900 Subject: [PATCH 7/7] Remove AssertJ libraries --- .../TestContainerLogsUtils.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java index ebaaaf7bee631..0cb71b59909e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java @@ -38,10 +38,10 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; -import static org.assertj.core.api.Assertions.assertThat; - /** * This class contains several utility functions for log aggregation tests. + * Any assertion libraries shouldn't be used here because this class is used by + * multiple modules including MapReduce. */ public final class TestContainerLogsUtils { @@ -75,12 +75,16 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } - assertThat(fs.mkdirs(rootLogDirPath)).isTrue(); + fs.mkdirs(rootLogDirPath); + // Make sure the target dir is created. If not, FileNotFoundException is thrown + fs.getFileStatus(rootLogDirPath); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } - assertThat(fs.mkdirs(appLogsDir)).isTrue(); + fs.mkdirs(appLogsDir); + // Make sure the target dir is created. If not, FileNotFoundException is thrown + fs.getFileStatus(appLogsDir); createContainerLogInLocalDir(appLogsDir, containerToContent, fs, fileName); // upload container logs to remote log dir @@ -94,7 +98,9 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, if (fs.exists(path) && deleteRemoteLogDir) { fs.delete(path, true); } - assertThat(fs.mkdirs(path)).isTrue(); + fs.mkdirs(path); + // Make sure the target dir is created. If not, FileNotFoundException is thrown + fs.getFileStatus(path); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId, appId, containerToContent.keySet(), path); } @@ -110,7 +116,9 @@ private static void createContainerLogInLocalDir(Path appLogsDir, if (fs.exists(containerLogsDir)) { fs.delete(containerLogsDir, true); } - assertThat(fs.mkdirs(containerLogsDir)).isTrue(); + fs.mkdirs(containerLogsDir); + // Make sure the target dir is created. If not, FileNotFoundException is thrown + fs.getFileStatus(containerLogsDir); Writer writer = new FileWriter(new File(containerLogsDir.toString(), fileName)); writer.write(content);