test with hadoop 3.2.4

apache · Nov 21, 2023 · 4a1483d · 4a1483d
1 parent 6d7e033
commit 4a1483d
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 30 deletions.
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -934,7 +934,7 @@ object MRClientProjects {
   }
 
   def mrIt: Project = {
-    val hadoopVersion = "3.3.6"
+    val hadoopVersion = "3.2.4"
 
     Project("celeborn-mr-it", file("tests/mr-it"))
       // ref: https://www.scala-sbt.org/1.x/docs/Multi-Project.html#Classpath+dependencies
@@ -948,12 +948,7 @@ object MRClientProjects {
         copyDepsSettings,
         libraryDependencies ++= Seq(
           "org.apache.hadoop" % "hadoop-client-minicluster" % hadoopVersion % "test",
-          "org.apache.hadoop" % "hadoop-mapreduce-examples" % hadoopVersion % "test",
-          "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion,
-          "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion,
-          "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll (
-            ExclusionRule("com.google.guava", "guava")),
-          "org.bouncycastle" % "bcpkix-jdk15on" % "1.68"
+          "org.apache.hadoop" % "hadoop-mapreduce-examples" % hadoopVersion % "test"
         ) ++ commonUnitTestDependencies
       )
   }

diff --git a/tests/mr-it/pom.xml b/tests/mr-it/pom.xml
@@ -28,10 +28,6 @@
   <packaging>jar</packaging>
   <name>Celeborn MapReduce Integration Test</name>
 
-  <properties>
-    <hadoop.version>3.3.6</hadoop.version>
-  </properties>
-
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
@@ -96,6 +92,10 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-yarn-server-nodemanager</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-common</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -120,46 +120,37 @@
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.bouncycastle</groupId>
-      <artifactId>bcpkix-jdk15on</artifactId>
-      <version>1.68</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-api</artifactId>
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-client</artifactId>
-      <version>1.19.4</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-json</artifactId>
-      <version>1.19.4</version>
-      <scope>test</scope>
-    </dependency>
 
     <!-- The compile scope is to generate mapreduce_lib -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>
       <version>${hadoop.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-web-proxy</artifactId>
       <version>${hadoop.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-common</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>provided</scope>
     </dependency>
   </dependencies>
 

diff --git a/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala b/tests/mr-it/src/test/scala/org/apache/celeborn/tests/mr/WordCountTest.scala
@@ -131,8 +131,12 @@ class WordCountTest extends AnyFunSuite with Logging with MiniClusterFeature
 
     val mapreduceLibPath = (Utils.getCodeSourceLocation(getClass).split("/").dropRight(1) ++ Array(
       "mapreduce_lib")).mkString("/")
+    val excludeJarList =
+      Seq("hadoop-client-api", "hadoop-client-runtime", "hadoop-client-minicluster")
     Files.list(Paths.get(mapreduceLibPath)).iterator().asScala.foreach(path => {
-      job.addFileToClassPath(new Path(path.toString))
+      if (!excludeJarList.exists(path.toFile.getPath.contains(_))) {
+        job.addFileToClassPath(new Path(path.toString))
+      }
     })
 
     val exitCode = job.waitForCompletion(true)