From 7c19418230ac99d7ce79d172e2b80d667ba87cfb Mon Sep 17 00:00:00 2001
From: Steve Vaughan Jr <s_vaughan@apple.com>
Date: Thu, 12 May 2022 17:44:15 -0400
Subject: [PATCH 1/4] Add dependencies for parallel builds

Parallel builds failed because the assembly plugin did not understand the required dependencies.
---
 hadoop-mapreduce-project/pom.xml | 41 ++++++++++++++++++++++++++++++++
 hadoop-yarn-project/pom.xml      |  7 ++++++
 2 files changed, 48 insertions(+)

diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index b32f1575636a5..821d1dcfd3af4 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -274,6 +274,47 @@
                 <artifactId>hadoop-assemblies</artifactId>
                 <version>${project.version}</version>
               </dependency>
+              <!-- Add dependencies for parallel builds -->
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-app</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-common</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-core</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-hs</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-examples</artifactId>
+                <version>${project.version}</version>
+              </dependency>
             </dependencies>
             <configuration>
               <tarLongFileMode>gnu</tarLongFileMode>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index abaf2e869c48f..e73a6d732c489 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -142,6 +142,13 @@
                 <artifactId>hadoop-assemblies</artifactId>
                 <version>${project.version}</version>
               </dependency>
+              <!-- Add dependencies for parallel builds -->
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
+                <version>${project.version}</version>
+                <type>war</type>
+              </dependency>
             </dependencies>
             <configuration>
               <tarLongFileMode>gnu</tarLongFileMode>

From 198694ceedf01a2fcc329d8fd3dc2b9cad3f4fab Mon Sep 17 00:00:00 2001
From: Steve Vaughan Jr <s_vaughan@apple.com>
Date: Thu, 16 Jun 2022 21:04:55 -0400
Subject: [PATCH 2/4] Add dependencies necessary to ensure hadoop-dist is last

---
 hadoop-dist/pom.xml    | 10 ++++++++++
 hadoop-project/pom.xml | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 0a5db2565b8c5..0b1c601267358 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -41,11 +41,21 @@
       <artifactId>hadoop-hdfs-client</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-native-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-app</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e37851b6d0786..f60be7535043f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -358,6 +358,11 @@
         <artifactId>hadoop-hdfs-client</artifactId>
         <version>${hadoop.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs-native-client</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs-rbf</artifactId>
@@ -391,6 +396,11 @@
         <artifactId>hadoop-mapreduce-client-common</artifactId>
         <version>${hadoop.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-api</artifactId>

From 79bb860af73f84f268464ce79a28c2bf41347d63 Mon Sep 17 00:00:00 2001
From: Steve Vaughan Jr <s_vaughan@apple.com>
Date: Thu, 16 Jun 2022 21:04:55 -0400
Subject: [PATCH 3/4] Add dependencies necessary to ensure hadoop-dist is last


From ebcd845988f18b68bbfd8f86a33b8d80c134c6db Mon Sep 17 00:00:00 2001
From: Steve Vaughan Jr <s_vaughan@apple.com>
Date: Mon, 15 Aug 2022 17:21:03 -0400
Subject: [PATCH 4/4] Update mapreduce to follow yarn pattern

Use dependencies in the mapreduce project for the purpose of assembly plugin and packaging.  This moves all assembly dependencies added for parallel builds up to the module.
---
 hadoop-mapreduce-project/pom.xml | 156 +++++--------------------------
 hadoop-yarn-project/pom.xml      |  12 +--
 2 files changed, 29 insertions(+), 139 deletions(-)

diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 821d1dcfd3af4..21554090d7855 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -41,118 +41,51 @@
     <module>hadoop-mapreduce-examples</module>
   </modules>
 
+  <!--
+  Do not add 3rd party dependencies here, add them to the POM of the leaf module.
+  The dependencies in this module are for the assembly plugin, packaging purposes.
+  -->
   <dependencies>
-    <dependency>
-      <groupId>com.google.protobuf</groupId>
-      <artifactId>protobuf-java</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-server</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.ant</groupId>
-          <artifactId>ant</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.velocity</groupId>
-          <artifactId>velocity</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <artifactId>paranamer-ant</artifactId>
-          <groupId>com.thoughtworks.paranamer</groupId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.xerial.snappy</groupId>
-          <artifactId>snappy-java</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-     <groupId>org.slf4j</groupId>
-       <artifactId>slf4j-api</artifactId>
-    </dependency>
-    <dependency>
-     <groupId>org.slf4j</groupId>
-       <artifactId>slf4j-log4j12</artifactId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
-      <scope>test</scope>
+      <artifactId>hadoop-mapreduce-client-common</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <type>test-jar</type>
-      <scope>test</scope>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.google.inject</groupId>
-      <artifactId>guice</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-server</artifactId>
+      <artifactId>hadoop-mapreduce-client-hs</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey.contribs</groupId>
-      <artifactId>jersey-guice</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.google.inject.extensions</groupId>
-      <artifactId>guice-servlet</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.hsqldb</groupId>
-      <artifactId>hsqldb</artifactId>
-      <scope>compile</scope>
-      <classifier>jdk8</classifier>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>${leveldbjni.group}</groupId>
-      <artifactId>leveldbjni-all</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-examples</artifactId>
+      <version>${project.version}</version>
     </dependency>
-
   </dependencies>
 
   <build>
@@ -274,47 +207,6 @@
                 <artifactId>hadoop-assemblies</artifactId>
                 <version>${project.version}</version>
               </dependency>
-              <!-- Add dependencies for parallel builds -->
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-app</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-common</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-core</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-hs</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-mapreduce-examples</artifactId>
-                <version>${project.version}</version>
-              </dependency>
             </dependencies>
             <configuration>
               <tarLongFileMode>gnu</tarLongFileMode>
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index e73a6d732c489..298fa597e9940 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -81,6 +81,11 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-services-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
+      <type>war</type>
+    </dependency>
   </dependencies>
 
   <build>
@@ -142,13 +147,6 @@
                 <artifactId>hadoop-assemblies</artifactId>
                 <version>${project.version}</version>
               </dependency>
-              <!-- Add dependencies for parallel builds -->
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
-                <version>${project.version}</version>
-                <type>war</type>
-              </dependency>
             </dependencies>
             <configuration>
               <tarLongFileMode>gnu</tarLongFileMode>