From 4456fc150a1ac0da6b8b2501976772311fefdb55 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Fri, 24 Sep 2021 09:49:22 -0700
Subject: [PATCH 1/3] first commit

---
 launcher/pom.xml | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/launcher/pom.xml b/launcher/pom.xml
index f72f696fba2d..f312d6cdd073 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -85,14 +85,33 @@
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
+  <profiles>
+    <profile>
+      <id>hadoop-2.7</id>
+    </profile>
+    <profile>
+      <id>hadoop-3.2</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <!--
+        Only declare for Hadoop 3.2 profile. Otherwise maven-shade-plugin may stuck in an infinite
+        loop building the dependency-reduced pom, perhaps due to a maven bug:
+          https://issues.apache.org/jira/browse/MSHADE-148
+      -->
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>

From d73562ed3635bb3454ac67029ca6541b30ae0c02 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Fri, 24 Sep 2021 15:29:43 -0700
Subject: [PATCH 2/3] use different names

---
 launcher/pom.xml               | 31 ++++++-------------------------
 pom.xml                        |  2 +-
 resource-managers/yarn/pom.xml | 24 ++++++++++++------------
 3 files changed, 19 insertions(+), 38 deletions(-)

diff --git a/launcher/pom.xml b/launcher/pom.xml
index f312d6cdd073..f72f696fba2d 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -85,33 +85,14 @@
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
-  <profiles>
-    <profile>
-      <id>hadoop-2.7</id>
-    </profile>
-    <profile>
-      <id>hadoop-3.2</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <!--
-        Only declare for Hadoop 3.2 profile. Otherwise maven-shade-plugin may stuck in an infinite
-        loop building the dependency-reduced pom, perhaps due to a maven bug:
-          https://issues.apache.org/jira/browse/MSHADE-148
-      -->
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>${hadoop-client-runtime.artifact}</artifactId>
-          <version>${hadoop.version}</version>
-          <scope>test</scope>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
-
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
diff --git a/pom.xml b/pom.xml
index 2cece056b1c8..4e895f805406 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3273,7 +3273,7 @@
         <curator.version>2.7.1</curator.version>
         <commons-io.version>2.4</commons-io.version>
         <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
-        <hadoop-client-runtime.artifact>hadoop-client</hadoop-client-runtime.artifact>
+        <hadoop-client-runtime.artifact>hadoop-yarn-api</hadoop-client-runtime.artifact>
         <hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
       </properties>
     </profile>
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index d6f7eabf988d..12a600fa0e96 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -82,6 +82,18 @@
         <activeByDefault>true</activeByDefault>
       </activation>
       <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>${hadoop-client-runtime.artifact}</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>${hadoop.deps.scope}</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>test</scope>
+        </dependency>
         <!-- Used by MiniYARNCluster -->
         <dependency>
           <groupId>org.bouncycastle</groupId>
@@ -127,18 +139,6 @@
       <artifactId>${hadoop-client-api.artifact}</artifactId>
       <version>${hadoop.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>${hadoop-client-runtime.artifact}</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>${hadoop.deps.scope}</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
 
     <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
     <dependency>

From 0c358b34a14c59158bff018777388605abf42dc3 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Fri, 24 Sep 2021 17:57:39 -0700
Subject: [PATCH 3/3] add more comments

---
 pom.xml | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4e895f805406..bd02fd25a75c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -249,9 +249,22 @@
     <parquet.test.deps.scope>test</parquet.test.deps.scope>
 
     <!--
-      These default to Hadoop 3.x shaded client/minicluster jars, but are switched to hadoop-client
-      when the Hadoop profile is hadoop-2.7, because these are only available in 3.x. Note that,
-      as result we have to include the same hadoop-client dependency multiple times in hadoop-2.7.
+      These default to Hadoop 3.x shaded client/minicluster jars, but since the shaded jars are
+      only available in 3.x, we have to switch to non-shaded Hadoop client jars when the active
+      profile is hadoop-2.7.
+
+      To make the above work, in the hadoop-2.7 profile section we bind hadoop-client-api to
+      hadoop-client and hadoop-client-runtime to hadoop-yarn-api, which is a dependency of the
+      former. This effectively moves the hadoop-yarn-api to the same level as hadoop-client, but
+      should be fine since both dependencies are required. We cannot use hadoop-client for both
+      of these because the maven enforcer plugin bans duplicate dependencies.
+
+      We still leave hadoop-client-minicluster to use hadoop-client because it is of test scope,
+      and is only used by spark-yarn module. To avoid the duplicate dependency issue we only
+      declare the dependency when we're using hadoop-3.2 profile, in
+      resource-managers/yarn/pom.xml.
+
+      Please check SPARK-36835 for more details.
     -->
     <hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
     <hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
@@ -3272,6 +3285,9 @@
         <hadoop.version>2.7.4</hadoop.version>
         <curator.version>2.7.1</curator.version>
         <commons-io.version>2.4</commons-io.version>
+        <!--
+          the declaration site above of these variables explains why we need to re-assign them here
+        -->
         <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
         <hadoop-client-runtime.artifact>hadoop-yarn-api</hadoop-client-runtime.artifact>
         <hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>