[KYUUBI #1769][FOLLOWUP] Exclude log4j 12 deps from spark-hive and fix detect deps change workflow

.github/workflows/style.yml

@@ -28,7 +28,7 @@ on:
 
 jobs:
   linter:
-    name: Style and Dep check
+    name: Style and Dependency check
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -41,12 +41,17 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-      - name: Server dependency check
-        run: build/dependency.sh
       - name: Scalastyle with Maven
         run: build/mvn scalastyle:check ${{ matrix.profiles }}
       - name: Upload scalastyle report
         if: failure()
         run: for log in `find * -name "scalastyle-output.xml"`;  do echo "=========$log========="; grep "error" $log; done
       - name: JavaStyle with Maven
         run: build/mvn spotless:check ${{ matrix.profiles }}
+      - name: Check dependency list
+        run: >-
+          build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
+              -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \
+              -Pflink-provided,spark-provided \
+              -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+          build/dependency.sh

build/dependency.sh

@@ -24,15 +24,15 @@ export LC_ALL=C
 
 PWD=$(cd "$(dirname "$0")"/.. || exit; pwd)
 
-MVN="${PWD}"/build/mvn
+MVN="${PWD}/build/mvn"
 
 
-DEP_PR="${PWD}"/dev/dependencyList.tmp
-DEP="${PWD}"/dev/dependencyList
+DEP_PR="${PWD}/dev/dependencyList.tmp"
+DEP="${PWD}/dev/dependencyList"
 
 
 function build_classpath() {
-  $MVN dependency:build-classpath -pl :kyuubi-ctl_2.12,:kyuubi-server_2.12,:kyuubi-assembly_2.12 |\
+  $MVN dependency:build-classpath -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly |\
     grep -v "INFO\|WARN" | \
     tail -1 | \
     tr ":" "\n" | \
@@ -49,12 +49,12 @@ function build_classpath() {
 
 function check_diff() {
     set +e
-    the_diff=$(diff ${DEP} ${DEP_PR})
+    the_diff=$(diff "${DEP}" "${DEP_PR}")
     set -e
     rm -rf "${DEP_PR}"
-    if [[ -n $the_diff ]]; then
+    if [[ -n "${the_diff}" ]]; then
         echo "Dependency List Changed Detected: "
-        echo ${the_diff}
+        echo "${the_diff}"
         echo "To update the dependency file, run './build/dependency.sh --replace'."
         exit 1
     fi

dev/dependencyList

@@ -94,7 +94,6 @@ simpleclient_tracer_common/0.14.1//simpleclient_tracer_common-0.14.1.jar
 simpleclient_tracer_otel/0.14.1//simpleclient_tracer_otel-0.14.1.jar
 simpleclient_tracer_otel_agent/0.14.1//simpleclient_tracer_otel_agent-0.14.1.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
-slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.27//snakeyaml-1.27.jar
 swagger-annotations/2.1.11//swagger-annotations-2.1.11.jar
 swagger-core/2.1.11//swagger-core-2.1.11.jar

dev/kyuubi-extension-spark-3-2/pom.xml

@@ -98,12 +98,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.binary.version}</artifactId>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client-runtime</artifactId>

dev/kyuubi-extension-spark-common/pom.xml

@@ -84,12 +84,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.binary.version}</artifactId>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client-runtime</artifactId>

pom.xml

@@ -347,9 +347,23 @@
                 <artifactId>spark-repl_${scala.binary.version}</artifactId>
                 <version>${spark.version}</version>
                 <exclusions>
+                    <!--
+                      Use Hadoop Shaded Client to gain more clean transitive dependencies
+                     -->
                     <exclusion>
-                        <groupId>*</groupId>
-                        <artifactId>*</artifactId>
+                        <groupId>org.apache.hadoop</groupId>
+                        <artifactId>hadoop-client</artifactId>
+                    </exclusion>
+                    <!--
+                      Use log4j2
+                     -->
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
                     </exclusion>
                 </exclusions>
             </dependency>
@@ -366,6 +380,9 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
+                    <!--
+                      Use log4j2
+                     -->
                     <exclusion>
                         <groupId>log4j</groupId>
                         <artifactId>log4j</artifactId>
@@ -393,6 +410,17 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-common</artifactId>
                     </exclusion>
+                    <!--
+                      Use log4j2
+                     -->
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -409,6 +437,10 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
+                    <!--
+                      The module is only used in Kyuubi Spark Extensions, so we don't care about which
+                      version of Log4j it depends on.
+                     -->
                 </exclusions>
             </dependency>
 
@@ -425,16 +457,13 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
+                    <!--
+                      The module is only used in Kyuubi Spark Extensions, so we don't care about which
+                      version of Log4j it depends on.
+                     -->
                 </exclusions>
             </dependency>
 
-            <dependency>
-                <groupId>org.scalatestplus</groupId>
-                <artifactId>scalacheck-1-15_${scala.binary.version}</artifactId>
-                <version>${scalacheck.version}</version>
-                <scope>test</scope>
-            </dependency>
-
             <dependency>
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-sql_${scala.binary.version}</artifactId>
@@ -448,6 +477,10 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
+                    <!--
+                      The module is only used in Kyuubi Spark Extensions, so we don't care about which
+                      version of Log4j it depends on.
+                     -->
                 </exclusions>
             </dependency>
 
@@ -568,18 +601,6 @@
                 <version>${slf4j.version}</version>
             </dependency>
 
-            <dependency>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-log4j12</artifactId>
-                <version>${slf4j.version}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>log4j</groupId>
-                        <artifactId>log4j</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>jcl-over-slf4j</artifactId>
@@ -775,6 +796,13 @@
                 <version>${scalatest.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.scalatestplus</groupId>
+                <artifactId>scalacheck-1-15_${scala.binary.version}</artifactId>
+                <version>${scalacheck.version}</version>
+                <scope>test</scope>
+            </dependency>
+
             <dependency>
                 <groupId>org.apache.hadoop</groupId>
                 <artifactId>hadoop-minikdc</artifactId>