From a8bcbe1b0c93d66cabc862a6e57bdd4b68915f36 Mon Sep 17 00:00:00 2001 From: Kewen Wang Date: Mon, 1 Dec 2025 11:52:27 -0800 Subject: [PATCH] Support Spark2 and Spark3 by subdirs in module presto-spark-classloader-interface Delete two modules presto-spark-classloader-spark2 and presto-spark-classloader-spark3 in the previous approach to support both Spark2 and Spark3. Move the previous implementations into subdirs into existing module presto-spark-classloader-interface. Clean up dependencies in pom files of root, presto-spark-base and presto-spark-launcher. --- pom.xml | 147 +----------------- presto-spark-base/pom.xml | 35 ++--- presto-spark-classloader-interface/pom.xml | 99 ++++++++++-- .../MutablePartitionId.java | 0 .../MutablePartitionIdOrdering.java | 0 ...stoSparkNativeExecutionShuffleManager.java | 0 .../PrestoSparkUtils.java | 0 .../classloader_interface/ScalaUtils.java | 0 .../MutablePartitionId.java | 0 .../MutablePartitionIdOrdering.java | 0 ...stoSparkNativeExecutionShuffleManager.java | 0 .../PrestoSparkUtils.java | 0 .../classloader_interface/ScalaUtils.java | 0 presto-spark-classloader-spark2/pom.xml | 32 ---- presto-spark-classloader-spark3/pom.xml | 39 ----- presto-spark-launcher/pom.xml | 72 +++++---- 16 files changed, 142 insertions(+), 282 deletions(-) rename {presto-spark-classloader-spark2/src/main/java => presto-spark-classloader-interface/src/main/java-spark2}/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java (100%) rename {presto-spark-classloader-spark2/src/main/java => presto-spark-classloader-interface/src/main/java-spark2}/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java (100%) rename {presto-spark-classloader-spark2/src/main/java => presto-spark-classloader-interface/src/main/java-spark2}/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java (100%) rename {presto-spark-classloader-spark2/src/main/java => presto-spark-classloader-interface/src/main/java-spark2}/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java (100%) rename {presto-spark-classloader-spark2/src/main/java => presto-spark-classloader-interface/src/main/java-spark2}/com/facebook/presto/spark/classloader_interface/ScalaUtils.java (100%) rename {presto-spark-classloader-spark3/src/main/java => presto-spark-classloader-interface/src/main/java-spark3}/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java (100%) rename {presto-spark-classloader-spark3/src/main/java => presto-spark-classloader-interface/src/main/java-spark3}/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java (100%) rename {presto-spark-classloader-spark3/src/main/java => presto-spark-classloader-interface/src/main/java-spark3}/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java (100%) rename {presto-spark-classloader-spark3/src/main/java => presto-spark-classloader-interface/src/main/java-spark3}/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java (100%) rename {presto-spark-classloader-spark3/src/main/java => presto-spark-classloader-interface/src/main/java-spark3}/com/facebook/presto/spark/classloader_interface/ScalaUtils.java (100%) delete mode 100644 presto-spark-classloader-spark2/pom.xml delete mode 100644 presto-spark-classloader-spark3/pom.xml diff --git a/pom.xml b/pom.xml index 84545e0df8373..2d896617e0254 100644 --- a/pom.xml +++ b/pom.xml @@ -2844,6 +2844,7 @@ + true @@ -2863,6 +2864,7 @@ org.basepom.maven duplicate-finder-maven-plugin + true com.github.benmanes.caffeine.* @@ -2982,6 +2984,7 @@ + @@ -3223,149 +3226,5 @@ - - spark2 - - - true - - !spark-version - - - - - 2.0.2-6 - - - - presto-spark-classloader-spark2 - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.3.0 - - - org.codehaus.mojo - extra-enforcer-rules - 1.11.0 - - - - true - - - - - org.codehaus.plexus:plexus-utils - com.google.guava:guava - com.fasterxml.jackson.core:jackson-annotations - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - - - - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - true - - com.github.benmanes.caffeine.* - - META-INF.versions.9.module-info - - META-INF.versions.11.module-info - - META-INF.versions.9.org.apache.lucene.* - - - - - - - - - - - - spark3 - - - - spark-version - 3 - - - - - 3.4.1-1 - - - - presto-spark-classloader-spark3 - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.3.0 - - - org.codehaus.mojo - extra-enforcer-rules - 1.11.0 - - - - true - - - - - org.codehaus.plexus:plexus-utils - com.google.guava:guava - com.fasterxml.jackson.core:jackson-annotations - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - - - - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - true - - com.github.benmanes.caffeine.* - - META-INF.versions.9.module-info - - META-INF.versions.11.module-info - - META-INF.versions.9.org.apache.lucene.* - - - - - - - - - - diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index b4fd0b206c38e..d5f390ca35986 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -467,6 +467,12 @@ javax.annotation:javax.annotation-api org.apache.httpcomponents:httpcore + + org.glassfish.hk2.external:jakarta.inject + org.apache.hadoop:hadoop-client-api + jakarta.annotation:jakarta.annotation-api + jakarta.validation:jakarta.validation-api + @@ -540,14 +546,6 @@ !spark-version - - - - com.facebook.presto - presto-spark-classloader-spark2 - ${project.version} - - @@ -560,12 +558,11 @@ + + 3 + + - - com.facebook.presto - presto-spark-classloader-spark3 - ${project.version} - com.facebook.presto.spark @@ -614,18 +611,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - - org.glassfish.hk2.external:jakarta.inject - org.apache.hadoop:hadoop-client-api - jakarta.annotation:jakarta.annotation-api - jakarta.validation:jakarta.validation-api - - - org.apache.maven.plugins diff --git a/presto-spark-classloader-interface/pom.xml b/presto-spark-classloader-interface/pom.xml index 8ff7dd7d2e464..097ff6f25194d 100644 --- a/presto-spark-classloader-interface/pom.xml +++ b/presto-spark-classloader-interface/pom.xml @@ -13,6 +13,7 @@ ${project.parent.basedir} true + 2 @@ -33,6 +34,81 @@ + + target/java-spark${spark.version.main}/classes + target/java-spark${spark.version.main}/test-classes + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + + + org.codehaus.mojo + extra-enforcer-rules + 1.6.2 + + + + true + + + + + org.codehaus.plexus:plexus-utils + com.google.guava:guava + com.fasterxml.jackson.core:jackson-annotations + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + + true + + com.github.benmanes.caffeine.* + + META-INF.versions.9.module-info + + META-INF.versions.11.module-info + + META-INF.versions.9.org.apache.lucene.* + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-sources + generate-sources + + add-source + + + + src/main/java-spark${spark.version.main} + + + + + + + + spark2 @@ -44,15 +120,12 @@ - - - com.facebook.presto - presto-spark-classloader-spark2 - ${project.version} - - - + + 2 + 2.0.2-6 + + spark3 @@ -63,12 +136,12 @@ + + 3 + 3.4.1-1 + + - - com.facebook.presto - presto-spark-classloader-spark3 - ${project.version} - org.scala-lang scala-library diff --git a/presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java b/presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java similarity index 100% rename from presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java rename to presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java diff --git a/presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java b/presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java similarity index 100% rename from presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java rename to presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java diff --git a/presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java b/presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java similarity index 100% rename from presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java rename to presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java diff --git a/presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java b/presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java similarity index 100% rename from presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java rename to presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java diff --git a/presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/ScalaUtils.java b/presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/ScalaUtils.java similarity index 100% rename from presto-spark-classloader-spark2/src/main/java/com/facebook/presto/spark/classloader_interface/ScalaUtils.java rename to presto-spark-classloader-interface/src/main/java-spark2/com/facebook/presto/spark/classloader_interface/ScalaUtils.java diff --git a/presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java b/presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java similarity index 100% rename from presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java rename to presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/MutablePartitionId.java diff --git a/presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java b/presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java similarity index 100% rename from presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java rename to presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/MutablePartitionIdOrdering.java diff --git a/presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java b/presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java similarity index 100% rename from presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java rename to presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/PrestoSparkNativeExecutionShuffleManager.java diff --git a/presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java b/presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java similarity index 100% rename from presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java rename to presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/PrestoSparkUtils.java diff --git a/presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/ScalaUtils.java b/presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/ScalaUtils.java similarity index 100% rename from presto-spark-classloader-spark3/src/main/java/com/facebook/presto/spark/classloader_interface/ScalaUtils.java rename to presto-spark-classloader-interface/src/main/java-spark3/com/facebook/presto/spark/classloader_interface/ScalaUtils.java diff --git a/presto-spark-classloader-spark2/pom.xml b/presto-spark-classloader-spark2/pom.xml deleted file mode 100644 index 0abd6b7178d44..0000000000000 --- a/presto-spark-classloader-spark2/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - presto-root - com.facebook.presto - 0.297-SNAPSHOT - - 4.0.0 - - presto-spark-classloader-spark2 - presto-spark-classloader-spark2 - - - ${project.parent.basedir} - - - - - - com.facebook.presto.spark - spark-core - provided - - - - com.google.guava - guava - - - - - \ No newline at end of file diff --git a/presto-spark-classloader-spark3/pom.xml b/presto-spark-classloader-spark3/pom.xml deleted file mode 100644 index aba2bf8725868..0000000000000 --- a/presto-spark-classloader-spark3/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - presto-root - com.facebook.presto - 0.296-SNAPSHOT - - 4.0.0 - - presto-spark-classloader-spark3 - presto-spark-classloader-spark3 - - - ${project.parent.basedir} - - - - - - com.google.guava - guava - - - - com.facebook.presto.spark - spark-core - 3.4.1-1 - compile - - - - org.scala-lang - scala-library - provided - - - - - \ No newline at end of file diff --git a/presto-spark-launcher/pom.xml b/presto-spark-launcher/pom.xml index 0cdc2ad84bade..87c8c11f71c83 100644 --- a/presto-spark-launcher/pom.xml +++ b/presto-spark-launcher/pom.xml @@ -51,35 +51,6 @@ - - - spark3 - - - - spark-version - 3 - - - - - - com.facebook.presto.spark - spark-core - 3.4.1-1 - provided - - - - org.scala-lang - scala-library - provided - - - - - - @@ -139,4 +110,47 @@ + + + spark2 + + + true + + !spark-version + + + + + + spark3 + + + + spark-version + 3 + + + + + 3 + + + + + com.facebook.presto.spark + spark-core + 3.4.1-1 + provided + + + + org.scala-lang + scala-library + provided + + + + +