Spark: refactor Spark modules to support multiple versions

.gitignore

@@ -26,10 +26,10 @@ lib/
 site/site
 
 # benchmark output
-spark2/benchmark/*
-!spark2/benchmark/.gitkeep
-spark3/benchmark/*
-!spark3/benchmark/.gitkeep
+spark/2_4/core/benchmark/*
+!spark/2_4/core/benchmark/.gitkeep
+spark/3_0/core/benchmark/*
+!spark/3_0/core/benchmark/.gitkeep
 
 __pycache__/
 *.py[cod]

jmh.gradle

@@ -19,9 +19,9 @@
 
 def jmhProjects = []
 if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
-  jmhProjects = [ project("iceberg-spark2"), project("iceberg-spark3") ]
+  jmhProjects = [ project("iceberg-spark") ]
 } else if (JavaVersion.current() == JavaVersion.VERSION_11) {
-  jmhProjects = [ project("iceberg-spark3") ]
+  jmhProjects = [ project("iceberg-spark") ]
 } else {
   throw new GradleException("The JMH benchamrks must be run with JDK 8 or JDK 11")
 }

settings.gradle

@@ -32,10 +32,6 @@ include 'orc'
 include 'arrow'
 include 'parquet'
 include 'bundled-guava'
-include 'spark'
-include 'spark3'
-include 'spark3-extensions'
-include 'spark3-runtime'
 include 'pig'
 include 'hive-metastore'
 include 'nessie'
@@ -54,22 +50,38 @@ project(':orc').name = 'iceberg-orc'
 project(':arrow').name = 'iceberg-arrow'
 project(':parquet').name = 'iceberg-parquet'
 project(':bundled-guava').name = 'iceberg-bundled-guava'
-project(':spark').name = 'iceberg-spark'
-project(':spark3').name = 'iceberg-spark3'
-project(':spark3-extensions').name = 'iceberg-spark3-extensions'
-project(':spark3-runtime').name = 'iceberg-spark3-runtime'
 project(':pig').name = 'iceberg-pig'
 project(':hive-metastore').name = 'iceberg-hive-metastore'
 project(':nessie').name = 'iceberg-nessie'
 
+// spark
+include 'spark-common'
+include 'spark'
+include 'spark-extensions'
+include 'spark-runtime'
+
+var SparkSrcVersion = System.getProperty("sparkSrc") != null ? System.getProperty("sparkSrc") : '3_0'
+
+if (SparkSrcVersion == '2_4' && JavaVersion.current() != JavaVersion.VERSION_1_8) {
+  throw new Exception("Expected java8 to build Spark 2.4, but got ${SparkSrcVersion}")
+}
+
+project(':spark-common').name = 'iceberg-spark-common'
+project(':spark').projectDir = file("spark/${SparkSrcVersion}/core")
+project(':spark').name = 'iceberg-spark'
+
+if (SparkSrcVersion != '2_4') {
+  project(':spark-extensions').projectDir = file("spark/${SparkSrcVersion}/extensions")
+  project(':spark-extensions').name = 'iceberg-spark-extensions'
+}
+
+project(':spark-runtime').projectDir = file("spark/${SparkSrcVersion}/runtime")
+project(':spark-runtime').name = 'iceberg-spark-runtime'
+
 if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
-  include 'spark2'
-  include 'spark-runtime'
   include 'hive3'
   include 'hive3-orc-bundle'
 
-  project(':spark2').name = 'iceberg-spark2'
-  project(':spark-runtime').name = 'iceberg-spark-runtime'
   project(':hive3').name = 'iceberg-hive3'
   project(':hive3-orc-bundle').name = 'iceberg-hive3-orc-bundle'
 }