Revert "Spark 4.0 integration"

.github/workflows/java-ci.yml

@@ -95,7 +95,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [17, 21]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
@@ -108,7 +108,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [17, 21]
+        jvm: [11, 17, 21]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4

.github/workflows/jmh-benchmarks.yml

@@ -28,8 +28,8 @@ on:
         description: 'The branch name'
         required: true
       spark_version:
-        description: 'The spark project version to use, such as iceberg-spark-4.0'
-        default: 'iceberg-spark-4.0'
+        description: 'The spark project version to use, such as iceberg-spark-3.5'
+        default: 'iceberg-spark-3.5'
         required: true
       benchmarks:
         description: 'A list of comma-separated double-quoted Benchmark names, such as "IcebergSourceFlatParquetDataReadBenchmark", "IcebergSourceFlatParquetDataFilterBenchmark"'

.github/workflows/publish-snapshot.yml

@@ -41,4 +41,4 @@ jobs:
       - run: |
           ./gradlew printVersion
           ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
-          ./gradlew -DflinkVersions= -DsparkVersions=3.4,3.5,4.0 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+          ./gradlew -DflinkVersions= -DsparkVersions=3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

.github/workflows/recurring-jmh-benchmarks.yml

@@ -41,7 +41,7 @@ jobs:
                     "IcebergSourceNestedParquetDataReadBenchmark", "IcebergSourceNestedParquetDataWriteBenchmark",
                     "IcebergSourceParquetEqDeleteBenchmark", "IcebergSourceParquetMultiDeleteFileBenchmark",
                     "IcebergSourceParquetPosDeleteBenchmark", "IcebergSourceParquetWithUnrelatedDeleteBenchmark"]
-        spark_version: ['iceberg-spark-4.0']
+        spark_version: ['iceberg-spark-3.5']
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/spark-ci.yml

@@ -71,17 +71,13 @@ jobs:
     strategy:
       matrix:
         jvm: [11, 17, 21]
-        spark: ['3.4', '3.5', '4.0']
+        spark: ['3.4', '3.5']
         scala: ['2.12', '2.13']
         exclude:
           # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369)
           # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831)
-          - jvm: 11
-            spark: '4.0'
           - jvm: 21
             spark: '3.4'
-          - spark: '4.0'
-            scala: '2.12'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.gitignore

@@ -35,8 +35,6 @@ spark/v3.4/spark/benchmark/*
 spark/v3.4/spark-extensions/benchmark/*
 spark/v3.5/spark/benchmark/*
 spark/v3.5/spark-extensions/benchmark/*
-spark/v4.0/spark/benchmark/*
-spark/v4.0/spark-extensions/benchmark/*
 */benchmark/*
 
 __pycache__/

build.gradle

@@ -120,9 +120,6 @@ allprojects {
   repositories {
     mavenCentral()
     mavenLocal()
-    maven {
-      url "https://repository.apache.org/content/repositories/orgapachespark-1480/"
-    }
   }
 }
 

core/src/main/java/org/apache/iceberg/MetadataColumns.java

@@ -51,7 +51,7 @@ private MetadataColumns() {}
   public static final int SPEC_ID_COLUMN_ID = Integer.MAX_VALUE - 4;
   public static final String SPEC_ID_COLUMN_DOC = "Spec ID used to track the file containing a row";
   public static final NestedField SPEC_ID =
-      NestedField.optional(
+      NestedField.required(
           SPEC_ID_COLUMN_ID, "_spec_id", Types.IntegerType.get(), SPEC_ID_COLUMN_DOC);
   // the partition column type is not static and depends on all specs in the table
   public static final int PARTITION_COLUMN_ID = Integer.MAX_VALUE - 5;

gradle.properties

@@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=2.0
 systemProp.knownFlinkVersions=1.19,1.20,2.0
-systemProp.defaultSparkVersions=4.0
-systemProp.knownSparkVersions=3.4,3.5,4.0
+systemProp.defaultSparkVersions=3.5
+systemProp.knownSparkVersions=3.4,3.5
 systemProp.defaultKafkaVersions=3
 systemProp.knownKafkaVersions=3
 systemProp.defaultScalaVersion=2.12

gradle/libs.versions.toml

@@ -24,7 +24,6 @@ activation = "1.1.1"
 aliyun-sdk-oss = "3.10.2"
 analyticsaccelerator = "1.0.0"
 antlr = "4.9.3"
-antlr413 = "4.13.1" # For Spark 4.0 support
 aircompressor = "0.27"
 apiguardian = "1.1.2"
 arrow = "15.0.2"
@@ -83,7 +82,6 @@ slf4j = "2.0.17"
 snowflake-jdbc = "3.24.0"
 spark34 = "3.4.4"
 spark35 = "3.5.5"
-spark40 = "4.0.0"
 sqlite-jdbc = "3.49.1.0"
 testcontainers = "1.21.0"
 tez08 = { strictly = "0.8.4"}  # see rich version usage explanation above
@@ -95,8 +93,6 @@ aliyun-sdk-oss = { module = "com.aliyun.oss:aliyun-sdk-oss", version.ref = "aliy
 analyticsaccelerator-s3 = { module = "software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3", version.ref = "analyticsaccelerator" }
 antlr-antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr" }
 antlr-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr" }
-antlr-antlr413 = { module = "org.antlr:antlr4", version.ref = "antlr413" }
-antlr-runtime413 = { module = "org.antlr:antlr4-runtime", version.ref = "antlr413" }
 arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.ref = "arrow" }
 arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" }
 avro-avro = { module = "org.apache.avro:avro", version.ref = "avro" }

hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java

@@ -282,7 +282,6 @@ private void initConf(HiveConf conf, int port, boolean directSql) {
     // Setting this to avoid thrift exception during running Iceberg tests outside Iceberg.
     conf.set(
         HiveConf.ConfVars.HIVE_IN_TEST.varname, HiveConf.ConfVars.HIVE_IN_TEST.getDefaultValue());
-    conf.set("datanucleus.connectionPoolingType", "DBCP");
   }
 
   private static void setupMetastoreDB(String dbURL) throws SQLException, IOException {

jmh.gradle

@@ -48,11 +48,6 @@ if (sparkVersions.contains("3.5")) {
   jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.5_${scalaVersion}"))
 }
 
-if (sparkVersions.contains("4.0")) {
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.0_2.13"))
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.0_2.13"))
-}
-
 configure(jmhProjects) {
   apply plugin: 'me.champeau.jmh'
   apply plugin: 'io.morethan.jmhreport'

settings.gradle

@@ -161,18 +161,6 @@ if (sparkVersions.contains("3.5")) {
   project(":iceberg-spark:spark-runtime-3.5_${scalaVersion}").name = "iceberg-spark-runtime-3.5_${scalaVersion}"
 }
 
-if (sparkVersions.contains("4.0")) {
-  include ":iceberg-spark:spark-4.0_2.13"
-  include ":iceberg-spark:spark-extensions-4.0_2.13"
-  include ":iceberg-spark:spark-runtime-4.0_2.13"
-  project(":iceberg-spark:spark-4.0_2.13").projectDir = file('spark/v4.0/spark')
-  project(":iceberg-spark:spark-4.0_2.13").name = "iceberg-spark-4.0_2.13"
-  project(":iceberg-spark:spark-extensions-4.0_2.13").projectDir = file('spark/v4.0/spark-extensions')
-  project(":iceberg-spark:spark-extensions-4.0_2.13").name = "iceberg-spark-extensions-4.0_2.13"
-  project(":iceberg-spark:spark-runtime-4.0_2.13").projectDir = file('spark/v4.0/spark-runtime')
-  project(":iceberg-spark:spark-runtime-4.0_2.13").name = "iceberg-spark-runtime-4.0_2.13"
-}
-
 if (kafkaVersions.contains("3")) {
   include 'kafka-connect'
   project(':kafka-connect').name = 'iceberg-kafka-connect'

spark/build.gradle

@@ -27,7 +27,3 @@ if (sparkVersions.contains("3.4")) {
 if (sparkVersions.contains("3.5")) {
   apply from: file("$projectDir/v3.5/build.gradle")
 }
-
-if (sparkVersions.contains("4.0")) {
-  apply from: file("$projectDir/v4.0/build.gradle")
-}