diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 398cdfefe4d4..e54471c301be 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -66,7 +66,7 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 8 - - run: ./gradlew build -x test -x javadoc -x integrationTest -x testSpark31 -x spark31IntegrationTest + - run: ./gradlew build -x test -x javadoc -x integrationTest build-javadoc: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 777b0b1ffe14..d8e6dd5cf374 100644 --- a/.gitignore +++ b/.gitignore @@ -26,10 +26,10 @@ lib/ site/site # benchmark output -spark2/benchmark/* -!spark2/benchmark/.gitkeep -spark3/benchmark/* -!spark3/benchmark/.gitkeep +spark/v2.4/spark2/benchmark/* +!spark/v2.4/spark2/benchmark/.gitkeep +spark/v3.0/spark3/benchmark/* +!spark/v3.0/spark3/benchmark/.gitkeep __pycache__/ *.py[cod] diff --git a/build.gradle b/build.gradle index 17004790ff46..593938810ad1 100644 --- a/build.gradle +++ b/build.gradle @@ -934,476 +934,6 @@ project(':iceberg-arrow') { } } -project(':iceberg-spark') { - configurations.all { - resolutionStrategy { - // Spark 2.4.4 can only use the below datanucleus version, the versions introduced - // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions - // introduced by Hive 1.2.1. - force 'org.datanucleus:datanucleus-api-jdo:3.2.6' - force 'org.datanucleus:datanucleus-core:3.2.10' - force 'org.datanucleus:datanucleus-rdbms:3.2.9' - } - } - - dependencies { - implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - api project(':iceberg-api') - implementation project(':iceberg-common') - implementation project(':iceberg-core') - api project(':iceberg-data') - implementation project(':iceberg-orc') - implementation project(':iceberg-parquet') - implementation project(':iceberg-arrow') - implementation project(':iceberg-hive-metastore') - - compileOnly "com.google.errorprone:error_prone_annotations" - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.11") { - exclude group: 'org.apache.avro', module: 'avro' - } - - implementation("org.apache.orc:orc-core::nohive") { - exclude group: 'org.apache.hadoop' - exclude group: 'commons-lang' - // These artifacts are shaded and included in the orc-core fat jar - exclude group: 'com.google.protobuf', module: 'protobuf-java' - exclude group: 'org.apache.hive', module: 'hive-storage-api' - } - - implementation("org.apache.arrow:arrow-vector") { - exclude group: 'io.netty', module: 'netty-buffer' - exclude group: 'io.netty', module: 'netty-common' - exclude group: 'com.google.code.findbugs', module: 'jsr305' - } - - testImplementation("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' - } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - } - - test { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") - - // Vectorized reads need more memory - maxHeapSize '2500m' - } -} - -if (jdkVersion == '8') { - - project(':iceberg-spark2') { - configurations.all { - resolutionStrategy { - // Spark 2.4.4 can only use the below datanucleus version, the versions introduced - // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions - // introduced by Hive 1.2.1. - force 'org.datanucleus:datanucleus-api-jdo:3.2.6' - force 'org.datanucleus:datanucleus-core:3.2.10' - force 'org.datanucleus:datanucleus-rdbms:3.2.9' - } - } - - dependencies { - implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - api project(':iceberg-api') - implementation project(':iceberg-common') - implementation project(':iceberg-core') - implementation project(':iceberg-data') - implementation project(':iceberg-orc') - implementation project(':iceberg-parquet') - implementation project(':iceberg-arrow') - implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark') - implementation "com.github.ben-manes.caffeine:caffeine" - - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.11") { - exclude group: 'org.apache.avro', module: 'avro' - } - - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - - testImplementation("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' - } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - } - - test { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") - - // Vectorized reads need more memory - maxHeapSize '2500m' - } - } - - // the runtime jar is a self-contained artifact for testing in a notebook - project(':iceberg-spark-runtime') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.jar.dependsOn tasks.shadowJar - - configurations { - implementation { - exclude group: 'org.apache.spark' - // included in Spark - exclude group: 'org.slf4j' - exclude group: 'org.apache.commons' - exclude group: 'commons-pool' - exclude group: 'commons-codec' - exclude group: 'org.xerial.snappy' - exclude group: 'javax.xml.bind' - exclude group: 'javax.annotation' - } - } - - dependencies { - implementation project(':iceberg-spark2') - implementation project(':iceberg-aws') - implementation(project(':iceberg-nessie')) { - exclude group: 'com.google.code.findbugs', module: 'jsr305' - } - } - - shadowJar { - configurations = [project.configurations.runtimeClasspath] - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' - relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' - relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' - relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' - relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' - relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' - relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' - relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' - relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' - relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' - // relocate Arrow and related deps to shade Iceberg specific version - relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' - relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' - relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' - relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' - - classifier null - } - - jar { - enabled = false - } - } -} - -project(':iceberg-spark3') { - apply plugin: 'scala' - - sourceSets { - // Compile test source against Spark 3.1 and main classes compiled against Spark 3.0 - spark31 { - java.srcDir "$projectDir/src/test/java" - resources.srcDir "$projectDir/src/test/resources" - compileClasspath += sourceSets.test.output + sourceSets.main.output - runtimeClasspath += sourceSets.test.output - } - } - - configurations { - spark31Implementation.extendsFrom testImplementation - spark31RuntimeOnly.extendsFrom testRuntimeOnly - } - - dependencies { - implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - api project(':iceberg-api') - implementation project(':iceberg-common') - implementation project(':iceberg-core') - implementation project(':iceberg-data') - implementation project(':iceberg-orc') - implementation project(':iceberg-parquet') - implementation project(':iceberg-arrow') - implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark') - - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - } - - implementation("org.apache.arrow:arrow-vector") { - exclude group: 'io.netty', module: 'netty-buffer' - exclude group: 'io.netty', module: 'netty-common' - exclude group: 'com.google.code.findbugs', module: 'jsr305' - } - - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - - testImplementation("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' - } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - testImplementation "org.xerial:sqlite-jdbc" - - spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - exclude group: 'junit' - } - } - - tasks.withType(Test) { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") - - // Vectorized reads need more memory - maxHeapSize '2560m' - } - - task testSpark31(type: Test) { - dependsOn classes - group = "verification" - description = "Test against Spark 3.1" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + sourceSets.main.output - } - check.dependsOn testSpark31 -} - -project(":iceberg-spark3-extensions") { - apply plugin: 'java-library' - apply plugin: 'scala' - apply plugin: 'antlr' - - sourceSets { - // Compile test source against Spark 3.1 and main classes compiled against Spark 3.0 - spark31 { - // Main source is in scala, but test source is only in java - java.srcDir "$projectDir/src/test/java" - resources.srcDir "$projectDir/src/test/resources" - compileClasspath += sourceSets.test.output + sourceSets.main.output - runtimeClasspath += sourceSets.test.output - } - } - - configurations { - spark31Implementation.extendsFrom testImplementation - spark31RuntimeOnly.extendsFrom testRuntimeOnly - - /* - The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This - bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate - runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with - the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add - the runtime dependency back to only the runtime configuration manually. - */ - implementation { - extendsFrom = extendsFrom.findAll { it != configurations.antlr } - } - } - - dependencies { - compileOnly "org.scala-lang:scala-library" - compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') - compileOnly project(':iceberg-api') - compileOnly project(':iceberg-core') - compileOnly project(':iceberg-common') - compileOnly project(':iceberg-spark') - compileOnly project(':iceberg-spark3') - compileOnly project(':iceberg-hive-metastore') - compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - } - - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') - - spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - } - - // Required because we remove antlr plugin dependencies from the compile configuration, see note above - // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime - runtimeOnly "org.antlr:antlr4-runtime:4.7.1" - antlr "org.antlr:antlr4:4.7.1" - } - - generateGrammarSource { - maxHeapSize = "64m" - arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions'] - } - - task testSpark31(type: Test) { - dependsOn classes - group = "verification" - description = "Test against Spark 3.1" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + sourceSets.main.output - } - check.dependsOn testSpark31 -} - -project(':iceberg-spark3-runtime') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.jar.dependsOn tasks.shadowJar - - sourceSets { - integration { - java.srcDir "$projectDir/src/integration/java" - resources.srcDir "$projectDir/src/integration/resources" - } - spark31 { - java.srcDir "$projectDir/src/integration/java" - resources.srcDir "$projectDir/src/integration/resources" - compileClasspath += sourceSets.integration.output - runtimeClasspath += sourceSets.integration.output - } - } - - configurations { - implementation { - exclude group: 'org.apache.spark' - // included in Spark - exclude group: 'org.slf4j' - exclude group: 'org.apache.commons' - exclude group: 'commons-pool' - exclude group: 'commons-codec' - exclude group: 'org.xerial.snappy' - exclude group: 'javax.xml.bind' - exclude group: 'javax.annotation' - exclude group: 'com.github.luben' - exclude group: 'com.ibm.icu' - exclude group: 'org.glassfish' - exclude group: 'org.abego.treelayout' - exclude group: 'org.antlr', module: 'ST4' - exclude group: 'org.antlr', module: 'antlr4' - } - spark31Implementation.extendsFrom integrationImplementation - spark31CompileOnly.extendsFrom integrationCompileOnly - } - - dependencies { - api project(':iceberg-api') - implementation project(':iceberg-spark3') - implementation project(':iceberg-spark3-extensions') - implementation project(':iceberg-aws') - implementation(project(':iceberg-nessie')) { - exclude group: 'com.google.code.findbugs', module: 'jsr305' - } - - integrationImplementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}" - integrationImplementation 'org.junit.vintage:junit-vintage-engine' - integrationImplementation 'org.slf4j:slf4j-simple' - integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark3-extensions', configuration: 'testArtifacts') - // Not allowed on our classpath, only the runtime jar is allowed - integrationCompileOnly project(':iceberg-spark3-extensions') - integrationCompileOnly project(':iceberg-spark3') - integrationCompileOnly project(':iceberg-api') - - spark31Implementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}" - } - - shadowJar { - configurations = [project.configurations.runtimeClasspath] - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' - relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' - relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' - relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' - relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' - relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' - relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' - relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' - relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' - relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' - // relocate Arrow and related deps to shade Iceberg specific version - relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' - relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' - relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' - relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' - // relocate Antlr runtime and related deps to shade Iceberg specific version - relocate 'org.antlr.v4', 'org.apache.iceberg.shaded.org.antlr.v4' - - classifier null - } - - task integrationTest(type: Test) { - description = "Test Spark3 Runtime Jar against Spark 3.0" - group = "verification" - testClassesDirs = sourceSets.integration.output.classesDirs - classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) - inputs.file(shadowJar.archiveFile.get().asFile.path) - } - integrationTest.dependsOn shadowJar - - task spark31IntegrationTest(type: Test) { - dependsOn classes - description = "Test Spark3 Runtime Jar against Spark 3.1" - group = "verification" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) - } - spark31IntegrationTest.dependsOn shadowJar - - check.dependsOn integrationTest, spark31IntegrationTest - - jar { - enabled = false - } -} - project(':iceberg-pig') { dependencies { implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') diff --git a/gradle.properties b/gradle.properties index 5ea7f6ed959e..d76ea669a4fa 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,5 +15,7 @@ jmhOutputPath=build/reports/jmh/human-readable-output.txt jmhIncludeRegex=.* +systemProp.defaultSparkVersions=2.4,3.0 +systemProp.knownSparkVersions=2.4,3.0 org.gradle.parallel=true org.gradle.jvmargs=-Xmx768m diff --git a/jmh.gradle b/jmh.gradle index edcb39c795fb..0b002f5a3e4e 100644 --- a/jmh.gradle +++ b/jmh.gradle @@ -17,15 +17,21 @@ * under the License. */ -def jmhProjects = [] -if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - jmhProjects = [ project("iceberg-spark2"), project("iceberg-spark3") ] -} else if (JavaVersion.current() == JavaVersion.VERSION_11) { - jmhProjects = [ project("iceberg-spark3") ] -} else { +if (jdkVersion != '8' && jdkVersion != '11') { throw new GradleException("The JMH benchamrks must be run with JDK 8 or JDK 11") } +def sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",") +def jmhProjects = [] + +if (jdkVersion == '8' && sparkVersions.contains("2.4")) { + jmhProjects.add(project(":iceberg-spark:iceberg-spark2")) +} + +if (sparkVersions.contains("3.0")) { + jmhProjects.add(project(":iceberg-spark:iceberg-spark3")) +} + configure(jmhProjects) { apply plugin: 'me.champeau.gradle.jmh' diff --git a/settings.gradle b/settings.gradle index 7f052475d008..9ccf71f78a95 100644 --- a/settings.gradle +++ b/settings.gradle @@ -33,9 +33,6 @@ include 'arrow' include 'parquet' include 'bundled-guava' include 'spark' -include 'spark3' -include 'spark3-extensions' -include 'spark3-runtime' include 'pig' include 'hive-metastore' include 'nessie' @@ -55,21 +52,43 @@ project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' project(':bundled-guava').name = 'iceberg-bundled-guava' project(':spark').name = 'iceberg-spark' -project(':spark3').name = 'iceberg-spark3' -project(':spark3-extensions').name = 'iceberg-spark3-extensions' -project(':spark3-runtime').name = 'iceberg-spark3-runtime' project(':pig').name = 'iceberg-pig' project(':hive-metastore').name = 'iceberg-hive-metastore' project(':nessie').name = 'iceberg-nessie' +List knownSparkVersions = System.getProperty("knownSparkVersions").split(",") +List sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",") + +if (!knownSparkVersions.containsAll(sparkVersions)) { + throw new GradleException("Found unsupported Spark versions: " + (sparkVersions - knownSparkVersions)) +} + +if (sparkVersions.contains("3.0")) { + include ':iceberg-spark:spark3' + include ':iceberg-spark:spark3-extensions' + include ':iceberg-spark:spark3-runtime' + project(':iceberg-spark:spark3').projectDir = file('spark/v3.0/spark3') + project(':iceberg-spark:spark3').name = 'iceberg-spark3' + project(':iceberg-spark:spark3-extensions').projectDir = file('spark/v3.0/spark3-extensions') + project(':iceberg-spark:spark3-extensions').name = 'iceberg-spark3-extensions' + project(':iceberg-spark:spark3-runtime').projectDir = file('spark/v3.0/spark3-runtime') + project(':iceberg-spark:spark3-runtime').name = 'iceberg-spark3-runtime' +} + if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - include 'spark2' - include 'spark-runtime' + if (sparkVersions.contains("2.4")) { + include ':iceberg-spark:spark2' + include ':iceberg-spark:spark-runtime' + + project(':iceberg-spark:spark2').projectDir = file('spark/v2.4/spark2') + project(':iceberg-spark:spark2').name = 'iceberg-spark2' + project(':iceberg-spark:spark-runtime').projectDir = file('spark/v2.4/spark-runtime') + project(':iceberg-spark:spark-runtime').name = 'iceberg-spark-runtime' + } + include 'hive3' include 'hive3-orc-bundle' - project(':spark2').name = 'iceberg-spark2' - project(':spark-runtime').name = 'iceberg-spark-runtime' project(':hive3').name = 'iceberg-hive3' project(':hive3-orc-bundle').name = 'iceberg-hive3-orc-bundle' } diff --git a/spark/build.gradle b/spark/build.gradle new file mode 100644 index 000000000000..b3432689310f --- /dev/null +++ b/spark/build.gradle @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +project(':iceberg-spark') { + configurations.all { + resolutionStrategy { + // Spark 2.4.4 can only use the below datanucleus version, the versions introduced + // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions + // introduced by Hive 1.2.1. + force 'org.datanucleus:datanucleus-api-jdo:3.2.6' + force 'org.datanucleus:datanucleus-core:3.2.10' + force 'org.datanucleus:datanucleus-rdbms:3.2.9' + } + } + + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + api project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + + compileOnly "com.google.errorprone:error_prone_annotations" + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_2.11") { + exclude group: 'org.apache.avro', module: 'avro' + } + + implementation("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + + implementation("org.apache.arrow:arrow-vector") { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + testImplementation("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + } + + test { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") + + // Vectorized reads need more memory + maxHeapSize '2500m' + } +} + +// add enabled Spark version modules to the build +def sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",") + +if (jdkVersion == '8' && sparkVersions.contains("2.4")) { + apply from: file("$projectDir/v2.4/build.gradle") +} + +if (sparkVersions.contains("3.0")) { + apply from: file("$projectDir/v3.0/build.gradle") +} + diff --git a/spark/v2.4/build.gradle b/spark/v2.4/build.gradle new file mode 100644 index 000000000000..b9d3c1aeb168 --- /dev/null +++ b/spark/v2.4/build.gradle @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +if (jdkVersion != '8') { + throw new GradleException("Spark 2.4 must be built with Java 8") +} + +project(':iceberg-spark:iceberg-spark2') { + configurations.all { + resolutionStrategy { + // Spark 2.4.4 can only use the below datanucleus version, the versions introduced + // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions + // introduced by Hive 1.2.1. + force 'org.datanucleus:datanucleus-api-jdo:3.2.6' + force 'org.datanucleus:datanucleus-core:3.2.10' + force 'org.datanucleus:datanucleus-rdbms:3.2.9' + } + } + + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark') + implementation "com.github.ben-manes.caffeine:caffeine" + + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_2.11") { + exclude group: 'org.apache.avro', module: 'avro' + } + + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + + testImplementation("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + } + + test { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") + + // Vectorized reads need more memory + maxHeapSize '2500m' + } +} + +// the runtime jar is a self-contained artifact for testing in a notebook +project(':iceberg-spark:iceberg-spark-runtime') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.jar.dependsOn tasks.shadowJar + + configurations { + implementation { + exclude group: 'org.apache.spark' + // included in Spark + exclude group: 'org.slf4j' + exclude group: 'org.apache.commons' + exclude group: 'commons-pool' + exclude group: 'commons-codec' + exclude group: 'org.xerial.snappy' + exclude group: 'javax.xml.bind' + exclude group: 'javax.annotation' + } + } + + dependencies { + implementation project(':iceberg-spark:iceberg-spark2') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + } + + shadowJar { + configurations = [project.configurations.runtimeClasspath] + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' + relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' + relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' + relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' + relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' + relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' + relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' + relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' + relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' + relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' + // relocate Arrow and related deps to shade Iceberg specific version + relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' + relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' + relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' + relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' + + classifier null + } + + jar { + enabled = false + } +} diff --git a/spark-runtime/LICENSE b/spark/v2.4/spark-runtime/LICENSE similarity index 100% rename from spark-runtime/LICENSE rename to spark/v2.4/spark-runtime/LICENSE diff --git a/spark-runtime/NOTICE b/spark/v2.4/spark-runtime/NOTICE similarity index 100% rename from spark-runtime/NOTICE rename to spark/v2.4/spark-runtime/NOTICE diff --git a/spark2/benchmark/.gitkeep b/spark/v2.4/spark2/benchmark/.gitkeep similarity index 100% rename from spark2/benchmark/.gitkeep rename to spark/v2.4/spark2/benchmark/.gitkeep diff --git a/spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java b/spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java rename to spark/v2.4/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java diff --git a/spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/v2.4/spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 100% rename from spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark/v2.4/spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/README.md b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/README.md similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/README.md rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/README.md diff --git a/spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java b/spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java rename to spark/v2.4/spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java diff --git a/spark2/src/test/resources/data/books.json b/spark/v2.4/spark2/src/test/resources/data/books.json similarity index 100% rename from spark2/src/test/resources/data/books.json rename to spark/v2.4/spark2/src/test/resources/data/books.json diff --git a/spark2/src/test/resources/data/new-books.json b/spark/v2.4/spark2/src/test/resources/data/new-books.json similarity index 100% rename from spark2/src/test/resources/data/new-books.json rename to spark/v2.4/spark2/src/test/resources/data/new-books.json diff --git a/spark/v3.0/build.gradle b/spark/v3.0/build.gradle new file mode 100644 index 000000000000..480f6adbb164 --- /dev/null +++ b/spark/v3.0/build.gradle @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +project(':iceberg-spark:iceberg-spark3') { + apply plugin: 'scala' + + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark') + + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.arrow' + } + + implementation("org.apache.arrow:arrow-vector") { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + + testImplementation("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation "org.xerial:sqlite-jdbc" + } + + tasks.withType(Test) { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") + + // Vectorized reads need more memory + maxHeapSize '2560m' + } +} + +project(":iceberg-spark:iceberg-spark3-extensions") { + apply plugin: 'java-library' + apply plugin: 'scala' + apply plugin: 'antlr' + + configurations { + /* + The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This + bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate + runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with + the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add + the runtime dependency back to only the runtime configuration manually. + */ + implementation { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } + } + + dependencies { + compileOnly "org.scala-lang:scala-library" + compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') + compileOnly project(':iceberg-api') + compileOnly project(':iceberg-core') + compileOnly project(':iceberg-common') + compileOnly project(':iceberg-spark') + compileOnly project(':iceberg-spark:iceberg-spark3') + compileOnly project(':iceberg-hive-metastore') + compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.arrow' + } + + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts') + + // Required because we remove antlr plugin dependencies from the compile configuration, see note above + // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime + runtimeOnly "org.antlr:antlr4-runtime:4.7.1" + antlr "org.antlr:antlr4:4.7.1" + } + + generateGrammarSource { + maxHeapSize = "64m" + arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions'] + } +} + +project(':iceberg-spark:iceberg-spark3-runtime') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.jar.dependsOn tasks.shadowJar + + sourceSets { + integration { + java.srcDir "$projectDir/src/integration/java" + resources.srcDir "$projectDir/src/integration/resources" + } + } + + configurations { + implementation { + exclude group: 'org.apache.spark' + // included in Spark + exclude group: 'org.slf4j' + exclude group: 'org.apache.commons' + exclude group: 'commons-pool' + exclude group: 'commons-codec' + exclude group: 'org.xerial.snappy' + exclude group: 'javax.xml.bind' + exclude group: 'javax.annotation' + exclude group: 'com.github.luben' + exclude group: 'com.ibm.icu' + exclude group: 'org.glassfish' + exclude group: 'org.abego.treelayout' + exclude group: 'org.antlr', module: 'ST4' + exclude group: 'org.antlr', module: 'antlr4' + } + } + + dependencies { + api project(':iceberg-api') + implementation project(':iceberg-spark:iceberg-spark3') + implementation project(':iceberg-spark:iceberg-spark3-extensions') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + integrationImplementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}" + integrationImplementation 'org.junit.vintage:junit-vintage-engine' + integrationImplementation 'org.slf4j:slf4j-simple' + integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark:iceberg-spark3-extensions', configuration: 'testArtifacts') + // Not allowed on our classpath, only the runtime jar is allowed + integrationCompileOnly project(':iceberg-spark:iceberg-spark3-extensions') + integrationCompileOnly project(':iceberg-spark:iceberg-spark3') + integrationCompileOnly project(':iceberg-api') + } + + shadowJar { + configurations = [project.configurations.runtimeClasspath] + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' + relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' + relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' + relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' + relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' + relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' + relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' + relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' + relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' + relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' + // relocate Arrow and related deps to shade Iceberg specific version + relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' + relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' + relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' + relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' + // relocate Antlr runtime and related deps to shade Iceberg specific version + relocate 'org.antlr.v4', 'org.apache.iceberg.shaded.org.antlr.v4' + + classifier null + } + + task integrationTest(type: Test) { + description = "Test Spark3 Runtime Jar against Spark 3.0" + group = "verification" + testClassesDirs = sourceSets.integration.output.classesDirs + classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) + inputs.file(shadowJar.archiveFile.get().asFile.path) + } + integrationTest.dependsOn shadowJar + + jar { + enabled = false + } +} + diff --git a/spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/v3.0/spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 similarity index 100% rename from spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 rename to spark/v3.0/spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 diff --git a/spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala rename to spark/v3.0/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java b/spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java rename to spark/v3.0/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java diff --git a/spark3-runtime/LICENSE b/spark/v3.0/spark3-runtime/LICENSE similarity index 100% rename from spark3-runtime/LICENSE rename to spark/v3.0/spark3-runtime/LICENSE diff --git a/spark3-runtime/NOTICE b/spark/v3.0/spark3-runtime/NOTICE similarity index 100% rename from spark3-runtime/NOTICE rename to spark/v3.0/spark3-runtime/NOTICE diff --git a/spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java b/spark/v3.0/spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java similarity index 100% rename from spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java rename to spark/v3.0/spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java diff --git a/spark3/benchmark/.gitkeep b/spark/v3.0/spark3/benchmark/.gitkeep similarity index 100% rename from spark3/benchmark/.gitkeep rename to spark/v3.0/spark3/benchmark/.gitkeep diff --git a/spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/OrderField.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/OrderField.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/OrderField.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/OrderField.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java b/spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java rename to spark/v3.0/spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java diff --git a/spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java b/spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java rename to spark/v3.0/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java diff --git a/spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/v3.0/spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 100% rename from spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark/v3.0/spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala b/spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala rename to spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala b/spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala rename to spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala rename to spark/v3.0/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala diff --git a/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java b/spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java rename to spark/v3.0/spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java