diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index d2837acd500a..5aebbf38afe8 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -75,34 +75,7 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} restore-keys: ${{ runner.os }}-gradle - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark2:check :iceberg-spark:iceberg-spark-runtime:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v2 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - - spark3-tests: - runs-on: ubuntu-latest - strategy: - matrix: - jvm: [8, 11] - spark: ['3.0'] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v2 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: ${{ runner.os }}-gradle - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark3:check :iceberg-spark:iceberg-spark3-extensions:check :iceberg-spark:iceberg-spark3-runtime:check -Pquick=true -x javadoc + - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v2 if: failure() with: @@ -115,7 +88,7 @@ jobs: strategy: matrix: jvm: [8, 11] - spark: ['3.1', '3.2'] + spark: ['3.0', '3.1', '3.2'] env: SPARK_LOCAL_IP: localhost steps: diff --git a/jmh.gradle b/jmh.gradle index 402af5633b89..7941b2b689b1 100644 --- a/jmh.gradle +++ b/jmh.gradle @@ -26,11 +26,11 @@ def scalaVersion = System.getProperty("scalaVersion") != null ? System.getProper def jmhProjects = [] if (jdkVersion == '8' && sparkVersions.contains("2.4")) { - jmhProjects.add(project(":iceberg-spark:iceberg-spark2")) + jmhProjects.add(project(":iceberg-spark:iceberg-spark-2.4")) } if (sparkVersions.contains("3.0")) { - jmhProjects.add(project(":iceberg-spark:iceberg-spark3")) + jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.0_2.12")) } if (sparkVersions.contains("3.1")) { diff --git a/settings.gradle b/settings.gradle index 09d38c66892b..db48d4884cf7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -114,15 +114,15 @@ if (flinkVersions.contains("1.14")) { } if (sparkVersions.contains("3.0")) { - include ':iceberg-spark:spark3' - include ':iceberg-spark:spark3-extensions' - include ':iceberg-spark:spark3-runtime' - project(':iceberg-spark:spark3').projectDir = file('spark/v3.0/spark') - project(':iceberg-spark:spark3').name = 'iceberg-spark3' - project(':iceberg-spark:spark3-extensions').projectDir = file('spark/v3.0/spark-extensions') - project(':iceberg-spark:spark3-extensions').name = 'iceberg-spark3-extensions' - project(':iceberg-spark:spark3-runtime').projectDir = file('spark/v3.0/spark-runtime') - project(':iceberg-spark:spark3-runtime').name = 'iceberg-spark3-runtime' + include ':iceberg-spark:spark-3.0_2.12' + include ':iceberg-spark:spark-extensions-3.0_2.12' + include ':iceberg-spark:spark-runtime-3.0_2.12' + project(':iceberg-spark:spark-3.0_2.12').projectDir = file('spark/v3.0/spark') + project(':iceberg-spark:spark-3.0_2.12').name = 'iceberg-spark-3.0_2.12' + project(':iceberg-spark:spark-extensions-3.0_2.12').projectDir = file('spark/v3.0/spark-extensions') + project(':iceberg-spark:spark-extensions-3.0_2.12').name = 'iceberg-spark-extensions-3.0_2.12' + project(':iceberg-spark:spark-runtime-3.0_2.12').projectDir = file('spark/v3.0/spark-runtime') + project(':iceberg-spark:spark-runtime-3.0_2.12').name = 'iceberg-spark-runtime-3.0_2.12' } if (sparkVersions.contains("3.1")) { @@ -160,13 +160,13 @@ if (hiveVersions.contains("2") || hiveVersions.contains("3")) { if (JavaVersion.current() == JavaVersion.VERSION_1_8) { if (sparkVersions.contains("2.4")) { - include ':iceberg-spark:spark2' - include ':iceberg-spark:spark-runtime' + include ':iceberg-spark:spark-2.4' + include ':iceberg-spark:spark-runtime-2.4' - project(':iceberg-spark:spark2').projectDir = file('spark/v2.4/spark') - project(':iceberg-spark:spark2').name = 'iceberg-spark2' - project(':iceberg-spark:spark-runtime').projectDir = file('spark/v2.4/spark-runtime') - project(':iceberg-spark:spark-runtime').name = 'iceberg-spark-runtime' + project(':iceberg-spark:spark-2.4').projectDir = file('spark/v2.4/spark') + project(':iceberg-spark:spark-2.4').name = 'iceberg-spark-2.4' + project(':iceberg-spark:spark-runtime-2.4').projectDir = file('spark/v2.4/spark-runtime') + project(':iceberg-spark:spark-runtime-2.4').name = 'iceberg-spark-runtime-2.4' } if (hiveVersions.contains("3")) { diff --git a/spark/v2.4/build.gradle b/spark/v2.4/build.gradle index eb805ae7fc14..6fa849d04a21 100644 --- a/spark/v2.4/build.gradle +++ b/spark/v2.4/build.gradle @@ -22,8 +22,8 @@ if (jdkVersion != '8') { } def sparkProjects = [ - project(':iceberg-spark:iceberg-spark2'), - project(':iceberg-spark:iceberg-spark-runtime') + project(':iceberg-spark:iceberg-spark-2.4'), + project(':iceberg-spark:iceberg-spark-runtime-2.4') ] configure(sparkProjects) { @@ -37,7 +37,7 @@ configure(sparkProjects) { } } -project(':iceberg-spark:iceberg-spark2') { +project(':iceberg-spark:iceberg-spark-2.4') { configurations.all { resolutionStrategy { // Spark 2.4.4 can only use the below datanucleus version, the versions introduced @@ -105,7 +105,7 @@ project(':iceberg-spark:iceberg-spark2') { } // the runtime jar is a self-contained artifact for testing in a notebook -project(':iceberg-spark:iceberg-spark-runtime') { +project(':iceberg-spark:iceberg-spark-runtime-2.4') { apply plugin: 'com.github.johnrengelman.shadow' tasks.jar.dependsOn tasks.shadowJar @@ -125,7 +125,7 @@ project(':iceberg-spark:iceberg-spark-runtime') { } dependencies { - implementation project(':iceberg-spark:iceberg-spark2') + implementation project(':iceberg-spark:iceberg-spark-2.4') implementation project(':iceberg-aws') implementation(project(':iceberg-aliyun')) { exclude group: 'edu.umd.cs.findbugs', module: 'findbugs' diff --git a/spark/v3.0/build.gradle b/spark/v3.0/build.gradle index 5bbc91318ea6..74afc9e48ebb 100644 --- a/spark/v3.0/build.gradle +++ b/spark/v3.0/build.gradle @@ -18,9 +18,9 @@ */ def sparkProjects = [ - project(':iceberg-spark:iceberg-spark3'), - project(":iceberg-spark:iceberg-spark3-extensions"), - project(':iceberg-spark:iceberg-spark3-runtime') + project(':iceberg-spark:iceberg-spark-3.0_2.12'), + project(":iceberg-spark:iceberg-spark-extensions-3.0_2.12"), + project(':iceberg-spark:iceberg-spark-runtime-3.0_2.12') ] configure(sparkProjects) { @@ -38,7 +38,7 @@ configure(sparkProjects) { } } -project(':iceberg-spark:iceberg-spark3') { +project(':iceberg-spark:iceberg-spark-3.0_2.12') { apply plugin: 'scala' apply plugin: 'com.github.alisiikh.scalastyle' @@ -105,7 +105,7 @@ project(':iceberg-spark:iceberg-spark3') { } } -project(":iceberg-spark:iceberg-spark3-extensions") { +project(":iceberg-spark:iceberg-spark-extensions-3.0_2.12") { apply plugin: 'java-library' apply plugin: 'scala' apply plugin: 'com.github.alisiikh.scalastyle' @@ -132,7 +132,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") { compileOnly project(':iceberg-data') compileOnly project(':iceberg-orc') compileOnly project(':iceberg-common') - compileOnly project(':iceberg-spark:iceberg-spark3') + compileOnly project(':iceberg-spark:iceberg-spark-3.0_2.12') compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' @@ -146,7 +146,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") { testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark:iceberg-spark-3.0_2.12', configuration: 'testArtifacts') testImplementation "org.apache.avro:avro" @@ -162,7 +162,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") { } } -project(':iceberg-spark:iceberg-spark3-runtime') { +project(':iceberg-spark:iceberg-spark-runtime-3.0_2.12') { apply plugin: 'com.github.johnrengelman.shadow' tasks.jar.dependsOn tasks.shadowJar @@ -196,8 +196,8 @@ project(':iceberg-spark:iceberg-spark3-runtime') { dependencies { api project(':iceberg-api') - implementation project(':iceberg-spark:iceberg-spark3') - implementation project(':iceberg-spark:iceberg-spark3-extensions') + implementation project(':iceberg-spark:iceberg-spark-3.0_2.12') + implementation project(':iceberg-spark:iceberg-spark-extensions-3.0_2.12') implementation project(':iceberg-aws') implementation(project(':iceberg-aliyun')) { exclude group: 'edu.umd.cs.findbugs', module: 'findbugs' @@ -214,11 +214,11 @@ project(':iceberg-spark:iceberg-spark3-runtime') { integrationImplementation 'org.slf4j:slf4j-simple' integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark:iceberg-spark3-extensions', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.0_2.12', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark:iceberg-spark-extensions-3.0_2.12', configuration: 'testArtifacts') // Not allowed on our classpath, only the runtime jar is allowed - integrationCompileOnly project(':iceberg-spark:iceberg-spark3-extensions') - integrationCompileOnly project(':iceberg-spark:iceberg-spark3') + integrationCompileOnly project(':iceberg-spark:iceberg-spark-extensions-3.0_2.12') + integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.0_2.12') integrationCompileOnly project(':iceberg-api') }