Skip to content

Build: Align the module name among spark2.4, spark3.0, spark3.1 and spark3.2 #4158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openinx merged 2 commits into from
Mar 8, 2022
Merged

Build: Align the module name among spark2.4, spark3.0, spark3.1 and spark3.2 #4158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6199163
Build: Align the module name among spark2.4, spark3.0, spark3.1 and s…
openinx Jan 4, 2022
441bebd
Address the CI issues and spark 2.4 scala issues.
openinx Feb 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 2 additions & 29 deletions .github/workflows/spark-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,34 +75,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark2:check :iceberg-spark:iceberg-spark-runtime:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
name: test logs
path: |
**/build/testlogs

spark3-tests:
runs-on: ubuntu-latest
strategy:
matrix:
jvm: [8, 11]
spark: ['3.0']
env:
SPARK_LOCAL_IP: localhost
steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.jvm }}
- uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark3:check :iceberg-spark:iceberg-spark3-extensions:check :iceberg-spark:iceberg-spark3-runtime:check -Pquick=true -x javadoc
- run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
Expand All @@ -115,7 +88,7 @@ jobs:
strategy:
matrix:
jvm: [8, 11]
spark: ['3.1', '3.2']
spark: ['3.0', '3.1', '3.2']
env:
SPARK_LOCAL_IP: localhost
steps:
Expand Down
4 changes: 2 additions & 2 deletions jmh.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ def scalaVersion = System.getProperty("scalaVersion") != null ? System.getProper
def jmhProjects = []

if (jdkVersion == '8' && sparkVersions.contains("2.4")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark2"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-2.4"))
Copy link
Contributor

@rdblue rdblue Feb 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also add the scala version here?

Copy link
Member Author

@openinx openinx Feb 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As we've discussed in this conversation #4158 (comment), spark 2.4 should be scala-free, which means it won't add any scala code or pull in any transitive scala dependency for downstream iceberg-spark-2.4 users. So in theory we should not add the scala suffix version to the iceberg-spark-2.4 artifacts because the same artifact should be work for both spark with scala 2.12 and spark with scala 2.11.

}

if (sparkVersions.contains("3.0")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark3"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.0_2.12"))
}

if (sparkVersions.contains("3.1")) {
Expand Down
30 changes: 15 additions & 15 deletions settings.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,15 @@ if (flinkVersions.contains("1.14")) {
}

if (sparkVersions.contains("3.0")) {
include ':iceberg-spark:spark3'
include ':iceberg-spark:spark3-extensions'
include ':iceberg-spark:spark3-runtime'
project(':iceberg-spark:spark3').projectDir = file('spark/v3.0/spark')
project(':iceberg-spark:spark3').name = 'iceberg-spark3'
project(':iceberg-spark:spark3-extensions').projectDir = file('spark/v3.0/spark-extensions')
project(':iceberg-spark:spark3-extensions').name = 'iceberg-spark3-extensions'
project(':iceberg-spark:spark3-runtime').projectDir = file('spark/v3.0/spark-runtime')
project(':iceberg-spark:spark3-runtime').name = 'iceberg-spark3-runtime'
include ':iceberg-spark:spark-3.0_2.12'
include ':iceberg-spark:spark-extensions-3.0_2.12'
include ':iceberg-spark:spark-runtime-3.0_2.12'
project(':iceberg-spark:spark-3.0_2.12').projectDir = file('spark/v3.0/spark')
project(':iceberg-spark:spark-3.0_2.12').name = 'iceberg-spark-3.0_2.12'
project(':iceberg-spark:spark-extensions-3.0_2.12').projectDir = file('spark/v3.0/spark-extensions')
project(':iceberg-spark:spark-extensions-3.0_2.12').name = 'iceberg-spark-extensions-3.0_2.12'
project(':iceberg-spark:spark-runtime-3.0_2.12').projectDir = file('spark/v3.0/spark-runtime')
project(':iceberg-spark:spark-runtime-3.0_2.12').name = 'iceberg-spark-runtime-3.0_2.12'
}

if (sparkVersions.contains("3.1")) {
Expand Down Expand Up @@ -160,13 +160,13 @@ if (hiveVersions.contains("2") || hiveVersions.contains("3")) {

if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
if (sparkVersions.contains("2.4")) {
include ':iceberg-spark:spark2'
include ':iceberg-spark:spark-runtime'
include ':iceberg-spark:spark-2.4'
include ':iceberg-spark:spark-runtime-2.4'

project(':iceberg-spark:spark2').projectDir = file('spark/v2.4/spark')
project(':iceberg-spark:spark2').name = 'iceberg-spark2'
project(':iceberg-spark:spark-runtime').projectDir = file('spark/v2.4/spark-runtime')
project(':iceberg-spark:spark-runtime').name = 'iceberg-spark-runtime'
project(':iceberg-spark:spark-2.4').projectDir = file('spark/v2.4/spark')
project(':iceberg-spark:spark-2.4').name = 'iceberg-spark-2.4'
project(':iceberg-spark:spark-runtime-2.4').projectDir = file('spark/v2.4/spark-runtime')
project(':iceberg-spark:spark-runtime-2.4').name = 'iceberg-spark-runtime-2.4'
}

if (hiveVersions.contains("3")) {
Expand Down
10 changes: 5 additions & 5 deletions spark/v2.4/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ if (jdkVersion != '8') {
}

def sparkProjects = [
project(':iceberg-spark:iceberg-spark2'),
project(':iceberg-spark:iceberg-spark-runtime')
project(':iceberg-spark:iceberg-spark-2.4'),
project(':iceberg-spark:iceberg-spark-runtime-2.4')
]

configure(sparkProjects) {
Expand All @@ -37,7 +37,7 @@ configure(sparkProjects) {
}
}

project(':iceberg-spark:iceberg-spark2') {
project(':iceberg-spark:iceberg-spark-2.4') {
configurations.all {
resolutionStrategy {
// Spark 2.4.4 can only use the below datanucleus version, the versions introduced
Expand Down Expand Up @@ -105,7 +105,7 @@ project(':iceberg-spark:iceberg-spark2') {
}

// the runtime jar is a self-contained artifact for testing in a notebook
project(':iceberg-spark:iceberg-spark-runtime') {
project(':iceberg-spark:iceberg-spark-runtime-2.4') {
apply plugin: 'com.github.johnrengelman.shadow'

tasks.jar.dependsOn tasks.shadowJar
Expand All @@ -125,7 +125,7 @@ project(':iceberg-spark:iceberg-spark-runtime') {
}

dependencies {
implementation project(':iceberg-spark:iceberg-spark2')
implementation project(':iceberg-spark:iceberg-spark-2.4')
implementation project(':iceberg-aws')
implementation(project(':iceberg-aliyun')) {
exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
Expand Down
28 changes: 14 additions & 14 deletions spark/v3.0/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
*/

def sparkProjects = [
project(':iceberg-spark:iceberg-spark3'),
project(":iceberg-spark:iceberg-spark3-extensions"),
project(':iceberg-spark:iceberg-spark3-runtime')
project(':iceberg-spark:iceberg-spark-3.0_2.12'),
project(":iceberg-spark:iceberg-spark-extensions-3.0_2.12"),
project(':iceberg-spark:iceberg-spark-runtime-3.0_2.12')
]

configure(sparkProjects) {
Expand All @@ -38,7 +38,7 @@ configure(sparkProjects) {
}
}

project(':iceberg-spark:iceberg-spark3') {
project(':iceberg-spark:iceberg-spark-3.0_2.12') {
apply plugin: 'scala'
apply plugin: 'com.github.alisiikh.scalastyle'

Expand Down Expand Up @@ -105,7 +105,7 @@ project(':iceberg-spark:iceberg-spark3') {
}
}

project(":iceberg-spark:iceberg-spark3-extensions") {
project(":iceberg-spark:iceberg-spark-extensions-3.0_2.12") {
apply plugin: 'java-library'
apply plugin: 'scala'
apply plugin: 'com.github.alisiikh.scalastyle'
Expand All @@ -132,7 +132,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") {
compileOnly project(':iceberg-data')
compileOnly project(':iceberg-orc')
compileOnly project(':iceberg-common')
compileOnly project(':iceberg-spark:iceberg-spark3')
compileOnly project(':iceberg-spark:iceberg-spark-3.0_2.12')
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.arrow'
Expand All @@ -146,7 +146,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") {
testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark-3.0_2.12', configuration: 'testArtifacts')

testImplementation "org.apache.avro:avro"

Expand All @@ -162,7 +162,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") {
}
}

project(':iceberg-spark:iceberg-spark3-runtime') {
project(':iceberg-spark:iceberg-spark-runtime-3.0_2.12') {
apply plugin: 'com.github.johnrengelman.shadow'

tasks.jar.dependsOn tasks.shadowJar
Expand Down Expand Up @@ -196,8 +196,8 @@ project(':iceberg-spark:iceberg-spark3-runtime') {

dependencies {
api project(':iceberg-api')
implementation project(':iceberg-spark:iceberg-spark3')
implementation project(':iceberg-spark:iceberg-spark3-extensions')
implementation project(':iceberg-spark:iceberg-spark-3.0_2.12')
implementation project(':iceberg-spark:iceberg-spark-extensions-3.0_2.12')
implementation project(':iceberg-aws')
implementation(project(':iceberg-aliyun')) {
exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
Expand All @@ -214,11 +214,11 @@ project(':iceberg-spark:iceberg-spark3-runtime') {
integrationImplementation 'org.slf4j:slf4j-simple'
integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark3', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark3-extensions', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.0_2.12', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-extensions-3.0_2.12', configuration: 'testArtifacts')
// Not allowed on our classpath, only the runtime jar is allowed
integrationCompileOnly project(':iceberg-spark:iceberg-spark3-extensions')
integrationCompileOnly project(':iceberg-spark:iceberg-spark3')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-extensions-3.0_2.12')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.0_2.12')
integrationCompileOnly project(':iceberg-api')
}

Expand Down