diff --git a/.travis.yml b/.travis.yml index ff97566e1837..06d940b51f53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,20 @@ dist: bionic matrix: include: + - language: java + jdk: openjdk11 + name: java11 + before_cache: + - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock + - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ + cache: + directories: + - $HOME/.gradle/caches/ + - $HOME/.gradle/wrapper/ + - language: java jdk: openjdk8 - name: java + name: java8 before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ diff --git a/build.gradle b/build.gradle index 21e0df5d4f7b..2eac69e6d2fc 100644 --- a/build.gradle +++ b/build.gradle @@ -41,12 +41,14 @@ plugins { id 'nebula.dependency-lock' version '9.0.0' } -if (JavaVersion.current() != JavaVersion.VERSION_1_8) { - throw new GradleException("This build must be run with Java 8") +if (JavaVersion.current() == JavaVersion.VERSION_1_8) { + project.ext.jdkVersion = '8' +} else if (JavaVersion.current() == JavaVersion.VERSION_11) { + project.ext.jdkVersion = '11' +} else { + throw new GradleException("This build must be run with JDK 8 or 11") } -apply from: 'jmh.gradle' - dependencyRecommendations { propertiesFile file: file('versions.props') } @@ -136,7 +138,7 @@ project(':iceberg-bundled-guava') { tasks.jar.dependsOn tasks.shadowJar - dependencies { + dependencies { compileOnly('com.google.guava:guava') { exclude group: 'com.google.code.findbugs' // may be LGPL - use ALv2 findbugs-annotations instead @@ -185,7 +187,7 @@ project(':iceberg-common') { compile project(path: ':iceberg-bundled-guava', configuration: 'shadow') } } - + project(':iceberg-core') { dependencies { compile project(':iceberg-api') @@ -462,56 +464,122 @@ project(':iceberg-spark') { } } -project(':iceberg-spark2') { +if (jdkVersion == '8') { + apply from: 'jmh.gradle' - configurations.all { - resolutionStrategy { - // Spark 2.4.4 can only use the below datanucleus version, the versions introduced - // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions - // introduced by Hive 1.2.1. - force 'org.datanucleus:datanucleus-api-jdo:3.2.6' - force 'org.datanucleus:datanucleus-core:3.2.10' - force 'org.datanucleus:datanucleus-rdbms:3.2.9' + project(':iceberg-spark2') { + configurations.all { + resolutionStrategy { + // Spark 2.4.4 can only use the below datanucleus version, the versions introduced + // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions + // introduced by Hive 1.2.1. + force 'org.datanucleus:datanucleus-api-jdo:3.2.6' + force 'org.datanucleus:datanucleus-core:3.2.10' + force 'org.datanucleus:datanucleus-rdbms:3.2.9' + } } - } - dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-data') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-arrow') - compile project(':iceberg-hive') - compile project(':iceberg-spark') + dependencies { + compile project(':iceberg-api') + compile project(':iceberg-common') + compile project(':iceberg-core') + compile project(':iceberg-data') + compile project(':iceberg-orc') + compile project(':iceberg-parquet') + compile project(':iceberg-arrow') + compile project(':iceberg-hive') + compile project(':iceberg-spark') + + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_2.11") { + exclude group: 'org.apache.avro', module: 'avro' + } - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.11") { - exclude group: 'org.apache.avro', module: 'avro' + testCompile project(path: ':iceberg-spark', configuration: 'testArtifacts') + + testCompile "org.apache.hadoop:hadoop-hdfs::tests" + testCompile "org.apache.hadoop:hadoop-common::tests" + testCompile("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } - testCompile project(path: ':iceberg-spark', configuration: 'testArtifacts') + test { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") - testCompile "org.apache.hadoop:hadoop-hdfs::tests" - testCompile "org.apache.hadoop:hadoop-common::tests" - testCompile("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' + // Vectorized reads need more memory + maxHeapSize '2500m' } - testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } - test { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") + // the runtime jar is a self-contained artifact for testing in a notebook + project(':iceberg-spark-runtime') { + apply plugin: 'com.github.johnrengelman.shadow' - // Vectorized reads need more memory - maxHeapSize '2500m' + tasks.jar.dependsOn tasks.shadowJar + + configurations { + compile { + exclude group: 'org.apache.spark' + // included in Spark + exclude group: 'org.slf4j' + exclude group: 'org.apache.commons' + exclude group: 'commons-pool' + exclude group: 'commons-codec' + exclude group: 'org.xerial.snappy' + exclude group: 'javax.xml.bind' + exclude group: 'javax.annotation' + } + } + + dependencies { + compile project(':iceberg-spark2') + compile 'org.apache.spark:spark-hive_2.11' + } + + shadowJar { + configurations = [project.configurations.compile] + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' + relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' + relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' + relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' + relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' + relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' + relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' + relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' + // relocate Avro's jackson dependency to share parquet-jackson locations + relocate 'org.codehaus.jackson', 'org.apache.iceberg.shaded.org.apache.parquet.shaded.org.codehaus.jackson' + relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' + relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' + // relocate Arrow and related deps to shade Iceberg specific version + relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' + relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' + relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' + + classifier null + } + + jar { + classifier = 'empty' + } } } @@ -582,69 +650,6 @@ project(':iceberg-pig') { } } -// the runtime jar is a self-contained artifact for testing in a notebook -project(':iceberg-spark-runtime') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.jar.dependsOn tasks.shadowJar - - configurations { - compile { - exclude group: 'org.apache.spark' - // included in Spark - exclude group: 'org.slf4j' - exclude group: 'org.apache.commons' - exclude group: 'commons-pool' - exclude group: 'commons-codec' - exclude group: 'org.xerial.snappy' - exclude group: 'javax.xml.bind' - exclude group: 'javax.annotation' - } - } - - dependencies { - compile project(':iceberg-spark2') - compile 'org.apache.spark:spark-hive_2.11' - } - - shadowJar { - configurations = [project.configurations.compile] - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' - relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' - relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' - relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' - relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' - relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' - relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' - relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' - // relocate Avro's jackson dependency to share parquet-jackson locations - relocate 'org.codehaus.jackson', 'org.apache.iceberg.shaded.org.apache.parquet.shaded.org.codehaus.jackson' - relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' - relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' - // relocate Arrow and related deps to shade Iceberg specific version - relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' - relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' - relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' - - classifier null - } - - jar { - classifier = 'empty' - } -} - @Memoized boolean isVersionFileExists() { return file('version.txt').exists() diff --git a/deploy.gradle b/deploy.gradle index 358886c91664..f5adbd8ad27f 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -17,6 +17,10 @@ * under the License. */ +if (project.hasProperty('release') && jdkVersion != JavaVersion.VERSION_1_8) { + throw new GradleException("Releases must be built with Java 8") +} + subprojects { apply plugin: 'maven' // make pom files for deployment apply plugin: 'maven-publish' diff --git a/settings.gradle b/settings.gradle index 1eeb9c65b7ea..275a991f24e3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,10 +29,8 @@ include 'arrow' include 'parquet' include 'bundled-guava' include 'spark' -include 'spark2' include 'spark3' include 'spark3-runtime' -include 'spark-runtime' include 'pig' include 'hive' @@ -47,10 +45,15 @@ project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' project(':bundled-guava').name = 'iceberg-bundled-guava' project(':spark').name = 'iceberg-spark' -project(':spark2').name = 'iceberg-spark2' -project(':spark-runtime').name = 'iceberg-spark-runtime' project(':spark3').name = 'iceberg-spark3' project(':spark3-runtime').name = 'iceberg-spark3-runtime' project(':pig').name = 'iceberg-pig' project(':hive').name = 'iceberg-hive' +if (JavaVersion.current() == JavaVersion.VERSION_1_8) { + include 'spark2' + include 'spark-runtime' + + project(':spark2').name = 'iceberg-spark2' + project(':spark-runtime').name = 'iceberg-spark-runtime' +}