diff --git a/build.gradle b/build.gradle index 2156dbccc898..efbd7d0d1358 100644 --- a/build.gradle +++ b/build.gradle @@ -338,6 +338,7 @@ project(':iceberg-mr') { compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } + compileOnly("org.apache.hive:hive-serde") testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') diff --git a/mr/dependencies.lock b/mr/dependencies.lock index b0e2c279c6ff..9fbd7551d7a1 100644 --- a/mr/dependencies.lock +++ b/mr/dependencies.lock @@ -1,4 +1,365 @@ { + "allProcessors": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "annotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "checkstyle": { + "antlr:antlr": { + "locked": "2.7.7", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.1.3", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "26.0-jre", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.puppycrawl.tools:checkstyle": { + "locked": "8.13" + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.9.3", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.4", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-collections:commons-collections": { + "locked": "3.2.2", + "transitive": [ + "commons-beanutils:commons-beanutils" + ] + }, + "net.sf.saxon:Saxon-HE": { + "locked": "9.8.0-14", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.antlr:antlr4-runtime": { + "locked": "4.7.1", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.14", + "transitive": [ + "com.google.guava:guava" + ] + } + }, "compile": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -225,6 +586,12 @@ } }, "compileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -234,10 +601,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -255,7 +635,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -265,6 +647,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -279,7 +667,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -315,7 +704,20 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -338,6 +740,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -345,6 +748,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -353,6 +757,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -362,6 +767,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -380,18 +791,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -414,6 +830,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -437,17 +859,24 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -476,20 +905,48 @@ "org.apache.orc:orc-core" ] }, - "io.netty:netty": { - "locked": "3.7.0.Final", + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.zookeeper:zookeeper" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" ] }, - "io.netty:netty-all": { - "locked": "4.0.23.Final", + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-common" ] }, - "javax.annotation:javax.annotation-api": { + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.netty:netty": { + "locked": "3.7.0.Final", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs", + "org.apache.zookeeper:zookeeper" + ] + }, + "io.netty:netty-all": { + "locked": "4.0.23.Final", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ "org.apache.parquet:parquet-format-structures" @@ -502,11 +959,26 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -519,11 +991,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -534,9 +1013,35 @@ "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -545,7 +1050,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -565,7 +1077,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -598,6 +1111,24 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ @@ -617,6 +1148,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -700,6 +1232,40 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -708,15 +1274,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -758,9 +1328,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -810,12 +1416,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -830,7 +1457,8 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -883,6 +1511,18 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.fusesource.leveldbjni:leveldbjni-all": { "locked": "1.8", "transitive": [ @@ -901,6 +1541,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -916,16 +1560,24 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -947,6 +1599,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -968,6 +1632,12 @@ } }, "compileOnly": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -977,14 +1647,53 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.6.0", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-databind": { + "locked": "2.6.5", + "transitive": [ + "io.dropwizard.metrics:metrics-json", + "org.apache.hive:hive-common" + ] + }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -1004,16 +1713,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -1028,22 +1753,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -1052,21 +1796,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -1076,6 +1827,18 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.thoughtworks.paranamer:paranamer": { + "locked": "2.3", + "transitive": [ + "org.apache.avro:avro" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -1094,18 +1857,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -1113,7 +1881,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -1128,6 +1897,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -1139,7 +1914,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -1151,24 +1927,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -1178,6 +1966,39 @@ "org.apache.hadoop:hadoop-common" ] }, + "io.airlift:aircompressor": { + "locked": "0.8", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "io.airlift:slice": { + "locked": "0.29", + "transitive": [ + "io.airlift:aircompressor" + ] + }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -1194,7 +2015,9 @@ "javax.activation:activation": { "locked": "1.1", "transitive": [ - "javax.xml.bind:jaxb-api" + "javax.mail:mail", + "javax.xml.bind:jaxb-api", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.inject:javax.inject": { @@ -1204,17 +2027,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -1222,7 +2060,9 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "javax.xml.stream:stax-api": { @@ -1232,11 +2072,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -1245,14 +2092,54 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, + "org.apache.avro:avro": { + "locked": "1.7.7", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" + ] + }, "org.apache.commons:commons-compress": { - "locked": "1.4.1", + "locked": "1.9", "transitive": [ + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -1272,7 +2159,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -1305,11 +2193,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -1324,6 +2235,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -1357,73 +2269,214 @@ "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hadoop:hadoop-client" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + ] + }, + "org.apache.hadoop:hadoop-yarn-api": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-client": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common" + ] + }, + "org.apache.hadoop:hadoop-yarn-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-nodemanager": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.httpcomponents:httpcore": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-api": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-client": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-common": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-common": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-nodemanager": { - "locked": "2.7.3", + "org.apache.orc:orc-core": { + "locked": "1.3.4", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hive:hive-common" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-serde" ] }, - "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "org.apache.thrift:libfb303": { + "locked": "0.9.3", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hive:hive-service-rpc" ] }, - "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "org.apache.thrift:libthrift": { + "locked": "0.9.3", "transitive": [ - "org.apache.httpcomponents:httpclient" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" ] }, "org.apache.zookeeper:zookeeper": { @@ -1434,13 +2487,16 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.codehaus.jackson:jackson-core-asl": { "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", @@ -1460,6 +2516,7 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", @@ -1478,7 +2535,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -1486,13 +2557,26 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.openjdk.jol:jol-core": { + "locked": "0.2", + "transitive": [ + "io.airlift:slice" ] }, "org.slf4j:slf4j-api": { - "locked": "1.7.10", + "locked": "1.7.21", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", "org.apache.directory.api:api-util", @@ -1507,6 +2591,18 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.orc:orc-core", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -1516,10 +2612,22 @@ "com.google.inject:guice" ] }, - "org.tukaani:xz": { - "locked": "1.0", + "org.xerial.snappy:snappy-java": { + "locked": "1.0.5", + "transitive": [ + "org.apache.avro:avro" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", "transitive": [ - "org.apache.commons:commons-compress" + "org.apache.hive:hive-service-rpc" ] }, "xerces:xercesImpl": { @@ -1767,6 +2875,154 @@ ] } }, + "errorprone": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "errorproneJavac": { + "com.google.errorprone:javac": { + "locked": "9+181-r4173-1", + "requested": "9+181-r4173-1" + } + }, "runtime": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -2217,7 +3473,155 @@ ] } }, + "testAnnotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, "testCompile": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -2227,10 +3631,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -2248,7 +3665,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -2258,6 +3677,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -2272,7 +3697,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -2298,16 +3724,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -2322,22 +3764,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -2346,21 +3807,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -2370,6 +3838,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -2388,18 +3862,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -2407,7 +3886,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -2422,6 +3902,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -2433,7 +3919,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -2445,24 +3932,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -2484,6 +3983,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -2497,6 +4017,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -2510,17 +4037,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -2528,16 +4070,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -2549,12 +4100,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -2563,7 +4141,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -2583,7 +4168,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -2616,11 +4202,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -2635,6 +4244,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -2684,8 +4294,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -2700,8 +4313,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -2709,7 +4331,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -2718,6 +4343,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -2726,15 +4411,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -2776,9 +4465,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -2828,12 +4553,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -2848,7 +4594,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -2898,7 +4646,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -2906,8 +4668,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -2935,6 +4699,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -2950,16 +4718,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -2985,6 +4764,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3006,6 +4797,12 @@ } }, "testCompileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -3015,10 +4812,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -3036,7 +4846,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -3046,6 +4858,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -3060,7 +4878,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -3087,7 +4906,8 @@ "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "com.google.inject:guice": { @@ -3111,7 +4931,20 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -3134,6 +4967,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -3141,6 +4975,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -3149,6 +4984,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -3158,6 +4994,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -3176,18 +5018,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -3210,6 +5057,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -3233,17 +5086,24 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -3272,6 +5132,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -3285,6 +5166,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -3298,11 +5186,26 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -3315,11 +5218,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -3333,9 +5243,35 @@ "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -3344,7 +5280,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -3364,7 +5307,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -3397,6 +5341,24 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ @@ -3416,6 +5378,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -3499,6 +5462,40 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -3507,15 +5504,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -3557,9 +5558,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -3609,12 +5646,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -3629,7 +5687,8 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -3682,6 +5741,18 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.fusesource.leveldbjni:leveldbjni-all": { "locked": "1.8", "transitive": [ @@ -3709,6 +5780,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -3724,16 +5799,24 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -3759,6 +5842,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3780,6 +5875,12 @@ } }, "testRuntime": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -3789,10 +5890,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -3810,7 +5924,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -3820,6 +5936,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -3834,7 +5956,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -3860,16 +5983,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -3884,22 +6023,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -3908,21 +6066,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -3932,6 +6097,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -3950,18 +6121,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -3969,7 +6145,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -3984,6 +6161,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -3995,7 +6178,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -4007,24 +6191,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -4046,6 +6242,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -4059,6 +6276,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -4072,17 +6296,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -4090,16 +6329,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -4111,12 +6359,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -4125,7 +6400,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -4145,7 +6427,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -4178,11 +6461,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -4197,6 +6503,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -4246,8 +6553,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -4262,8 +6572,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -4271,7 +6590,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -4280,6 +6602,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -4288,15 +6670,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -4338,9 +6724,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -4390,12 +6812,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -4410,7 +6853,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -4460,7 +6905,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -4468,8 +6927,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -4497,6 +6958,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -4512,16 +6977,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -4547,6 +7023,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -4568,6 +7056,12 @@ } }, "testRuntimeClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -4577,10 +7071,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -4598,7 +7105,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -4608,6 +7117,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -4622,7 +7137,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -4648,16 +7164,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -4672,22 +7204,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -4696,21 +7247,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -4720,6 +7278,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -4738,18 +7302,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -4757,7 +7326,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -4772,6 +7342,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -4783,7 +7359,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -4795,24 +7372,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -4834,6 +7423,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -4847,6 +7457,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -4860,17 +7477,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -4878,16 +7510,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -4899,12 +7540,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -4913,7 +7581,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -4933,7 +7608,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -4966,11 +7642,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -4985,6 +7684,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -5034,8 +7734,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -5050,8 +7753,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -5059,7 +7771,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -5068,6 +7783,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -5076,15 +7851,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -5126,9 +7905,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -5178,12 +7993,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -5198,7 +8034,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -5248,7 +8086,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -5256,8 +8108,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -5285,6 +8139,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -5300,16 +8158,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -5335,6 +8204,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java new file mode 100644 index 000000000000..f4b9aae0f55a --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.expressions.Expression; + +public class InputFormatConfig { + + private InputFormatConfig() {} + + // configuration values for Iceberg input formats + public static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; + public static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; + public static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; + public static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; + public static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; + public static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; + public static final String READ_SCHEMA = "iceberg.mr.read.schema"; + public static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; + public static final String SPLIT_SIZE = "iceberg.mr.split.size"; + public static final String TABLE_PATH = "iceberg.mr.table.path"; + public static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; + public static final String LOCALITY = "iceberg.mr.locality"; + public static final String CATALOG = "iceberg.mr.catalog"; + + public static final String CATALOG_NAME = "iceberg.catalog"; + public static final String HADOOP_CATALOG = "hadoop.catalog"; + public static final String HADOOP_TABLES = "hadoop.tables"; + public static final String HIVE_CATALOG = "hive.catalog"; + public static final String ICEBERG_SNAPSHOTS_TABLE_SUFFIX = ".snapshots"; + public static final String SNAPSHOT_TABLE = "iceberg.snapshots.table"; + public static final String SNAPSHOT_TABLE_SUFFIX = "__snapshots"; + public static final String TABLE_LOCATION = "location"; + public static final String TABLE_NAME = "name"; + + public enum InMemoryDataModel { + PIG, + HIVE, + GENERIC // Default data model is of Iceberg Generics + } + + public static class ConfigBuilder { + private final Configuration conf; + + public ConfigBuilder(Configuration conf) { + this.conf = conf; + // defaults + conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); + conf.setBoolean(CASE_SENSITIVE, true); + conf.setBoolean(REUSE_CONTAINERS, false); + conf.setBoolean(LOCALITY, false); + } + + public ConfigBuilder filter(Expression expression) { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfigBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder schema(Schema schema) { + conf.set(TABLE_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + return this; + } + + public ConfigBuilder reuseContainers(boolean reuse) { + conf.setBoolean(InputFormatConfig.REUSE_CONTAINERS, reuse); + return this; + } + + public ConfigBuilder caseSensitive(boolean caseSensitive) { + conf.setBoolean(InputFormatConfig.CASE_SENSITIVE, caseSensitive); + return this; + } + + public ConfigBuilder snapshotId(long snapshotId) { + conf.setLong(SNAPSHOT_ID, snapshotId); + return this; + } + + public ConfigBuilder asOfTime(long asOfTime) { + conf.setLong(AS_OF_TIMESTAMP, asOfTime); + return this; + } + + public ConfigBuilder splitSize(long splitSize) { + conf.setLong(SPLIT_SIZE, splitSize); + return this; + } + + /** + * If this API is called. The input splits constructed will have host location information + */ + public ConfigBuilder preferLocality() { + conf.setBoolean(LOCALITY, true); + return this; + } + + public ConfigBuilder catalogFunc(Class> catalogFuncClass) { + conf.setClass(CATALOG, catalogFuncClass, Function.class); + return this; + } + + public ConfigBuilder useHiveRows() { + conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name()); + return this; + } + + public ConfigBuilder usePigTuples() { + conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.PIG.name()); + return this; + } + + /** + * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only + * partially applies the filter, it will return the residual filter back. If the platform can correctly apply the + * residual filters, then it should call this api. Otherwise the current api will throw an exception if the passed + * in filter is not completely satisfied. + */ + public ConfigBuilder skipResidualFiltering() { + conf.setBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, true); + return this; + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java new file mode 100644 index 000000000000..871e0e677835 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Properties; +import javax.annotation.Nullable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Table; +import org.apache.iceberg.mr.mapred.serde.objectinspector.IcebergObjectInspector; + +public class IcebergSerDe extends AbstractSerDe { + + private ObjectInspector inspector; + + @Override + public void initialize(@Nullable Configuration configuration, Properties serDeProperties) throws SerDeException { + final Table table; + + try { + table = TableResolver.resolveTableFromConfiguration(configuration, serDeProperties); + } catch (IOException e) { + throw new UncheckedIOException("Unable to resolve table from configuration: ", e); + } + + try { + this.inspector = IcebergObjectInspector.create(table.schema()); + } catch (Exception e) { + throw new SerDeException(e); + } + } + + @Override + public Class getSerializedClass() { + return IcebergWritable.class; + } + + @Override + public Writable serialize(Object o, ObjectInspector objectInspector) { + throw new UnsupportedOperationException("Serialization is not supported."); + } + + @Override + public SerDeStats getSerDeStats() { + return null; + } + + @Override + public Object deserialize(Writable writable) { + return ((IcebergWritable) writable).record(); + } + + @Override + public ObjectInspector getObjectInspector() { + return inspector; + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java new file mode 100644 index 000000000000..1eb67f9d3158 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.DataInput; +import java.io.DataOutput; +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.Record; + +/** + * Wraps an Iceberg Record in a Writable which Hive can use in the SerDe. + */ +public class IcebergWritable implements Writable { + + private Record record; + private Schema schema; + + public IcebergWritable(Record record, Schema schema) { + this.record = record; + this.schema = schema; + } + + @SuppressWarnings("checkstyle:HiddenField") + public void wrapRecord(Record record) { + this.record = record; + } + + public Record record() { + return record; + } + + public Schema schema() { + return schema; + } + + @SuppressWarnings("checkstyle:HiddenField") + public void wrapSchema(Schema schema) { + this.schema = schema; + } + + @Override + public void write(DataOutput dataOutput) { + throw new UnsupportedOperationException("write is not supported."); + } + + @Override + public void readFields(DataInput dataInput) { + throw new UnsupportedOperationException("readFields is not supported."); + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java new file mode 100644 index 000000000000..7ee670d5e782 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +final class TableResolver { + + private TableResolver() { + } + + static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { + Configuration configuration = new Configuration(conf); + for (Map.Entry entry : properties.entrySet()) { + configuration.set(entry.getKey().toString(), entry.getValue().toString()); + } + return resolveTableFromConfiguration(configuration); + } + + static Table resolveTableFromConfiguration(Configuration conf) throws IOException { + //Default to HadoopTables + String catalogName = conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + + switch (catalogName) { + case InputFormatConfig.HADOOP_TABLES: + String tableLocation = conf.get(InputFormatConfig.TABLE_LOCATION); + Preconditions.checkNotNull(tableLocation, InputFormatConfig.TABLE_LOCATION + " is not set."); + HadoopTables tables = new HadoopTables(conf); + return tables.load(tableLocation); + + case InputFormatConfig.HIVE_CATALOG: + String tableName = conf.get(InputFormatConfig.TABLE_NAME); + Preconditions.checkNotNull(tableName, InputFormatConfig.TABLE_NAME + " is not set."); + throw new UnsupportedOperationException(InputFormatConfig.HIVE_CATALOG + " is not supported yet"); + + default: + throw new NoSuchNamespaceException("Catalog " + catalogName + " not supported."); + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java new file mode 100644 index 000000000000..6ec48ebc6370 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.BytesWritable; +import org.apache.iceberg.util.ByteBuffers; + +public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements BinaryObjectInspector { + + private static final IcebergBinaryObjectInspector BYTE_ARRAY = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return (byte[]) o; + } + }; + + private static final IcebergBinaryObjectInspector BYTE_BUFFER = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return ByteBuffers.toByteArray((ByteBuffer) o); + } + }; + + public static IcebergBinaryObjectInspector byteArray() { + return BYTE_ARRAY; + } + + public static IcebergBinaryObjectInspector byteBuffer() { + return BYTE_BUFFER; + } + + private IcebergBinaryObjectInspector() { + super(TypeInfoFactory.binaryTypeInfo); + } + + abstract byte[] toByteArray(Object object); + + @Override + public byte[] getPrimitiveJavaObject(Object o) { + return toByteArray(o); + } + + @Override + public BytesWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new BytesWritable(getPrimitiveJavaObject(o)); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + byte[] bytes = (byte[]) o; + return Arrays.copyOf(bytes, bytes.length); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java new file mode 100644 index 000000000000..2af9a172e963 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Date; +import java.time.LocalDate; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.util.DateTimeUtil; + +public final class IcebergDateObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements DateObjectInspector { + + private static final IcebergDateObjectInspector INSTANCE = new IcebergDateObjectInspector(); + + public static IcebergDateObjectInspector get() { + return INSTANCE; + } + + private IcebergDateObjectInspector() { + super(TypeInfoFactory.dateTypeInfo); + } + + @Override + public Date getPrimitiveJavaObject(Object o) { + return o == null ? null : Date.valueOf((LocalDate) o); + } + + @Override + public DateWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new DateWritable(DateTimeUtil.daysFromDate((LocalDate) o)); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new Date(((Date) o).getTime()); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java new file mode 100644 index 000000000000..fa8f91c2e99c --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.math.BigDecimal; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +public final class IcebergDecimalObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements HiveDecimalObjectInspector { + + private static final Cache CACHE = Caffeine.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + + public static IcebergDecimalObjectInspector get(int precision, int scale) { + Preconditions.checkArgument(scale < precision); + Preconditions.checkArgument(precision <= HiveDecimal.MAX_PRECISION); + Preconditions.checkArgument(scale <= HiveDecimal.MAX_SCALE); + + Integer key = precision << 8 | scale; + return CACHE.get(key, k -> new IcebergDecimalObjectInspector(precision, scale)); + } + + private IcebergDecimalObjectInspector(int precision, int scale) { + super(new DecimalTypeInfo(precision, scale)); + } + + @Override + public HiveDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : HiveDecimal.create((BigDecimal) o); + } + + @Override + public HiveDecimalWritable getPrimitiveWritableObject(Object o) { + HiveDecimal decimal = getPrimitiveJavaObject(o); + return decimal == null ? null : new HiveDecimalWritable(decimal); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + HiveDecimal decimal = (HiveDecimal) o; + return HiveDecimal.create(decimal.bigDecimalValue()); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java new file mode 100644 index 000000000000..5e251f7092a9 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.util.List; +import javax.annotation.Nullable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; + +public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor { + + public static ObjectInspector create(@Nullable Schema schema) { + if (schema == null) { + return IcebergRecordObjectInspector.empty(); + } + + return TypeUtil.visit(schema, new IcebergObjectInspector()); + } + + public static ObjectInspector create(Types.NestedField... fields) { + return create(new Schema(fields)); + } + + @Override + public ObjectInspector field(Types.NestedField field, ObjectInspector fieldObjectInspector) { + return fieldObjectInspector; + } + + @Override + public ObjectInspector list(Types.ListType listTypeInfo, ObjectInspector listObjectInspector) { + return ObjectInspectorFactory.getStandardListObjectInspector(listObjectInspector); + } + + @Override + public ObjectInspector map(Types.MapType mapType, + ObjectInspector keyObjectInspector, ObjectInspector valueObjectInspector) { + return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); + } + + @Override + public ObjectInspector primitive(Type.PrimitiveType primitiveType) { + final PrimitiveTypeInfo primitiveTypeInfo; + + switch (primitiveType.typeId()) { + case BINARY: + return IcebergBinaryObjectInspector.byteBuffer(); + case BOOLEAN: + primitiveTypeInfo = TypeInfoFactory.booleanTypeInfo; + break; + case DATE: + return IcebergDateObjectInspector.get(); + case DECIMAL: + Types.DecimalType type = (Types.DecimalType) primitiveType; + return IcebergDecimalObjectInspector.get(type.precision(), type.scale()); + case DOUBLE: + primitiveTypeInfo = TypeInfoFactory.doubleTypeInfo; + break; + case FIXED: + return IcebergBinaryObjectInspector.byteArray(); + case FLOAT: + primitiveTypeInfo = TypeInfoFactory.floatTypeInfo; + break; + case INTEGER: + primitiveTypeInfo = TypeInfoFactory.intTypeInfo; + break; + case LONG: + primitiveTypeInfo = TypeInfoFactory.longTypeInfo; + break; + case STRING: + case UUID: + primitiveTypeInfo = TypeInfoFactory.stringTypeInfo; + break; + case TIMESTAMP: + boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC(); + return IcebergTimestampObjectInspector.get(adjustToUTC); + + case TIME: + default: + throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); + } + + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveTypeInfo); + } + + @Override + public ObjectInspector schema(Schema schema, ObjectInspector structObjectInspector) { + return structObjectInspector; + } + + @Override + public ObjectInspector struct(Types.StructType structType, List fieldObjectInspectors) { + return new IcebergRecordObjectInspector(structType, fieldObjectInspectors); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java new file mode 100644 index 000000000000..c49396b1f168 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types; + +public final class IcebergRecordObjectInspector extends StructObjectInspector { + + private static final IcebergRecordObjectInspector EMPTY = + new IcebergRecordObjectInspector(Types.StructType.of(), Collections.emptyList()); + + private final List structFields; + + public IcebergRecordObjectInspector(Types.StructType structType, List objectInspectors) { + Preconditions.checkArgument(structType.fields().size() == objectInspectors.size()); + + this.structFields = Lists.newArrayListWithExpectedSize(structType.fields().size()); + + int position = 0; + + for (Types.NestedField field : structType.fields()) { + ObjectInspector oi = objectInspectors.get(position); + IcebergRecordStructField structField = new IcebergRecordStructField(field, oi, position); + structFields.add(structField); + position++; + } + } + + public static IcebergRecordObjectInspector empty() { + return EMPTY; + } + + @Override + public List getAllStructFieldRefs() { + return structFields; + } + + @Override + public StructField getStructFieldRef(String name) { + return ObjectInspectorUtils.getStandardStructFieldRef(name, structFields); + } + + @Override + public Object getStructFieldData(Object o, StructField structField) { + return ((Record) o).get(((IcebergRecordStructField) structField).position()); + } + + @Override + public List getStructFieldsDataAsList(Object o) { + Record record = (Record) o; + return structFields + .stream() + .map(f -> record.get(f.position())) + .collect(Collectors.toList()); + } + + @Override + public String getTypeName() { + return ObjectInspectorUtils.getStandardStructTypeName(this); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + IcebergRecordObjectInspector that = (IcebergRecordObjectInspector) o; + return structFields.equals(that.structFields); + } + + @Override + public int hashCode() { + return structFields.hashCode(); + } + + private static class IcebergRecordStructField implements StructField { + + private final Types.NestedField field; + private final ObjectInspector oi; + private final int position; + + IcebergRecordStructField(Types.NestedField field, ObjectInspector oi, int position) { + this.field = field; + this.oi = oi; + this.position = position; // position in the record + } + + @Override + public String getFieldName() { + return field.name(); + } + + @Override + public ObjectInspector getFieldObjectInspector() { + return oi; + } + + @Override + public int getFieldID() { + return field.fieldId(); + } + + @Override + public String getFieldComment() { + return field.doc(); + } + + int position() { + return position; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + IcebergRecordStructField that = (IcebergRecordStructField) o; + return field.equals(that.field) && oi.equals(that.oi) && position == that.position; + } + + @Override + public int hashCode() { + return Objects.hash(field, oi, position); + } + + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java new file mode 100644 index 000000000000..974e59f69194 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements TimestampObjectInspector { + + private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return ((OffsetDateTime) o).toLocalDateTime(); + } + }; + + private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return (LocalDateTime) o; + } + }; + + public static IcebergTimestampObjectInspector get(boolean adjustToUTC) { + return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE; + } + + private IcebergTimestampObjectInspector() { + super(TypeInfoFactory.timestampTypeInfo); + } + + + abstract LocalDateTime toLocalDateTime(Object object); + + @Override + public Timestamp getPrimitiveJavaObject(Object o) { + return o == null ? null : Timestamp.valueOf(toLocalDateTime(o)); + } + + @Override + public TimestampWritable getPrimitiveWritableObject(Object o) { + Timestamp ts = getPrimitiveJavaObject(o); + return ts == null ? null : new TimestampWritable(ts); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + Timestamp ts = (Timestamp) o; + Timestamp copy = new Timestamp(ts.getTime()); + copy.setNanos(ts.getNanos()); + return copy; + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index b209a80e4fee..e8f76ab63cd2 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -65,6 +65,7 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; @@ -78,133 +79,23 @@ /** * Generic Mrv2 InputFormat API for Iceberg. + * * @param T is the in memory data model which can either be Pig tuples, Hive rows. Default is Iceberg records */ public class IcebergInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); - static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; - static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; - static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; - static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; - static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; - static final String SPLIT_SIZE = "iceberg.mr.split.size"; - static final String TABLE_PATH = "iceberg.mr.table.path"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - static final String LOCALITY = "iceberg.mr.locality"; - static final String CATALOG = "iceberg.mr.catalog"; - static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; - private transient List splits; - private enum InMemoryDataModel { - PIG, - HIVE, - GENERIC // Default data model is of Iceberg Generics - } - /** * Configures the {@code Job} to use the {@code IcebergInputFormat} and * returns a helper to add further configuration. * * @param job the {@code Job} to configure */ - public static ConfigBuilder configure(Job job) { + public static InputFormatConfig.ConfigBuilder configure(Job job) { job.setInputFormatClass(IcebergInputFormat.class); - return new ConfigBuilder(job.getConfiguration()); - } - - public static class ConfigBuilder { - private final Configuration conf; - - public ConfigBuilder(Configuration conf) { - this.conf = conf; - // defaults - conf.setEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); - conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); - conf.setBoolean(CASE_SENSITIVE, true); - conf.setBoolean(REUSE_CONTAINERS, false); - conf.setBoolean(LOCALITY, false); - } - - public ConfigBuilder readFrom(String path) { - conf.set(TABLE_PATH, path); - Table table = findTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - return this; - } - - public ConfigBuilder filter(Expression expression) { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfigBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - public ConfigBuilder reuseContainers(boolean reuse) { - conf.setBoolean(REUSE_CONTAINERS, reuse); - return this; - } - - public ConfigBuilder caseSensitive(boolean caseSensitive) { - conf.setBoolean(CASE_SENSITIVE, caseSensitive); - return this; - } - - public ConfigBuilder snapshotId(long snapshotId) { - conf.setLong(SNAPSHOT_ID, snapshotId); - return this; - } - - public ConfigBuilder asOfTime(long asOfTime) { - conf.setLong(AS_OF_TIMESTAMP, asOfTime); - return this; - } - - public ConfigBuilder splitSize(long splitSize) { - conf.setLong(SPLIT_SIZE, splitSize); - return this; - } - - /** - * If this API is called. The input splits - * constructed will have host location information - */ - public ConfigBuilder preferLocality() { - conf.setBoolean(LOCALITY, true); - return this; - } - - public ConfigBuilder catalogFunc(Class> catalogFuncClass) { - conf.setClass(CATALOG, catalogFuncClass, Function.class); - return this; - } - - public ConfigBuilder useHiveRows() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name()); - return this; - } - - public ConfigBuilder usePigTuples() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.PIG.name()); - return this; - } - - /** - * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only - * partially applies the filter, it will return the residual filter back. If the platform can correctly apply - * the residual filters, then it should call this api. Otherwise the current api will throw an exception if the - * passed in filter is not completely satisfied. - */ - public ConfigBuilder skipResidualFiltering() { - conf.setBoolean(SKIP_RESIDUAL_FILTERING, true); - return this; - } + return new InputFormatConfig.ConfigBuilder(job.getConfiguration()); } @Override @@ -217,36 +108,38 @@ public List getSplits(JobContext context) { Configuration conf = context.getConfiguration(); Table table = findTable(conf); TableScan scan = table.newScan() - .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(SNAPSHOT_ID, -1); + .caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); if (snapshotId != -1) { scan = scan.useSnapshot(snapshotId); } - long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); + long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); if (asOfTime != -1) { scan = scan.asOfTime(asOfTime); } - long splitSize = conf.getLong(SPLIT_SIZE, 0); + long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); if (splitSize > 0) { scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); } - String schemaStr = conf.get(READ_SCHEMA); + String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); if (schemaStr != null) { scan.project(SchemaParser.fromJson(schemaStr)); } // TODO add a filter parser to get rid of Serialization - Expression filter = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); + Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); if (filter != null) { scan = scan.filter(filter); } splits = Lists.newArrayList(); - boolean applyResidual = !conf.getBoolean(SKIP_RESIDUAL_FILTERING, false); - InMemoryDataModel model = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); + InputFormatConfig.InMemoryDataModel model = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, + InputFormatConfig.InMemoryDataModel.GENERIC); try (CloseableIterable tasksIterable = scan.planTasks()) { tasksIterable.forEach(task -> { - if (applyResidual && (model == InMemoryDataModel.HIVE || model == InMemoryDataModel.PIG)) { + if (applyResidual && (model == InputFormatConfig.InMemoryDataModel.HIVE || + model == InputFormatConfig.InMemoryDataModel.PIG)) { //TODO: We do not support residual evaluation for HIVE and PIG in memory data model yet checkResiduals(task); } @@ -282,7 +175,7 @@ private static final class IcebergRecordReader extends RecordReader private Schema expectedSchema; private boolean reuseContainers; private boolean caseSensitive; - private InMemoryDataModel inMemoryDataModel; + private InputFormatConfig.InMemoryDataModel inMemoryDataModel; private Iterator tasks; private T currentRow; private CloseableIterator currentIterator; @@ -294,12 +187,13 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { CombinedScanTask task = ((IcebergSplit) split).task; this.context = newContext; this.tasks = task.files().iterator(); - this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); - String readSchemaStr = conf.get(READ_SCHEMA); + this.tableSchema = SchemaParser.fromJson(conf.get(InputFormatConfig.TABLE_SCHEMA)); + String readSchemaStr = conf.get(InputFormatConfig.READ_SCHEMA); this.expectedSchema = readSchemaStr != null ? SchemaParser.fromJson(readSchemaStr) : tableSchema; - this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); - this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); - this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + this.reuseContainers = conf.getBoolean(InputFormatConfig.REUSE_CONTAINERS, false); + this.caseSensitive = conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true); + this.inMemoryDataModel = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, + InputFormatConfig.InMemoryDataModel.GENERIC); this.currentIterator = open(tasks.next(), expectedSchema).iterator(); } @@ -370,7 +264,7 @@ private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, Schema readSchema) { - boolean applyResidual = !context.getConfiguration().getBoolean(SKIP_RESIDUAL_FILTERING, false); + boolean applyResidual = !context.getConfiguration().getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); if (applyResidual && residual != null && residual != Expressions.alwaysTrue()) { Evaluator filter = new Evaluator(readSchema.asStruct(), residual, caseSensitive); @@ -461,14 +355,14 @@ private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask ta } private static Table findTable(Configuration conf) { - String path = conf.get(TABLE_PATH); + String path = conf.get(InputFormatConfig.TABLE_PATH); Preconditions.checkArgument(path != null, "Table path should not be null"); if (path.contains("/")) { HadoopTables tables = new HadoopTables(conf); return tables.load(path); } - String catalogFuncClass = conf.get(CATALOG); + String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); if (catalogFuncClass != null) { Function catalogFunc = (Function) DynConstructors.builder(Function.class) @@ -501,7 +395,7 @@ public long getLength() { @Override public String[] getLocations() { - boolean localityPreferred = conf.getBoolean(LOCALITY, false); + boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false); if (!localityPreferred) { return ANYWHERE; } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java new file mode 100644 index 000000000000..0d8cdc036e7f --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataWriter; +import org.apache.iceberg.data.orc.GenericOrcWriter; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.Types; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestHelpers { + + private TestHelpers() {} + + public static DataFile writeFile(File targetFile, Table table, StructLike partitionData, FileFormat fileFormat, + List records) throws IOException { + if (targetFile.exists()) { + if (!targetFile.delete()) { + throw new IOException("Unable to delete " + targetFile.getAbsolutePath()); + } + } + FileAppender appender; + switch (fileFormat) { + case AVRO: + appender = Avro.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(DataWriter::create) + .named(fileFormat.name()) + .build(); + break; + case PARQUET: + appender = Parquet.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericParquetWriter::buildWriter) + .named(fileFormat.name()) + .build(); + break; + case ORC: + appender = ORC.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericOrcWriter::buildWriter) + .build(); + break; + default: + throw new UnsupportedOperationException("Cannot write format: " + fileFormat); + } + + try { + appender.addAll(records); + } finally { + appender.close(); + } + + DataFiles.Builder builder = DataFiles.builder(table.spec()) + .withPath(targetFile.toString()) + .withFormat(fileFormat) + .withFileSizeInBytes(targetFile.length()) + .withMetrics(appender.metrics()); + if (partitionData != null) { + builder.withPartition(partitionData); + } + return builder.build(); + } + + public static Record createSimpleRecord(long id, String data) { + Schema schema = new Schema(required(1, "id", Types.StringType.get()), + optional(2, "data", Types.LongType.get())); + GenericRecord record = GenericRecord.create(schema); + record.setField("id", id); + record.setField("data", data); + return record; + } + + public static Record createCustomRecord(Schema schema, Object... dataValues) { + GenericRecord record = GenericRecord.create(schema); + List fields = schema.columns(); + for (int i = 0; i < fields.size(); i++) { + record.set(i, dataValues[i]); + } + return record; + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java new file mode 100644 index 000000000000..937ddba64da2 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.RandomGenericData; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.mr.mapred.serde.objectinspector.IcebergObjectInspector; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestIcebergSerDe { + + private static final Schema schema = new Schema(required(1, "string_field", Types.StringType.get())); + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Test + public void testInitialize() throws IOException, SerDeException { + File location = tmp.newFolder(); + Assert.assertTrue(location.delete()); + + Configuration conf = new Configuration(); + + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + properties.setProperty(InputFormatConfig.TABLE_LOCATION, location.toString()); + + HadoopTables tables = new HadoopTables(conf); + tables.create(schema, PartitionSpec.unpartitioned(), Collections.emptyMap(), location.toString()); + + IcebergSerDe serDe = new IcebergSerDe(); + serDe.initialize(conf, properties); + + Assert.assertEquals(IcebergObjectInspector.create(schema), serDe.getObjectInspector()); + } + + @Test + public void testDeserialize() { + IcebergSerDe serDe = new IcebergSerDe(); + + Record record = RandomGenericData.generate(schema, 1, 0).get(0); + IcebergWritable writable = new IcebergWritable(record, schema); + + Assert.assertEquals(record, serDe.deserialize(writable)); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java new file mode 100644 index 000000000000..ff5b6455072a --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.mapred.JobConf; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestTableResolver { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private Schema schema = new Schema(required(1, "string_field", Types.StringType.get())); + private File tableLocation; + + @Before + public void before() throws IOException, SerDeException { + tableLocation = tmp.newFolder(); + Configuration conf = new Configuration(); + HadoopTables tables = new HadoopTables(conf); + tables.create(schema, PartitionSpec.unpartitioned(), Collections.emptyMap(), tableLocation.toString()); + } + + @Test + public void resolveTableFromConfigurationDefault() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test + public void resolveTableFromConfigurationHadoopTables() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test(expected = NullPointerException.class) + public void resolveTableFromConfigurationHadoopTablesNoLocation() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + + TableResolver.resolveTableFromConfiguration(conf); + } + + @Test(expected = NoSuchNamespaceException.class) + public void resolveTableFromConfigurationInvalidName() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, "invalid-name"); + + TableResolver.resolveTableFromConfiguration(conf); + } + + @Test + public void resolveTableFromJobConfDefault() throws IOException { + JobConf conf = new JobConf(); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test + public void resolveTableFromPropertiesDefault() throws IOException { + Configuration conf = new Configuration(); + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf, properties); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test(expected = UnsupportedOperationException.class) + public void resolveTableFromConfigurationHiveCatalog() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HIVE_CATALOG); + conf.set(InputFormatConfig.TABLE_NAME, "table_a"); + + TableResolver.resolveTableFromConfiguration(conf); + } + + @Test(expected = NullPointerException.class) + public void resolveTableFromConfigurationHiveCatalogMissingTableName() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HIVE_CATALOG); + + TableResolver.resolveTableFromConfiguration(conf); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java new file mode 100644 index 000000000000..f28e78b4867b --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.nio.ByteBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.BytesWritable; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergBinaryObjectInspector { + + @Test + public void testIcebergByteArrayObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteArray(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + byte[] bytes = new byte[] {0, 1}; + + Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(bytes)); + Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(bytes)); + + byte[] copy = (byte[]) oi.copyObject(bytes); + + Assert.assertArrayEquals(bytes, copy); + Assert.assertNotSame(bytes, copy); + + Assert.assertFalse(oi.preferWritable()); + } + + @Test + public void testIcebergByteBufferObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteBuffer(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + byte[] bytes = new byte[] {0, 1, 2, 3}; + + ByteBuffer buffer = ByteBuffer.wrap(bytes); + Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(buffer)); + Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(buffer)); + + ByteBuffer slice = ByteBuffer.wrap(bytes, 1, 2).slice(); + Assert.assertArrayEquals(new byte[] {1, 2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {1, 2}), oi.getPrimitiveWritableObject(slice)); + + slice.position(1); + Assert.assertArrayEquals(new byte[] {2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {2}), oi.getPrimitiveWritableObject(slice)); + + byte[] copy = (byte[]) oi.copyObject(bytes); + + Assert.assertArrayEquals(bytes, copy); + Assert.assertNotSame(bytes, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java new file mode 100644 index 000000000000..28962aa352f4 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Date; +import java.time.LocalDate; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergDateObjectInspector { + + @Test + public void testIcebergDateObjectInspector() { + DateObjectInspector oi = IcebergDateObjectInspector.get(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.DATE, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.dateTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.dateTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(Date.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(DateWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + LocalDate local = LocalDate.of(2020, 1, 1); + Date date = Date.valueOf("2020-01-01"); + + Assert.assertEquals(date, oi.getPrimitiveJavaObject(local)); + Assert.assertEquals(new DateWritable(date), oi.getPrimitiveWritableObject(local)); + + Date copy = (Date) oi.copyObject(date); + + Assert.assertEquals(date, copy); + Assert.assertNotSame(date, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java new file mode 100644 index 000000000000..77489a4478e0 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.math.BigDecimal; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergDecimalObjectInspector { + + @Test + public void testCache() { + HiveDecimalObjectInspector oi = IcebergDecimalObjectInspector.get(38, 18); + + Assert.assertSame(oi, IcebergDecimalObjectInspector.get(38, 18)); + Assert.assertNotSame(oi, IcebergDecimalObjectInspector.get(28, 18)); + Assert.assertNotSame(oi, IcebergDecimalObjectInspector.get(38, 28)); + } + + @Test + public void testIcebergDecimalObjectInspector() { + HiveDecimalObjectInspector oi = IcebergDecimalObjectInspector.get(38, 18); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.DECIMAL, oi.getPrimitiveCategory()); + + Assert.assertEquals(new DecimalTypeInfo(38, 18), oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.decimalTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(38, oi.precision()); + Assert.assertEquals(18, oi.scale()); + + Assert.assertEquals(HiveDecimal.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(HiveDecimalWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + HiveDecimal one = HiveDecimal.create(BigDecimal.ONE); + + Assert.assertEquals(one, oi.getPrimitiveJavaObject(BigDecimal.ONE)); + Assert.assertEquals(new HiveDecimalWritable(one), oi.getPrimitiveWritableObject(BigDecimal.ONE)); + + HiveDecimal copy = (HiveDecimal) oi.copyObject(one); + + Assert.assertEquals(one, copy); + Assert.assertNotSame(one, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java new file mode 100644 index 000000000000..908c9dbfbef8 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.required; + + +public class TestIcebergObjectInspector { + + private int id = 0; + + private final Schema schema = new Schema( + required(id++, "binary_field", Types.BinaryType.get(), "binary comment"), + required(id++, "boolean_field", Types.BooleanType.get(), "boolean comment"), + required(id++, "date_field", Types.DateType.get(), "date comment"), + required(id++, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), + required(id++, "double_field", Types.DoubleType.get(), "double comment"), + required(id++, "fixed_field", Types.FixedType.ofLength(3), "fixed comment"), + required(id++, "float_field", Types.FloatType.get(), "float comment"), + required(id++, "integer_field", Types.IntegerType.get(), "integer comment"), + required(id++, "long_field", Types.LongType.get(), "long comment"), + required(id++, "string_field", Types.StringType.get(), "string comment"), + required(id++, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), + required(id++, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), + required(id++, "uuid_field", Types.UUIDType.get(), "uuid comment"), + required(id++, "list_field", + Types.ListType.ofRequired(id++, Types.StringType.get()), "list comment"), + required(id++, "map_field", + Types.MapType.ofRequired(id++, id++, Types.StringType.get(), Types.IntegerType.get()), + "map comment"), + required(id++, "struct_field", Types.StructType.of( + Types.NestedField.required(id++, "nested_field", Types.StringType.get(), "nested field comment")), + "struct comment" + ) + ); + + @Test + public void testIcebergObjectInspector() { + int fieldId = 0; + + ObjectInspector oi = IcebergObjectInspector.create(schema); + Assert.assertNotNull(oi); + Assert.assertEquals(ObjectInspector.Category.STRUCT, oi.getCategory()); + + StructObjectInspector soi = (StructObjectInspector) oi; + + // binary + StructField binaryField = soi.getStructFieldRef("binary_field"); + Assert.assertEquals(fieldId++, binaryField.getFieldID()); + Assert.assertEquals("binary_field", binaryField.getFieldName()); + Assert.assertEquals("binary comment", binaryField.getFieldComment()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteBuffer(), binaryField.getFieldObjectInspector()); + + // boolean + StructField booleanField = soi.getStructFieldRef("boolean_field"); + Assert.assertEquals(fieldId++, booleanField.getFieldID()); + Assert.assertEquals("boolean_field", booleanField.getFieldName()); + Assert.assertEquals("boolean comment", booleanField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(boolean.class), booleanField.getFieldObjectInspector()); + + // date + StructField dateField = soi.getStructFieldRef("date_field"); + Assert.assertEquals(fieldId++, dateField.getFieldID()); + Assert.assertEquals("date_field", dateField.getFieldName()); + Assert.assertEquals("date comment", dateField.getFieldComment()); + Assert.assertEquals(IcebergDateObjectInspector.get(), dateField.getFieldObjectInspector()); + + // decimal + StructField decimalField = soi.getStructFieldRef("decimal_field"); + Assert.assertEquals(fieldId++, decimalField.getFieldID()); + Assert.assertEquals("decimal_field", decimalField.getFieldName()); + Assert.assertEquals("decimal comment", decimalField.getFieldComment()); + Assert.assertEquals(IcebergDecimalObjectInspector.get(38, 18), decimalField.getFieldObjectInspector()); + + // double + StructField doubleField = soi.getStructFieldRef("double_field"); + Assert.assertEquals(fieldId++, doubleField.getFieldID()); + Assert.assertEquals("double_field", doubleField.getFieldName()); + Assert.assertEquals("double comment", doubleField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(double.class), doubleField.getFieldObjectInspector()); + + // fixed + StructField fixedField = soi.getStructFieldRef("fixed_field"); + Assert.assertEquals(fieldId++, fixedField.getFieldID()); + Assert.assertEquals("fixed_field", fixedField.getFieldName()); + Assert.assertEquals("fixed comment", fixedField.getFieldComment()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteArray(), fixedField.getFieldObjectInspector()); + + // float + StructField floatField = soi.getStructFieldRef("float_field"); + Assert.assertEquals(fieldId++, floatField.getFieldID()); + Assert.assertEquals("float_field", floatField.getFieldName()); + Assert.assertEquals("float comment", floatField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(float.class), floatField.getFieldObjectInspector()); + + // integer + StructField integerField = soi.getStructFieldRef("integer_field"); + Assert.assertEquals(fieldId++, integerField.getFieldID()); + Assert.assertEquals("integer_field", integerField.getFieldName()); + Assert.assertEquals("integer comment", integerField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(int.class), integerField.getFieldObjectInspector()); + + // long + StructField longField = soi.getStructFieldRef("long_field"); + Assert.assertEquals(fieldId++, longField.getFieldID()); + Assert.assertEquals("long_field", longField.getFieldName()); + Assert.assertEquals("long comment", longField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(long.class), longField.getFieldObjectInspector()); + + // string + StructField stringField = soi.getStructFieldRef("string_field"); + Assert.assertEquals(fieldId++, stringField.getFieldID()); + Assert.assertEquals("string_field", stringField.getFieldName()); + Assert.assertEquals("string comment", stringField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(String.class), stringField.getFieldObjectInspector()); + + // timestamp without tz + StructField timestampField = soi.getStructFieldRef("timestamp_field"); + Assert.assertEquals(fieldId++, timestampField.getFieldID()); + Assert.assertEquals("timestamp_field", timestampField.getFieldName()); + Assert.assertEquals("timestamp comment", timestampField.getFieldComment()); + Assert.assertEquals(IcebergTimestampObjectInspector.get(false), timestampField.getFieldObjectInspector()); + + // timestamp with tz + StructField timestampTzField = soi.getStructFieldRef("timestamptz_field"); + Assert.assertEquals(fieldId++, timestampTzField.getFieldID()); + Assert.assertEquals("timestamptz_field", timestampTzField.getFieldName()); + Assert.assertEquals("timestamptz comment", timestampTzField.getFieldComment()); + Assert.assertEquals(IcebergTimestampObjectInspector.get(true), timestampTzField.getFieldObjectInspector()); + + // UUID + StructField uuidField = soi.getStructFieldRef("uuid_field"); + Assert.assertEquals(fieldId++, uuidField.getFieldID()); + Assert.assertEquals("uuid_field", uuidField.getFieldName()); + Assert.assertEquals("uuid comment", uuidField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(String.class), uuidField.getFieldObjectInspector()); + + // list + StructField listField = soi.getStructFieldRef("list_field"); + Assert.assertEquals(fieldId++, listField.getFieldID()); + Assert.assertEquals("list_field", listField.getFieldName()); + Assert.assertEquals("list comment", listField.getFieldComment()); + Assert.assertEquals(getListObjectInspector(String.class), listField.getFieldObjectInspector()); + + // skip element id + fieldId++; + + // map + StructField mapField = soi.getStructFieldRef("map_field"); + Assert.assertEquals(fieldId++, mapField.getFieldID()); + Assert.assertEquals("map_field", mapField.getFieldName()); + Assert.assertEquals("map comment", mapField.getFieldComment()); + Assert.assertEquals(getMapObjectInspector(String.class, int.class), mapField.getFieldObjectInspector()); + + // skip key and value ids + fieldId += 2; + + // struct + StructField structField = soi.getStructFieldRef("struct_field"); + Assert.assertEquals(fieldId, structField.getFieldID()); + Assert.assertEquals("struct_field", structField.getFieldName()); + Assert.assertEquals("struct comment", structField.getFieldComment()); + + ObjectInspector expectedObjectInspector = new IcebergRecordObjectInspector( + (Types.StructType) schema.findType(fieldId), ImmutableList.of(getPrimitiveObjectInspector(String.class))); + Assert.assertEquals(expectedObjectInspector, structField.getFieldObjectInspector()); + } + + @Test + public void testIcebergObjectInspectorUnsupportedTypes() { + AssertHelpers.assertThrows( + "Hive does not support time type", IllegalArgumentException.class, "TIME type is not supported", + () -> IcebergObjectInspector.create(required(1, "time_field", Types.TimeType.get()))); + } + + private static ObjectInspector getPrimitiveObjectInspector(Class clazz) { + PrimitiveTypeInfo typeInfo = (PrimitiveTypeInfo) TypeInfoFactory.getPrimitiveTypeInfoFromJavaPrimitive(clazz); + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(typeInfo); + } + + private static ObjectInspector getListObjectInspector(Class clazz) { + return ObjectInspectorFactory.getStandardListObjectInspector(getPrimitiveObjectInspector(clazz)); + } + + private static ObjectInspector getMapObjectInspector(Class keyClazz, Class valueClazz) { + return ObjectInspectorFactory.getStandardMapObjectInspector( + getPrimitiveObjectInspector(keyClazz), getPrimitiveObjectInspector(valueClazz)); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java new file mode 100644 index 000000000000..edfaa1722185 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.RandomGenericData; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestIcebergRecordObjectInspector { + + @Test + public void testIcebergRecordObjectInspector() { + Schema schema = new Schema( + required(1, "integer_field", Types.IntegerType.get()), + required(2, "struct_field", Types.StructType.of( + Types.NestedField.required(3, "string_field", Types.StringType.get()))) + ); + + Record record = RandomGenericData.generate(schema, 1, 0L).get(0); + Record innerRecord = record.get(1, Record.class); + + StructObjectInspector soi = (StructObjectInspector) IcebergObjectInspector.create(schema); + Assert.assertEquals(ImmutableList.of(record.get(0), record.get(1)), soi.getStructFieldsDataAsList(record)); + + StructField integerField = soi.getStructFieldRef("integer_field"); + Assert.assertEquals(record.get(0), soi.getStructFieldData(record, integerField)); + + StructField structField = soi.getStructFieldRef("struct_field"); + Object innerData = soi.getStructFieldData(record, structField); + Assert.assertEquals(innerRecord, innerData); + + StructObjectInspector innerSoi = (StructObjectInspector) structField.getFieldObjectInspector(); + StructField stringField = innerSoi.getStructFieldRef("string_field"); + + Assert.assertEquals(ImmutableList.of(innerRecord.get(0)), innerSoi.getStructFieldsDataAsList(innerRecord)); + Assert.assertEquals(innerRecord.get(0), innerSoi.getStructFieldData(innerData, stringField)); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java new file mode 100644 index 000000000000..a1f6c18b8dd6 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergTimestampObjectInspector { + + @Test + public void testIcebergTimestampObjectInspector() { + TimestampObjectInspector oi = IcebergTimestampObjectInspector.get(false); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.timestampTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(Timestamp.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(TimestampWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + LocalDateTime local = LocalDateTime.of(2020, 1, 1, 0, 0); + Timestamp ts = Timestamp.valueOf("2020-01-01 00:00:00"); + + Assert.assertEquals(ts, oi.getPrimitiveJavaObject(local)); + Assert.assertEquals(new TimestampWritable(ts), oi.getPrimitiveWritableObject(local)); + + Timestamp copy = (Timestamp) oi.copyObject(ts); + + Assert.assertEquals(ts, copy); + Assert.assertNotSame(ts, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index a5dbbef0ac4e..05842f80e6bd 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -58,6 +58,7 @@ import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; @@ -128,8 +129,8 @@ public void testUnpartitionedTable() throws Exception { .appendFile(dataFile) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); validate(job, expectedRecords); } @@ -148,8 +149,8 @@ public void testPartitionedTable() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); validate(job, expectedRecords); } @@ -171,9 +172,10 @@ public void testFilterExp() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) - .filter(Expressions.equal("date", "2020-03-20")); + .schema(table.schema()) + .filter(Expressions.equal("date", "2020-03-20")); validate(job, expectedRecords); } @@ -201,20 +203,22 @@ public void testResiduals() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, expectedRecords); // skip residual filtering job = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(job); configBuilder.skipResidualFiltering().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, writeRecords); } @@ -235,8 +239,9 @@ public void testFailedResidualFiltering() throws Exception { .commit(); Job jobShouldFail1 = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); configBuilder.useHiveRows().readFrom(location.toString()) + .schema(table.schema()) .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 0))); @@ -248,6 +253,7 @@ public void testFailedResidualFiltering() throws Exception { Job jobShouldFail2 = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(jobShouldFail2); configBuilder.usePigTuples().readFrom(location.toString()) + .schema(table.schema()) .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 0))); @@ -272,8 +278,9 @@ public void testProjection() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(table.schema()) .readFrom(location.toString()) .project(projectedSchema); List outputRecords = readRecords(job.getConfiguration()); @@ -311,27 +318,43 @@ public void testIdentityPartitionProjections() throws Exception { append.commit(); // individual fields - validateIdentityPartitionProjections(location.toString(), withColumns("date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("id"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("id"), inputRecords); // field pairs - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level"), inputRecords); // out-of-order pairs - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date"), inputRecords); // full projection - validateIdentityPartitionProjections(location.toString(), LOG_SCHEMA, inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), LOG_SCHEMA, inputRecords); // out-of-order triplets - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level", "date"), inputRecords); } private static Schema withColumns(String... names) { @@ -344,10 +367,11 @@ private static Schema withColumns(String... names) { } private void validateIdentityPartitionProjections( - String tablePath, Schema projectedSchema, List inputRecords) throws Exception { + String tablePath, Schema tableSchema, Schema projectedSchema, List inputRecords) throws Exception { Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(tableSchema) .readFrom(tablePath) .project(projectedSchema); List actualRecords = readRecords(job.getConfiguration()); @@ -381,8 +405,9 @@ public void testSnapshotReads() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(table.schema()) .readFrom(location.toString()) .snapshotId(snapshotId); @@ -401,8 +426,8 @@ public void testLocality() throws Exception { .appendFile(writeFile(table, null, format, expectedRecords)) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); for (InputSplit split : splits(job.getConfiguration())) { Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); @@ -438,9 +463,10 @@ public void testCustomCatalog() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .catalogFunc(HadoopCatalogFunc.class) + .schema(table.schema()) .readFrom(tableIdentifier.toString()); validate(job, expectedRecords); } diff --git a/versions.props b/versions.props index 7c743a41106b..84371690456f 100644 --- a/versions.props +++ b/versions.props @@ -3,6 +3,7 @@ org.apache.avro:avro = 1.9.2 org.apache.flink:* = 1.10.1 org.apache.hadoop:* = 2.7.3 org.apache.hive:hive-metastore = 2.3.7 +org.apache.hive:hive-serde = 2.3.7 org.apache.orc:* = 1.6.3 org.apache.parquet:* = 1.11.0 org.apache.spark:spark-hive_2.11 = 2.4.5