From efedb63b070a3bf75b88165fbdd618d3570a0310 Mon Sep 17 00:00:00 2001 From: Balaji Varadarajan Date: Thu, 31 Jan 2019 09:51:47 -0800 Subject: [PATCH] Make HoodieDeltaStreamer hive-sync to CDH Hive Servers --- docs/quickstart.md | 4 +- hoodie-client/pom.xml | 42 +----- hoodie-hadoop-mr/pom.xml | 70 +++------- hoodie-hive/pom.xml | 84 +++--------- hoodie-spark/pom.xml | 87 ++++-------- hoodie-utilities/pom.xml | 128 ++++++++++-------- packaging/hoodie-hadoop-mr-bundle/pom.xml | 154 ++++++--------------- packaging/hoodie-hive-bundle/pom.xml | 89 +++--------- packaging/hoodie-spark-bundle/pom.xml | 89 +++--------- pom.xml | 157 +++++++--------------- 10 files changed, 274 insertions(+), 630 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 41ec9a96d0f6b..bb7be6edda54b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -33,7 +33,7 @@ Hoodie requires Java 8 to be installed. Hoodie works with Spark-2.x versions. We | Hadoop | Hive | Spark | Instructions to Build Hoodie | | ---- | ----- | ---- | ---- | -| 2.6.0-cdh5.7.2 | 1.1.0-cdh5.7.2 | spark-2.[1-3].x | Use "mvn clean install -DskipTests -Dhive11". Jars will have ".hive11" as suffix | +| 2.6.0-cdh5.7.2 | 1.1.0-cdh5.7.2 | spark-2.[1-3].x | Use "mvn clean install -DskipTests -Dhadoop.version=2.6.0-cdh5.7.2 -Dhive.version=1.1.0-cdh5.7.2" | | Apache hadoop-2.8.4 | Apache hive-2.3.3 | spark-2.[1-3].x | Use "mvn clean install -DskipTests" | | Apache hadoop-2.7.3 | Apache hive-1.2.1 | spark-2.[1-3].x | Use "mvn clean install -DskipTests" | @@ -1244,4 +1244,4 @@ cd docker [INFO] Finished at: 2018-09-10T17:47:37-07:00 [INFO] Final Memory: 236M/1848M [INFO] ------------------------------------------------------------------------ -``` \ No newline at end of file +``` diff --git a/hoodie-client/pom.xml b/hoodie-client/pom.xml index eccfa802c58b8..c813ef24a499b 100644 --- a/hoodie-client/pom.xml +++ b/hoodie-client/pom.xml @@ -171,6 +171,13 @@ spark-sql_2.11 + + ${hive.groupid} + hive-exec + ${hive.version} + test + + org.apache.hbase hbase-client @@ -218,39 +225,4 @@ - - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-exec - ${hive12.version} - test - - - - - hive11 - - - hive11 - - - - - ${hive11.groupid} - hive-exec - ${hive11.version} - test - - - - - diff --git a/hoodie-hadoop-mr/pom.xml b/hoodie-hadoop-mr/pom.xml index fe1df4396a740..6ef41a32d24b0 100644 --- a/hoodie-hadoop-mr/pom.xml +++ b/hoodie-hadoop-mr/pom.xml @@ -60,6 +60,22 @@ org.apache.hadoop hadoop-hdfs + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + commons-logging + commons-logging + + + + + ${hive.groupid} + hive-exec + ${hive.version} + commons-logging commons-logging @@ -105,58 +121,4 @@ - - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - commons-logging - commons-logging - - - - - ${hive12.groupid} - hive-exec - ${hive12.version} - - - - - hive11 - - - hive11 - - - - - ${hive11.groupid} - hive-jdbc - ${hive11.version} - - - commons-logging - commons-logging - - - - - ${hive11.groupid} - hive-exec - ${hive11.version} - - - - diff --git a/hoodie-hive/pom.xml b/hoodie-hive/pom.xml index f74e5ea8350b0..da4e9a0d7e299 100644 --- a/hoodie-hive/pom.xml +++ b/hoodie-hive/pom.xml @@ -99,6 +99,27 @@ junit junit + + ${hive.groupid} + hive-service + ${hive.version} + + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + ${hive.groupid} + hive-metastore + ${hive.version} + + + ${hive.groupid} + hive-common + ${hive.version} + + org.apache.hadoop hadoop-hdfs @@ -175,67 +196,4 @@ - - - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-service - ${hive12.version} - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - - - ${hive12.groupid} - hive-common - ${hive12.version} - - - - - hive11 - - - hive11 - - - - - org.apache.hive - hive-service - ${hive11.version} - - - org.apache.hive - hive-jdbc - ${hive11.version} - - - org.apache.hive - hive-metastore - ${hive11.version} - - - org.apache.hive - hive-common - ${hive11.version} - - - - diff --git a/hoodie-spark/pom.xml b/hoodie-spark/pom.xml index c4869f3b22799..d800c1a7218b3 100644 --- a/hoodie-spark/pom.xml +++ b/hoodie-spark/pom.xml @@ -221,6 +221,30 @@ commons-configuration2 + + ${hive.groupid} + hive-service + ${hive.version} + + + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + + ${hive.groupid} + hive-metastore + ${hive.version} + + + + ${hive.groupid} + hive-common + ${hive.version} + + com.uber.hoodie hoodie-client @@ -264,67 +288,4 @@ test - - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-service - ${hive12.version} - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - - - ${hive12.groupid} - hive-common - ${hive12.version} - - - - - hive11 - - - hive11 - - - - - ${hive11.groupid} - hive-service - ${hive11.version} - - - ${hive11.groupid} - hive-jdbc - ${hive11.version} - - - ${hive11.groupid} - hive-metastore - ${hive11.version} - - - ${hive11.groupid} - hive-common - ${hive11.version} - - - - - diff --git a/hoodie-utilities/pom.xml b/hoodie-utilities/pom.xml index 8a9feac7f31ef..2f6c1bbb66bdc 100644 --- a/hoodie-utilities/pom.xml +++ b/hoodie-utilities/pom.xml @@ -56,6 +56,8 @@ true + commons-dbcp:commons-dbcp + commons-pool:commons-pool com.uber.hoodie:hoodie-common com.uber.hoodie:hoodie-client com.uber.hoodie:hoodie-spark @@ -76,8 +78,50 @@ com.yammer.metrics:metrics-core com.101tec:zkclient org.apache.kafka:kafka-clients + org.apache.hive:hive-common + org.apache.hive:hive-service + org.apache.hive:hive-metastore + org.apache.hive:hive-jdbc + + + org.apache.commons.dbcp. + com.uber.hoodie.org.apache.commons.dbcp. + + + org.apache.commons.pool. + com.uber.hoodie.org.apache.commons.pool. + + + org.apache.hive.jdbc. + com.uber.hoodie.org.apache.hive.jdbc. + + + org.apache.hadoop.hive.metastore. + com.uber.hoodie.org.apache.hadoop_hive.metastore. + + + org.apache.hive.common. + com.uber.hoodie.org.apache.hive.common. + + + org.apache.hadoop.hive.common. + com.uber.hoodie.org.apache.hadoop_hive.common. + + + org.apache.hadoop.hive.conf. + com.uber.hoodie.org.apache.hadoop_hive.conf. + + + org.apache.hive.service. + com.uber.hoodie.org.apache.hive.service. + + + org.apache.hadoop.hive.service. + com.uber.hoodie.org.apache.hadoop_hive.service. + + @@ -154,6 +198,30 @@ + + ${hive.groupid} + hive-exec + ${hive.version} + test + + + + ${hive.groupid} + hive-jdbc + ${hive.version} + standalone + + + org.slf4j + slf4j-api + + + javax.servlet + servlet-api + + + + com.uber.hoodie hoodie-hive @@ -185,6 +253,11 @@ commons-dbcp commons-dbcp + + commons-pool + commons-pool + + org.apache.httpcomponents httpcore @@ -303,59 +376,4 @@ - - - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - standalone - - - org.slf4j - slf4j-api - - - javax.servlet - servlet-api - - - - - - - hive11 - - - hive11 - - - - - org.apache.hive - hive-jdbc - ${hive11.version} - standalone - - - org.slf4j - slf4j-api - - - javax.servlet - servlet-api - - - - - - diff --git a/packaging/hoodie-hadoop-mr-bundle/pom.xml b/packaging/hoodie-hadoop-mr-bundle/pom.xml index 79a4a757a4ac4..4e2e72d477ca0 100644 --- a/packaging/hoodie-hadoop-mr-bundle/pom.xml +++ b/packaging/hoodie-hadoop-mr-bundle/pom.xml @@ -66,6 +66,48 @@ hadoop-auth + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + commons-logging + commons-logging + + + + + ${hive.groupid} + hive-exec + ${hive.version} + + + ${hive.groupid} + hive-service + ${hive.version} + + + ${hive.groupid} + hive-shims + ${hive.version} + + + ${hive.groupid} + hive-serde + ${hive.version} + + + ${hive.groupid} + hive-metastore + ${hive.version} + + + ${hive.groupid} + hive-common + ${hive.version} + + org.apache.hadoop hadoop-hdfs @@ -182,116 +224,4 @@ true - - - - hive12 - - - !hive11 - - - - - - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - commons-logging - commons-logging - - - - - ${hive12.groupid} - hive-exec - ${hive12.version} - - - ${hive12.groupid} - hive-service - ${hive12.version} - - - ${hive12.groupid} - hive-shims - ${hive12.version} - - - ${hive12.groupid} - hive-serde - ${hive12.version} - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - - - ${hive12.groupid} - hive-common - ${hive12.version} - - - - - hive11 - - - hive11 - - - - .hive11 - - - - ${hive11.groupid} - hive-service - ${hive11.version} - - - ${hive11.groupid} - hive-shims - ${hive11.version} - - - ${hive11.groupid} - hive-jdbc - ${hive11.version} - - - commons-logging - commons-logging - - - - - ${hive11.groupid} - hive-serde - ${hive11.version} - - - ${hive11.groupid} - hive-metastore - ${hive11.version} - - - ${hive11.groupid} - hive-common - ${hive11.version} - - - ${hive11.groupid} - hive-exec - ${hive11.version} - - - - - diff --git a/packaging/hoodie-hive-bundle/pom.xml b/packaging/hoodie-hive-bundle/pom.xml index eafad25922921..ee43709faa29b 100644 --- a/packaging/hoodie-hive-bundle/pom.xml +++ b/packaging/hoodie-hive-bundle/pom.xml @@ -44,6 +44,26 @@ org.apache.hadoop hadoop-auth + + ${hive.groupid} + hive-service + ${hive.version} + + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + ${hive.groupid} + hive-metastore + ${hive.version} + + + ${hive.groupid} + hive-common + ${hive.version} + com.google.guava guava @@ -195,73 +215,4 @@ true - - - - hive12 - - - !hive11 - - - - - - - - ${hive12.groupid} - hive-service - ${hive12.version} - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - - - ${hive12.groupid} - hive-common - ${hive12.version} - - - - - hive11 - - - hive11 - - - - .hive11 - - - - org.apache.hive - hive-service - ${hive11.version} - - - org.apache.hive - hive-jdbc - ${hive11.version} - - - org.apache.hive - hive-metastore - ${hive11.version} - - - org.apache.hive - hive-common - ${hive11.version} - - - - diff --git a/packaging/hoodie-spark-bundle/pom.xml b/packaging/hoodie-spark-bundle/pom.xml index 95b4aedd42e54..5c932e59c174a 100644 --- a/packaging/hoodie-spark-bundle/pom.xml +++ b/packaging/hoodie-spark-bundle/pom.xml @@ -239,6 +239,26 @@ org.apache.avro avro + + ${hive.groupid} + hive-service + ${hive.version} + + + ${hive.groupid} + hive-jdbc + ${hive.version} + + + ${hive.groupid} + hive-metastore + ${hive.version} + + + ${hive.groupid} + hive-common + ${hive.version} + org.apache.commons commons-configuration2 @@ -269,74 +289,5 @@ ${project.version} - - - - hive12 - - - !hive11 - - - - - - - - ${hive12.groupid} - hive-service - ${hive12.version} - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - - - ${hive12.groupid} - hive-common - ${hive12.version} - - - - - hive11 - - - hive11 - - - - .hive11 - - - - ${hive11.groupid} - hive-service - ${hive11.version} - - - ${hive11.groupid} - hive-jdbc - ${hive11.version} - - - ${hive11.groupid} - hive-metastore - ${hive11.version} - - - ${hive11.groupid} - hive-common - ${hive11.version} - - - - diff --git a/pom.xml b/pom.xml index 88196084c9a47..a0ec312c51f2a 100644 --- a/pom.xml +++ b/pom.xml @@ -129,10 +129,8 @@ 1.2.17 2.9.9 2.7.3 - org.apache.hive - 1.2.1 - org.apache.hive - 1.1.1 + org.apache.hive + 1.2.1 3.1.1 2.1.0 1.7.7 @@ -589,6 +587,11 @@ commons-dbcp 1.4 + + commons-pool + commons-pool + 1.4 + org.apache.httpcomponents httpcore @@ -656,7 +659,48 @@ jackson-mapper-asl 1.9.13 - + + ${hive.groupid} + hive-service + ${hive.version} + provided + + + ${hive.groupid} + hive-shims + ${hive.version} + provided + + + ${hive.groupid} + hive-jdbc + ${hive.version} + provided + + + ${hive.groupid} + hive-serde + ${hive.version} + provided + + + ${hive.groupid} + hive-metastore + ${hive.version} + provided + + + ${hive.groupid} + hive-common + ${hive.version} + provided + + + ${hive.groupid} + hive-exec + ${hive.version} + provided + org.apache.hadoop hadoop-hdfs @@ -708,109 +752,6 @@ - - hive12 - - - !hive11 - - - - - ${hive12.groupid} - hive-service - ${hive12.version} - provided - - - ${hive12.groupid} - hive-shims - ${hive12.version} - provided - - - ${hive12.groupid} - hive-jdbc - ${hive12.version} - provided - - - ${hive12.groupid} - hive-serde - ${hive12.version} - provided - - - ${hive12.groupid} - hive-metastore - ${hive12.version} - provided - - - ${hive12.groupid} - hive-common - ${hive12.version} - provided - - - ${hive12.groupid} - hive-exec - ${hive12.version} - provided - - - - - hive11 - - - hive11 - - - - - org.apache.hive - hive-service - ${hive11.version} - - - org.apache.hive - hive-shims - ${hive11.version} - provided - - - org.apache.hive - hive-jdbc - ${hive11.version} - provided - - - org.apache.hive - hive-serde - ${hive11.version} - provided - - - org.apache.hive - hive-metastore - ${hive11.version} - provided - - - org.apache.hive - hive-common - ${hive11.version} - provided - - - org.apache.hive - hive-exec - ${hive11.version} - provided - - - release