diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java index 7b5aa8a541c01..1dd592d6e0d09 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java @@ -38,10 +38,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.jdbc.HiveDriver; @@ -85,7 +87,7 @@ public class HoodieHiveClient { private final HoodieTableMetaClient metaClient; private final HoodieTableType tableType; private final PartitionValueExtractor partitionValueExtractor; - private HiveMetaStoreClient client; + private IMetaStoreClient client; private HiveSyncConfig syncConfig; private FileSystem fs; private Connection connection; @@ -106,8 +108,8 @@ public HoodieHiveClient(HiveSyncConfig cfg, HiveConf configuration, FileSystem f createHiveConnection(); } try { - this.client = new HiveMetaStoreClient(configuration); - } catch (MetaException e) { + this.client = Hive.get(configuration).getMSC(); + } catch (MetaException | HiveException e) { throw new HoodieHiveSyncException("Failed to create HiveMetaStoreClient", e); } @@ -609,7 +611,7 @@ public void close() { connection.close(); } if (client != null) { - client.close(); + Hive.closeCurrent(); client = null; } } catch (SQLException e) { diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml index ffdc3d5699453..55d07fab131b1 100644 --- a/packaging/hudi-spark-bundle/pom.xml +++ b/packaging/hudi-spark-bundle/pom.xml @@ -105,31 +105,31 @@ org.apache.hive.jdbc. - org.apache.hudi.org.apache.hive.jdbc. + ${spark.bundle.hive.shade.prefix}org.apache.hive.jdbc. org.apache.hadoop.hive.metastore. - org.apache.hudi.org.apache.hadoop_hive.metastore. + ${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore. org.apache.hive.common. - org.apache.hudi.org.apache.hive.common. + ${spark.bundle.hive.shade.prefix}org.apache.hive.common. org.apache.hadoop.hive.common. - org.apache.hudi.org.apache.hadoop_hive.common. + ${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.common. org.apache.hadoop.hive.conf. - org.apache.hudi.org.apache.hadoop_hive.conf. + ${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf. org.apache.hive.service. - org.apache.hudi.org.apache.hive.service. + ${spark.bundle.hive.shade.prefix}org.apache.hive.service. org.apache.hadoop.hive.service. - org.apache.hudi.org.apache.hadoop_hive.service. + ${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.service. com.codahale.metrics. @@ -144,40 +144,6 @@ org.apache.hudi.com.databricks. - @@ -246,37 +212,48 @@ ${hive.groupid} hive-service ${hive.version} - compile + ${spark.bundle.hive.scope} ${hive.groupid} hive-service-rpc ${hive.version} - compile + ${spark.bundle.hive.scope} ${hive.groupid} hive-jdbc ${hive.version} - compile + ${spark.bundle.hive.scope} ${hive.groupid} hive-metastore ${hive.version} - compile + ${spark.bundle.hive.scope} ${hive.groupid} hive-common ${hive.version} - compile + ${spark.bundle.hive.scope} + + + + + spark-bundle-shade-hive + + compile + org.apache.hudi. + + + diff --git a/packaging/hudi-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml index cc6091809f005..110ff0fde1c4f 100644 --- a/packaging/hudi-utilities-bundle/pom.xml +++ b/packaging/hudi-utilities-bundle/pom.xml @@ -113,31 +113,31 @@ org.apache.hive.jdbc. - org.apache.hudi.org.apache.hive.jdbc. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.jdbc. org.apache.hadoop.hive.metastore. - org.apache.hudi.org.apache.hadoop_hive.metastore. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore. org.apache.hive.common. - org.apache.hudi.org.apache.hive.common. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.common. org.apache.hadoop.hive.common. - org.apache.hudi.org.apache.hadoop_hive.common. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.common. org.apache.hadoop.hive.conf. - org.apache.hudi.org.apache.hadoop_hive.conf. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf. org.apache.hive.service. - org.apache.hudi.org.apache.hive.service. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.service. org.apache.hadoop.hive.service. - org.apache.hudi.org.apache.hadoop_hive.service. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.service. com.codahale.metrics. @@ -221,36 +221,46 @@ ${hive.groupid} hive-service ${hive.version} - compile + ${utilities.bundle.hive.scope} ${hive.groupid} hive-service-rpc ${hive.version} - compile + ${utilities.bundle.hive.scope} ${hive.groupid} hive-jdbc ${hive.version} - compile + ${utilities.bundle.hive.scope} ${hive.groupid} hive-metastore ${hive.version} - compile + ${utilities.bundle.hive.scope} ${hive.groupid} hive-common ${hive.version} - compile + ${utilities.bundle.hive.scope} + + + + utilities-bundle-shade-hive + + compile + org.apache.hudi. + + + diff --git a/pom.xml b/pom.xml index 55f12549c588c..3fb353794d8dc 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,10 @@ ${project.basedir} provided + provided + + provided +