Skip to content

[HUDI-306] Support Glue catalog and other hive metastore implementations #961

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vinothchandar merged 3 commits into from
Nov 12, 2019
Merged

[HUDI-306] Support Glue catalog and other hive metastore implementations #961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
756d376
Support Glue catalog and other metastore implementations
umehrot2 Sep 24, 2019
488c544
Remove shading from hudi utilities bundle
umehrot2 Nov 8, 2019
e090ad8
Add maven profile to optionally shade hive in utilities bundle
umehrot2 Nov 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,12 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.jdbc.HiveDriver;
Expand Down Expand Up @@ -85,7 +87,7 @@ public class HoodieHiveClient {
private final HoodieTableMetaClient metaClient;
private final HoodieTableType tableType;
private final PartitionValueExtractor partitionValueExtractor;
private HiveMetaStoreClient client;
private IMetaStoreClient client;
private HiveSyncConfig syncConfig;
private FileSystem fs;
private Connection connection;
Expand All @@ -106,8 +108,8 @@ public HoodieHiveClient(HiveSyncConfig cfg, HiveConf configuration, FileSystem f
createHiveConnection();
}
try {
this.client = new HiveMetaStoreClient(configuration);
} catch (MetaException e) {
this.client = Hive.get(configuration).getMSC();
} catch (MetaException | HiveException e) {
throw new HoodieHiveSyncException("Failed to create HiveMetaStoreClient", e);
}

Expand Down Expand Up @@ -609,7 +611,7 @@ public void close() {
connection.close();
}
if (client != null) {
client.close();
Hive.closeCurrent();
client = null;
}
} catch (SQLException e) {
Expand Down
69 changes: 23 additions & 46 deletions packaging/hudi-spark-bundle/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,31 +105,31 @@
</relocation>
<relocation>
<pattern>org.apache.hive.jdbc.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.jdbc.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.jdbc.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.metastore.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.metastore.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.common.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.common.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.conf.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.conf.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.service.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.service.</shadedPattern>
<shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>com.codahale.metrics.</pattern>
Expand All @@ -144,40 +144,6 @@
<shadedPattern>org.apache.hudi.com.databricks.</shadedPattern>
</relocation>
<!-- TODO: Revisit GH ISSUE #533 & PR#633-->
Copy link
Member

@vinothchandar vinothchandar Nov 12, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove this line as well?

<!--
<relocation>
<pattern>org.apache.hive.jdbc.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.jdbc.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.metastore.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.metastore.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.conf.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.conf.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.serde2.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.serde2.</shadedPattern>
</relocation>
-->
</relocations>
<filters>
<filter>
Expand Down Expand Up @@ -246,37 +212,48 @@
<groupId>${hive.groupid}</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${spark.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-service-rpc</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${spark.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${spark.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${spark.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${spark.bundle.hive.scope}</scope>
</dependency>

<!-- TODO: Reinvestigate PR 633 -->
</dependencies>

<profiles>
<profile>
<id>spark-bundle-shade-hive</id>
<properties>
<spark.bundle.hive.scope>compile</spark.bundle.hive.scope>
<spark.bundle.hive.shade.prefix>org.apache.hudi.</spark.bundle.hive.shade.prefix>
</properties>
</profile>
</profiles>
</project>

34 changes: 22 additions & 12 deletions packaging/hudi-utilities-bundle/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,31 +113,31 @@
</relocation>
<relocation>
<pattern>org.apache.hive.jdbc.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.jdbc.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hive.jdbc.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.metastore.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.metastore.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.common.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.common.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.common.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.conf.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.conf.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hive.service.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.service.</pattern>
<shadedPattern>org.apache.hudi.org.apache.hadoop_hive.service.</shadedPattern>
<shadedPattern>${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>com.codahale.metrics.</pattern>
Expand Down Expand Up @@ -221,36 +221,46 @@
<groupId>${hive.groupid}</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${utilities.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-service-rpc</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${utilities.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${utilities.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${utilities.bundle.hive.scope}</scope>
</dependency>

<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
<scope>compile</scope>
<scope>${utilities.bundle.hive.scope}</scope>
</dependency>
</dependencies>

<profiles>
<profile>
<id>utilities-bundle-shade-hive</id>
<properties>
<utilities.bundle.hive.scope>compile</utilities.bundle.hive.scope>
<utilities.bundle.hive.shade.prefix>org.apache.hudi.</utilities.bundle.hive.shade.prefix>
</properties>
</profile>
</profiles>
</project>

4 changes: 4 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@
<main.basedir>${project.basedir}</main.basedir>
<mr.bundle.avro.scope>provided</mr.bundle.avro.scope>
<mr.bundle.avro.shade.prefix></mr.bundle.avro.shade.prefix>
<spark.bundle.hive.scope>provided</spark.bundle.hive.scope>
<spark.bundle.hive.shade.prefix></spark.bundle.hive.shade.prefix>
<utilities.bundle.hive.scope>provided</utilities.bundle.hive.scope>
<utilities.bundle.hive.shade.prefix></utilities.bundle.hive.shade.prefix>
</properties>

<scm>
Expand Down