diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncTool.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AwsGlueCatalogSyncTool.java similarity index 91% rename from hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncTool.java rename to hudi-aws/src/main/java/org/apache/hudi/aws/sync/AwsGlueCatalogSyncTool.java index 4d008159e2696..b8f0d565df7f7 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncTool.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AwsGlueCatalogSyncTool.java @@ -37,9 +37,9 @@ * * @Experimental */ -public class AWSGlueCatalogSyncTool extends HiveSyncTool { +public class AwsGlueCatalogSyncTool extends HiveSyncTool { - public AWSGlueCatalogSyncTool(Properties props, Configuration hadoopConf) { + public AwsGlueCatalogSyncTool(Properties props, Configuration hadoopConf) { super(props, hadoopConf); } @@ -56,6 +56,6 @@ public static void main(String[] args) { cmd.usage(); System.exit(0); } - new AWSGlueCatalogSyncTool(params.toProps(), new Configuration()).syncHoodieTable(); + new AwsGlueCatalogSyncTool(params.toProps(), new Configuration()).syncHoodieTable(); } } diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java index e34adac580f70..25b6cfcff346a 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java @@ -18,7 +18,7 @@ package org.apache.hudi.sink.utils; -import org.apache.hudi.aws.sync.AWSGlueCatalogSyncTool; +import org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.configuration.FlinkOptions; @@ -74,7 +74,7 @@ private HiveSyncContext(Properties props, HiveConf hiveConf) { public HiveSyncTool hiveSyncTool() { HiveSyncMode syncMode = HiveSyncMode.of(props.getProperty(HIVE_SYNC_MODE.key())); if (syncMode == HiveSyncMode.GLUE) { - return new AWSGlueCatalogSyncTool(props, hiveConf); + return new AwsGlueCatalogSyncTool(props, hiveConf); } return new HiveSyncTool(props, hiveConf); } diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java index ebd5a8e093b5a..6123f64cd59f3 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java @@ -18,6 +18,7 @@ package org.apache.hudi.hive; +import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.sync.common.HoodieSyncConfig; @@ -28,32 +29,40 @@ import java.util.Properties; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_PASS; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BUCKET_SYNC; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BUCKET_SYNC_SPEC; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_COMMENT; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_MODE; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_SCHEMA_STRING_LENGTH_THRESHOLD; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_TABLE_PROPERTIES; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_TABLE_SERDE_PROPERTIES; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USER; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USE_JDBC; -import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USE_PRE_APACHE_INPUT_FORMAT; -import static org.apache.hudi.hive.HiveSyncConfigHolder.METASTORE_URIS; - /** * Configs needed to sync data into the Hive Metastore. */ public class HiveSyncConfig extends HoodieSyncConfig { + /* + * Config constants below are retained here for BWC purpose. + */ + public static final ConfigProperty HIVE_SYNC_ENABLED = HiveSyncConfigHolder.HIVE_SYNC_ENABLED; + public static final ConfigProperty HIVE_USER = HiveSyncConfigHolder.HIVE_USER; + public static final ConfigProperty HIVE_PASS = HiveSyncConfigHolder.HIVE_PASS; + public static final ConfigProperty HIVE_URL = HiveSyncConfigHolder.HIVE_URL; + public static final ConfigProperty HIVE_USE_PRE_APACHE_INPUT_FORMAT = HiveSyncConfigHolder.HIVE_USE_PRE_APACHE_INPUT_FORMAT; + /** + * @deprecated Use {@link #HIVE_SYNC_MODE} instead of this config from 0.9.0 + */ + @Deprecated + public static final ConfigProperty HIVE_USE_JDBC = HiveSyncConfigHolder.HIVE_USE_JDBC; + public static final ConfigProperty METASTORE_URIS = HiveSyncConfigHolder.METASTORE_URIS; + public static final ConfigProperty HIVE_AUTO_CREATE_DATABASE = HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; + public static final ConfigProperty HIVE_IGNORE_EXCEPTIONS = HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; + public static final ConfigProperty HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE = HiveSyncConfigHolder.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE; + public static final ConfigProperty HIVE_SUPPORT_TIMESTAMP_TYPE = HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE; + public static final ConfigProperty HIVE_TABLE_PROPERTIES = HiveSyncConfigHolder.HIVE_TABLE_PROPERTIES; + public static final ConfigProperty HIVE_TABLE_SERDE_PROPERTIES = HiveSyncConfigHolder.HIVE_TABLE_SERDE_PROPERTIES; + public static final ConfigProperty HIVE_SYNC_AS_DATA_SOURCE_TABLE = HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE; + public static final ConfigProperty HIVE_SYNC_SCHEMA_STRING_LENGTH_THRESHOLD = HiveSyncConfigHolder.HIVE_SYNC_SCHEMA_STRING_LENGTH_THRESHOLD; + public static final ConfigProperty HIVE_CREATE_MANAGED_TABLE = HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; + public static final ConfigProperty HIVE_BATCH_SYNC_PARTITION_NUM = HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM; + public static final ConfigProperty HIVE_SYNC_MODE = HiveSyncConfigHolder.HIVE_SYNC_MODE; + public static final ConfigProperty HIVE_SYNC_BUCKET_SYNC = HiveSyncConfigHolder.HIVE_SYNC_BUCKET_SYNC; + public static final ConfigProperty HIVE_SYNC_BUCKET_SYNC_SPEC = HiveSyncConfigHolder.HIVE_SYNC_BUCKET_SYNC_SPEC; + public static final ConfigProperty HIVE_SYNC_COMMENT = HiveSyncConfigHolder.HIVE_SYNC_COMMENT; + public static String getBucketSpec(String bucketCols, int bucketNum) { return "CLUSTERED BY (" + bucketCols + " INTO " + bucketNum + " BUCKETS"; } diff --git a/packaging/hudi-aws-bundle/pom.xml b/packaging/hudi-aws-bundle/pom.xml index c366159331f29..e9c17a4892699 100644 --- a/packaging/hudi-aws-bundle/pom.xml +++ b/packaging/hudi-aws-bundle/pom.xml @@ -279,7 +279,7 @@ hudi-aws ${project.version} - org.apache.parquet