From d371724cc9596ccabbc751bf73d1ccf625f8d471 Mon Sep 17 00:00:00 2001 From: RexXiong Date: Fri, 22 Jul 2022 10:35:19 +0800 Subject: [PATCH] [HUDI-4447] fix the sync problem when performing delete table operation --- .../org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala index cfb357ee90b1c..7d6db19edfab6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala @@ -258,6 +258,7 @@ trait ProvidesHoodieConfig extends Logging { val options = hoodieCatalogTable.catalogProperties val enableHive = isUsingHiveCatalog(sparkSession) + val partitionFields = hoodieCatalogTable.partitionFields.mkString(",") withSparkConf(sparkSession, options) { Map( @@ -273,7 +274,11 @@ trait ProvidesHoodieConfig extends Logging { HoodieSyncConfig.META_SYNC_ENABLED.key -> enableHive.toString, HiveSyncConfigHolder.HIVE_SYNC_ENABLED.key -> enableHive.toString, HiveSyncConfigHolder.HIVE_SYNC_MODE.key -> hiveSyncConfig.getStringOrDefault(HiveSyncConfigHolder.HIVE_SYNC_MODE, HiveSyncMode.HMS.name()), + HoodieSyncConfig.META_SYNC_DATABASE_NAME.key -> hiveSyncConfig.getStringOrDefault(HoodieSyncConfig.META_SYNC_DATABASE_NAME), + HoodieSyncConfig.META_SYNC_TABLE_NAME.key -> hiveSyncConfig.getStringOrDefault(HoodieSyncConfig.META_SYNC_TABLE_NAME), HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE.key -> hiveSyncConfig.getBoolean(HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE).toString, + HoodieSyncConfig.META_SYNC_PARTITION_FIELDS.key -> partitionFields, + HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS.key -> hiveSyncConfig.getStringOrDefault(HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS), HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key -> hoodieProps.getString(HoodieWriteConfig.DELETE_PARALLELISM_VALUE.key, "200"), SqlKeyGenerator.PARTITION_SCHEMA -> partitionSchema.toDDL )