From 093498f17cff948a69e5d14e5db66583d5b5f9c0 Mon Sep 17 00:00:00 2001 From: Jonathan Vexler <=> Date: Thu, 27 Oct 2022 16:22:53 -0700 Subject: [PATCH 1/3] added and tested change --- .../apache/hudi/sync/common/util/SyncUtilHelpers.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java index 1427eb2887dbe..d8b0f204e0b6d 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java @@ -27,6 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import java.util.Properties; @@ -34,7 +36,7 @@ * Helper class for syncing Hudi commit data with external metastores. */ public class SyncUtilHelpers { - + private static final Logger LOG = LogManager.getLogger(SyncUtilHelpers.class); /** * Create an instance of an implementation of {@link HoodieSyncTool} that will sync all the relevant meta information * with an external metastore such as Hive etc. to ensure Hoodie tables can be queried or read via external systems. @@ -53,6 +55,7 @@ public static void runHoodieMetaSync(String syncToolClassName, String targetBasePath, String baseFileFormat) { try { + LOG.warn("VEXLER HERE58"); instantiateMetaSyncTool(syncToolClassName, props, hadoopConfig, fs, targetBasePath, baseFileFormat).syncHoodieTable(); } catch (Throwable e) { throw new HoodieException("Could not sync using the meta sync class " + syncToolClassName, e); @@ -69,6 +72,12 @@ static HoodieSyncTool instantiateMetaSyncTool(String syncToolClassName, properties.putAll(props); properties.put(HoodieSyncConfig.META_SYNC_BASE_PATH.key(), targetBasePath); properties.put(HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT.key(), baseFileFormat); + String tableName = properties.getString(HoodieSyncConfig.META_SYNC_TABLE_NAME.key()); + LOG.warn("VEXLER HERE75"); + LOG.warn(("VEXLER TABLE NAME IS: " + tableName)); + if (!tableName.equals(tableName.toLowerCase())) { + LOG.warn("Table name \"" + tableName + "\" contains capital letters. Your metastore may auto convert this to lowercase and cause table not found errors."); + } if (ReflectionUtils.hasConstructor(syncToolClassName, new Class[] {Properties.class, Configuration.class})) { From 8b4fe718c7f01cf448a6a35e25fcdc8edb8f2fdf Mon Sep 17 00:00:00 2001 From: Jonathan Vexler <=> Date: Thu, 27 Oct 2022 16:29:55 -0700 Subject: [PATCH 2/3] removed debug prints --- .../java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java index d8b0f204e0b6d..86055813a928e 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java @@ -55,7 +55,6 @@ public static void runHoodieMetaSync(String syncToolClassName, String targetBasePath, String baseFileFormat) { try { - LOG.warn("VEXLER HERE58"); instantiateMetaSyncTool(syncToolClassName, props, hadoopConfig, fs, targetBasePath, baseFileFormat).syncHoodieTable(); } catch (Throwable e) { throw new HoodieException("Could not sync using the meta sync class " + syncToolClassName, e); @@ -73,8 +72,6 @@ static HoodieSyncTool instantiateMetaSyncTool(String syncToolClassName, properties.put(HoodieSyncConfig.META_SYNC_BASE_PATH.key(), targetBasePath); properties.put(HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT.key(), baseFileFormat); String tableName = properties.getString(HoodieSyncConfig.META_SYNC_TABLE_NAME.key()); - LOG.warn("VEXLER HERE75"); - LOG.warn(("VEXLER TABLE NAME IS: " + tableName)); if (!tableName.equals(tableName.toLowerCase())) { LOG.warn("Table name \"" + tableName + "\" contains capital letters. Your metastore may auto convert this to lowercase and cause table not found errors."); } From 7f91a9ba0aba793eb869a44e4ab7f214eea71e83 Mon Sep 17 00:00:00 2001 From: Jonathan Vexler <=> Date: Mon, 31 Oct 2022 14:03:47 -0400 Subject: [PATCH 3/3] made suggested change to warning message --- .../java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java index 86055813a928e..57db00ab98e05 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java @@ -73,7 +73,7 @@ static HoodieSyncTool instantiateMetaSyncTool(String syncToolClassName, properties.put(HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT.key(), baseFileFormat); String tableName = properties.getString(HoodieSyncConfig.META_SYNC_TABLE_NAME.key()); if (!tableName.equals(tableName.toLowerCase())) { - LOG.warn("Table name \"" + tableName + "\" contains capital letters. Your metastore may auto convert this to lowercase and cause table not found errors."); + LOG.warn("Table name \"" + tableName + "\" contains capital letters. Your metastore may automatically convert this to lower case and can cause table not found errors during subsequent syncs."); } if (ReflectionUtils.hasConstructor(syncToolClassName,