diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java index 7406b9ca048c3..80d5488d05bc8 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; /** * Complex key generator, which takes names of fields to be used for recordKey and partitionPath as configs. @@ -48,9 +49,11 @@ public class ComplexKeyGenerator extends KeyGenerator { public ComplexKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.asList(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(",")); + this.recordKeyFields = Arrays.asList(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(",")) + .stream().map(String::trim).collect(Collectors.toList()); this.partitionPathFields = - Arrays.asList(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(",")); + Arrays.asList(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(",")) + .stream().map(String::trim).collect(Collectors.toList()); this.hiveStylePartitioning = props.getBoolean(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY(), Boolean.parseBoolean(DataSourceWriteOptions.DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL())); } diff --git a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala index 2ac5fce6b3621..31a04d691f9ea 100644 --- a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala +++ b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala @@ -224,6 +224,17 @@ class TestDataSourceDefaults extends AssertionsForJUnit { case e: HoodieKeyException => // do nothing } + + // reset name and field1 values. + baseRecord.put("name", "name1") + baseRecord.put("field1", "field1") + val hk7 = new ComplexKeyGenerator(getKeyConfig("field1, name", "field1, name", "false")).getKey(baseRecord) + assertEquals("field1:field1,name:name1", hk7.getRecordKey) + assertEquals("field1/name1", hk7.getPartitionPath) + + val hk8 = new ComplexKeyGenerator(getKeyConfig("field1,", "field1,", "false")).getKey(baseRecord) + assertEquals("field1:field1", hk8.getRecordKey) + assertEquals("field1", hk8.getPartitionPath) } @Test def testGlobalDeleteKeyGenerator() = {