From 99835a065c73eecacc5d42b86e564af2eac1ea00 Mon Sep 17 00:00:00 2001 From: pratyakshsharma Date: Wed, 16 Sep 2020 23:46:13 +0530 Subject: [PATCH 1/4] [HUDI-1200]: fixed NPE in CustomKeyGenerator --- .../hudi/keygen/CustomKeyGenerator.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 2ac6b77b28a8d..9dbf4239a4dc3 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -25,6 +25,8 @@ import org.apache.hudi.exception.HoodieKeyException; import org.apache.avro.generic.GenericRecord; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import org.apache.spark.sql.Row; import java.io.IOException; @@ -48,6 +50,7 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator { private static final String DEFAULT_PARTITION_PATH_SEPARATOR = "/"; private static final String SPLIT_REGEX = ":"; + private final TypedProperties props; /** * Used as a part of config in CustomKeyGenerator.java. @@ -58,6 +61,7 @@ public enum PartitionKeyType { public CustomKeyGenerator(TypedProperties props) { super(props); + this.props = props; this.recordKeyFields = Arrays.stream(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(",")).map(String::trim).collect(Collectors.toList()); this.partitionPathFields = Arrays.stream(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(",")).map(String::trim).collect(Collectors.toList()); } @@ -95,17 +99,17 @@ private String getPartitionPath(Option record, Option row) { switch (keyType) { case SIMPLE: if (record.isPresent()) { - partitionPath.append(new SimpleKeyGenerator(config, partitionPathField).getPartitionPath(record.get())); + partitionPath.append(new SimpleKeyGenerator(props, partitionPathField).getPartitionPath(record.get())); } else { - partitionPath.append(new SimpleKeyGenerator(config, partitionPathField).getPartitionPath(row.get())); + partitionPath.append(new SimpleKeyGenerator(props, partitionPathField).getPartitionPath(row.get())); } break; case TIMESTAMP: try { if (record.isPresent()) { - partitionPath.append(new TimestampBasedKeyGenerator(config, partitionPathField).getPartitionPath(record.get())); + partitionPath.append(new TimestampBasedKeyGenerator(props, partitionPathField).getPartitionPath(record.get())); } else { - partitionPath.append(new TimestampBasedKeyGenerator(config, partitionPathField).getPartitionPath(row.get())); + partitionPath.append(new TimestampBasedKeyGenerator(props, partitionPathField).getPartitionPath(row.get())); } } catch (IOException ioe) { throw new HoodieDeltaStreamerException("Unable to initialise TimestampBasedKeyGenerator class"); @@ -126,16 +130,16 @@ private String getPartitionPath(Option record, Option row) { public String getRecordKey(GenericRecord record) { validateRecordKeyFields(); return getRecordKeyFields().size() == 1 - ? new SimpleKeyGenerator(config).getRecordKey(record) - : new ComplexKeyGenerator(config).getRecordKey(record); + ? new SimpleKeyGenerator(props).getRecordKey(record) + : new ComplexKeyGenerator(props).getRecordKey(record); } @Override public String getRecordKey(Row row) { validateRecordKeyFields(); return getRecordKeyFields().size() == 1 - ? new SimpleKeyGenerator(config).getRecordKey(row) - : new ComplexKeyGenerator(config).getRecordKey(row); + ? new SimpleKeyGenerator(props).getRecordKey(row) + : new ComplexKeyGenerator(props).getRecordKey(row); } private void validateRecordKeyFields() { From 45e7ade9c76c5569a29b8e06a3bfefa374daf14d Mon Sep 17 00:00:00 2001 From: pratyakshsharma Date: Thu, 17 Sep 2020 01:35:55 +0530 Subject: [PATCH 2/4] [HUDI-1200]: fixed build --- .../main/java/org/apache/hudi/keygen/CustomKeyGenerator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 9dbf4239a4dc3..9ebe516484cb8 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -25,8 +25,6 @@ import org.apache.hudi.exception.HoodieKeyException; import org.apache.avro.generic.GenericRecord; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.spark.sql.Row; import java.io.IOException; From a7232efe9b197e07c0dbbbfdf73da7f87613be30 Mon Sep 17 00:00:00 2001 From: pratyakshsharma Date: Thu, 17 Sep 2020 13:22:45 +0530 Subject: [PATCH 3/4] [HUDI-1200]: retrigger build --- .../src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 9ebe516484cb8..0800d892b2359 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -120,7 +120,6 @@ private String getPartitionPath(Option record, Option row) { partitionPath.append(DEFAULT_PARTITION_PATH_SEPARATOR); } partitionPath.deleteCharAt(partitionPath.length() - 1); - return partitionPath.toString(); } From 0d63b550bff2967711fd0b6514d40e9b66b3ffda Mon Sep 17 00:00:00 2001 From: pratyakshsharma Date: Mon, 5 Oct 2020 03:05:00 +0530 Subject: [PATCH 4/4] [HUDI-1200]: fixed NPE in CustomKeyGenerator --- .../apache/hudi/keygen/CustomKeyGenerator.java | 18 ++++++++---------- .../org/apache/hudi/keygen/KeyGenerator.java | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 0800d892b2359..e4576882c5d64 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -48,7 +48,6 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator { private static final String DEFAULT_PARTITION_PATH_SEPARATOR = "/"; private static final String SPLIT_REGEX = ":"; - private final TypedProperties props; /** * Used as a part of config in CustomKeyGenerator.java. @@ -59,7 +58,6 @@ public enum PartitionKeyType { public CustomKeyGenerator(TypedProperties props) { super(props); - this.props = props; this.recordKeyFields = Arrays.stream(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(",")).map(String::trim).collect(Collectors.toList()); this.partitionPathFields = Arrays.stream(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(",")).map(String::trim).collect(Collectors.toList()); } @@ -97,17 +95,17 @@ private String getPartitionPath(Option record, Option row) { switch (keyType) { case SIMPLE: if (record.isPresent()) { - partitionPath.append(new SimpleKeyGenerator(props, partitionPathField).getPartitionPath(record.get())); + partitionPath.append(new SimpleKeyGenerator(config, partitionPathField).getPartitionPath(record.get())); } else { - partitionPath.append(new SimpleKeyGenerator(props, partitionPathField).getPartitionPath(row.get())); + partitionPath.append(new SimpleKeyGenerator(config, partitionPathField).getPartitionPath(row.get())); } break; case TIMESTAMP: try { if (record.isPresent()) { - partitionPath.append(new TimestampBasedKeyGenerator(props, partitionPathField).getPartitionPath(record.get())); + partitionPath.append(new TimestampBasedKeyGenerator(config, partitionPathField).getPartitionPath(record.get())); } else { - partitionPath.append(new TimestampBasedKeyGenerator(props, partitionPathField).getPartitionPath(row.get())); + partitionPath.append(new TimestampBasedKeyGenerator(config, partitionPathField).getPartitionPath(row.get())); } } catch (IOException ioe) { throw new HoodieDeltaStreamerException("Unable to initialise TimestampBasedKeyGenerator class"); @@ -127,16 +125,16 @@ private String getPartitionPath(Option record, Option row) { public String getRecordKey(GenericRecord record) { validateRecordKeyFields(); return getRecordKeyFields().size() == 1 - ? new SimpleKeyGenerator(props).getRecordKey(record) - : new ComplexKeyGenerator(props).getRecordKey(record); + ? new SimpleKeyGenerator(config).getRecordKey(record) + : new ComplexKeyGenerator(config).getRecordKey(record); } @Override public String getRecordKey(Row row) { validateRecordKeyFields(); return getRecordKeyFields().size() == 1 - ? new SimpleKeyGenerator(props).getRecordKey(row) - : new ComplexKeyGenerator(props).getRecordKey(row); + ? new SimpleKeyGenerator(config).getRecordKey(row) + : new ComplexKeyGenerator(config).getRecordKey(row); } private void validateRecordKeyFields() { diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenerator.java index 26efa5eedf0e3..63cbce74a8c3f 100644 --- a/hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/KeyGenerator.java @@ -41,7 +41,7 @@ public abstract class KeyGenerator implements Serializable, KeyGeneratorInterfac private static final String STRUCT_NAME = "hoodieRowTopLevelField"; private static final String NAMESPACE = "hoodieRow"; - protected transient TypedProperties config; + protected TypedProperties config; private transient Function1 converterFn = null; protected KeyGenerator(TypedProperties config) {