From ebd75f565d7cd7ba648cc7aac760d0703ef6b9bd Mon Sep 17 00:00:00 2001 From: sivabalan Date: Tue, 20 Sep 2022 15:33:21 -0700 Subject: [PATCH 1/3] Adding org.apache.avro to hudi-hive-sync bundle --- packaging/hudi-hive-sync-bundle/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packaging/hudi-hive-sync-bundle/pom.xml b/packaging/hudi-hive-sync-bundle/pom.xml index 71334018a1c49..d108779ffedb3 100644 --- a/packaging/hudi-hive-sync-bundle/pom.xml +++ b/packaging/hudi-hive-sync-bundle/pom.xml @@ -72,6 +72,7 @@ org.apache.hudi:hudi-hive-sync com.beust:jcommander + org.apache.avro:avro org.apache.hbase:hbase-common org.apache.hbase:hbase-client org.apache.hbase:hbase-hadoop-compat @@ -256,5 +257,13 @@ compile + + + org.apache.avro + avro + ${avro.version} + compile + + From f7a21680fd5a5d8ba99fa05877e982d3e15882a7 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Wed, 28 Sep 2022 14:28:24 -0700 Subject: [PATCH 2/3] fixing avro version for spark3.3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9357efff67b29..62817b127edfc 100644 --- a/pom.xml +++ b/pom.xml @@ -2015,7 +2015,7 @@ ${scalatest.spark3.version} ${kafka.spark3.version} 1.12.2 - 1.11.0 + 1.11.1 1.7.4 4.8 2.13.3 @@ -2124,7 +2124,7 @@ ${scalatest.spark3.version} ${kafka.spark3.version} 1.12.2 - 1.11.0 + 1.11.1 1.7.4 4.8 2.13.3 From 95d5bc16716f6e553d2e34e779ef76e476161a22 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Wed, 28 Sep 2022 22:23:42 -0700 Subject: [PATCH 3/3] fixing test assertion for spark3.3 --- .../test/scala/org/apache/hudi/TestHoodieSparkUtils.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala index 3da7a3ddd4c52..89d8faaf2e702 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala @@ -196,7 +196,11 @@ class TestHoodieSparkUtils { fail("createRdd should fail, because records don't have a column which is not nullable in the passed in schema") } catch { case e: Exception => - assertTrue(e.getMessage.contains("null of string in field new_nested_col of test_namespace.test_struct_name.nullableInnerStruct of union")) + if (HoodieSparkUtils.gteqSpark3_3) { + assertTrue(e.getMessage.contains("null value for (non-nullable) string at test_struct_name.nullableInnerStruct[nullableInnerStruct].new_nested_col")) + } else { + assertTrue(e.getMessage.contains("null of string in field new_nested_col of test_namespace.test_struct_name.nullableInnerStruct of union")) + } } spark.stop() }