From e0214ef45953a08edff6498a2c612e94ad5c12bf Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 20 Oct 2025 17:16:14 +0800 Subject: [PATCH] [SPARK-53954][BUILD] Bump Avro 1.12.1 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 6 +++--- docs/sql-data-sources-avro.md | 4 ++-- pom.xml | 2 +- project/SparkBuild.scala | 2 +- .../main/scala/org/apache/spark/sql/avro/AvroOptions.scala | 4 ++-- .../org/apache/spark/sql/hive/client/HiveClientSuite.scala | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 72eb2cd27582..f83a164aaa31 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -21,9 +21,9 @@ arrow-memory-netty-buffer-patch/18.3.0//arrow-memory-netty-buffer-patch-18.3.0.j arrow-memory-netty/18.3.0//arrow-memory-netty-18.3.0.jar arrow-vector/18.3.0//arrow-vector-18.3.0.jar audience-annotations/0.12.0//audience-annotations-0.12.0.jar -avro-ipc/1.12.0//avro-ipc-1.12.0.jar -avro-mapred/1.12.0//avro-mapred-1.12.0.jar -avro/1.12.0//avro-1.12.0.jar +avro-ipc/1.12.1//avro-ipc-1.12.1.jar +avro-mapred/1.12.1//avro-mapred-1.12.1.jar +avro/1.12.1//avro-1.12.1.jar azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar azure-storage/7.0.1//azure-storage-7.0.1.jar diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md index 7c1b71636071..0c0d2dcbe243 100644 --- a/docs/sql-data-sources-avro.md +++ b/docs/sql-data-sources-avro.md @@ -477,7 +477,7 @@ Submission Guide for more details. [adm]: submitting-applications.html#advanced-dependency-management ## Supported types for Avro -> Spark SQL conversion -Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.12.0/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.12.0/specification/#complex-types) under records of Avro. +Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.12.1/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.12.1/specification/#complex-types) under records of Avro. @@ -541,7 +541,7 @@ In addition to the types listed above, it supports reading `union` types. The fo 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true. All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet. -It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.12.0/specification/#logical-types): +It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.12.1/specification/#logical-types):
Avro typeSpark SQL type
diff --git a/pom.xml b/pom.xml index 4d284f718956..7949c65dcf8f 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ --> 4.2.33 - 1.12.0 + 1.12.11.15.31.12.681 diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 49da986ae728..841e4934390b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1193,7 +1193,7 @@ object DependencyOverrides { dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides ++= jacksonDeps.key.value, dependencyOverrides += "jline" % "jline" % "2.14.6", - dependencyOverrides += "org.apache.avro" % "avro" % "1.12.0", + dependencyOverrides += "org.apache.avro" % "avro" % "1.12.1", dependencyOverrides += "org.slf4j" % "slf4j-api" % "2.0.17") } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala index da42333fad0f..d358f2fc0a76 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala @@ -83,14 +83,14 @@ private[sql] class AvroOptions( /** * Top level record name in write result, which is required in Avro spec. - * See https://avro.apache.org/docs/1.12.0/specification/#schema-record . + * See https://avro.apache.org/docs/1.12.1/specification/#schema-record . * Default value is "topLevelRecord" */ val recordName: String = parameters.getOrElse(RECORD_NAME, "topLevelRecord") /** * Record namespace in write result. Default value is "". - * See Avro spec for details: https://avro.apache.org/docs/1.12.0/specification/#schema-record . + * See Avro spec for details: https://avro.apache.org/docs/1.12.1/specification/#schema-record . */ val recordNamespace: String = parameters.getOrElse(RECORD_NAMESPACE, "") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala index d0d848bcb562..585b8a92c8be 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala @@ -816,7 +816,7 @@ class HiveClientSuite(version: String) extends HiveVersionSuite(version) { test("Decimal support of Avro Hive serde") { val tableName = "tab1" // TODO: add the other logical types. For details, see the link: - // https://avro.apache.org/docs/1.12.0/specification/#logical-types + // https://avro.apache.org/docs/1.12.1/specification/#logical-types val avroSchema = """{ | "name": "test_record",
Avro logical typeAvro typeSpark SQL type