diff --git a/core/pom.xml b/core/pom.xml index 4b3e04084d56c..30be9f76ef7ad 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -35,10 +35,6 @@ - - com.thoughtworks.paranamer - paranamer - org.apache.avro avro @@ -46,7 +42,6 @@ org.apache.avro avro-mapred - ${avro.mapred.classifier} com.google.guava @@ -171,6 +166,10 @@ jakarta.servlet-api ${jakartaservlet.version} + + commons-codec + commons-codec + org.apache.commons commons-lang3 diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java index 33be899b6b438..765ee035855d6 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java @@ -20,7 +20,7 @@ import java.util.Comparator; import java.util.LinkedList; -import org.apache.avro.reflect.Nullable; +import javax.annotation.Nullable; import org.apache.spark.TaskContext; import org.apache.spark.memory.MemoryConsumer; diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 3d0b5d8133933..e878bb83b242c 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -22,9 +22,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar arrow-vector/2.0.0//arrow-vector-2.0.0.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar -avro-ipc/1.8.2//avro-ipc-1.8.2.jar -avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar -avro/1.8.2//avro-1.8.2.jar +avro-ipc/1.10.1//avro-ipc-1.10.1.jar +avro-mapred/1.10.1//avro-mapred-1.10.1.jar +avro/1.10.1//avro-1.10.1.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar breeze_2.12/1.0//breeze_2.12-1.0.jar @@ -238,7 +238,7 @@ xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar xercesImpl/2.12.0//xercesImpl-2.12.0.jar xml-apis/1.4.01//xml-apis-1.4.01.jar xmlenc/0.52//xmlenc-0.52.jar -xz/1.5//xz-1.5.jar +xz/1.8//xz-1.8.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar zookeeper/3.6.2//zookeeper-3.6.2.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 99a8153c8590e..ebd156045ab2c 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -17,9 +17,9 @@ arrow-memory-netty/2.0.0//arrow-memory-netty-2.0.0.jar arrow-vector/2.0.0//arrow-vector-2.0.0.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar -avro-ipc/1.8.2//avro-ipc-1.8.2.jar -avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar -avro/1.8.2//avro-1.8.2.jar +avro-ipc/1.10.1//avro-ipc-1.10.1.jar +avro-mapred/1.10.1//avro-mapred-1.10.1.jar +avro/1.10.1//avro-1.10.1.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar breeze_2.12/1.0//breeze_2.12-1.0.jar @@ -205,7 +205,7 @@ transaction-api/1.1//transaction-api-1.1.jar univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar velocity/1.5//velocity-1.5.jar xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar -xz/1.5//xz-1.5.jar +xz/1.8//xz-1.8.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar zookeeper/3.6.2//zookeeper-3.6.2.jar diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md index 9ecc6eb91da5a..eeb4bffa57d17 100644 --- a/docs/sql-data-sources-avro.md +++ b/docs/sql-data-sources-avro.md @@ -339,7 +339,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache Submission Guide for more details. ## Supported types for Avro -> Spark SQL conversion -Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.8.2/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.8.2/spec.html#schema_complex) under records of Avro. +Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.10.1/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.10.1/spec.html#schema_complex) under records of Avro. @@ -403,7 +403,7 @@ In addition to the types listed above, it supports reading `union` types. The fo 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true. All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet. -It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types): +It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.10.1/spec.html#Logical+Types):
Avro typeSpark SQL type
diff --git a/external/avro/pom.xml b/external/avro/pom.xml index a8614c4ff76ab..b4ec472639c0a 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -70,6 +70,10 @@ org.apache.sparkspark-tags_${scala.binary.version} + + org.tukaani + xz + target/scala-${scala.binary.version}/classes diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala index 8972b0553c522..a2228f2564dd0 100644 --- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala +++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala @@ -51,14 +51,14 @@ private[sql] class AvroOptions( /** * Top level record name in write result, which is required in Avro spec. - * See https://avro.apache.org/docs/1.8.2/spec.html#schema_record . + * See https://avro.apache.org/docs/1.10.1/spec.html#schema_record . * Default value is "topLevelRecord" */ val recordName: String = parameters.getOrElse("recordName", "topLevelRecord") /** * Record namespace in write result. Default value is "". - * See Avro spec for details: https://avro.apache.org/docs/1.8.2/spec.html#schema_record . + * See Avro spec for details: https://avro.apache.org/docs/1.10.1/spec.html#schema_record . */ val recordNamespace: String = parameters.getOrElse("recordNamespace", "") diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index d3bfb716f515c..6f6e8c7c4cc63 100644 --- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -1015,7 +1015,7 @@ abstract class AvroSuite .save(s"$tempDir/${UUID.randomUUID()}") }.getCause.getMessage assert(message.contains("Caused by: java.lang.NullPointerException: " + - "in test_schema in string null of string in field Name")) + "null of string in string in field Name of test_schema in test_schema")) } } diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 121bc56c0ac49..bc5c77f3c18c7 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -83,7 +83,6 @@ org.apache.avro avro-mapred - ${avro.mapred.classifier} provided diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 9a98d7cca5ca3..ae41cf69fc704 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -100,15 +100,9 @@ ${hadoop-client-runtime.artifact} ${hadoop.version} - - org.apache.avro - avro-ipc - provided - org.apache.avro avro-mapred - ${avro.mapred.classifier} provided diff --git a/pom.xml b/pom.xml index f3cc07137f016..fe13056f0932c 100644 --- a/pom.xml +++ b/pom.xml @@ -148,8 +148,7 @@ the link to metrics.dropwizard.io in docs/monitoring.md. --> 4.1.1 - 1.8.2 - hadoop2 + 1.10.1 1.12.0 1.11.655 @@ -194,10 +193,6 @@ 1.1 3.141.59 2.40.0 - - 2.8 1.8 1.1.0 - - org.apache.avro - avro-ipc - tests - ${avro.version} - test - org.apache.avro avro-mapred ${avro.version} - ${avro.mapred.classifier} ${hive.deps.scope} + + org.apache.avro + avro-ipc-jetty + io.netty netty @@ -1266,10 +1229,19 @@ org.apache.velocity - velocity + velocity-engine-core + + + javax.annotation + javax.annotation-api + + org.tukaani + xz + 1.8 + org.apache.avro avro-mapred - ${avro.mapred.classifier} commons-httpclient
Avro logical typeAvro typeSpark SQL type