From 6a18ca5bc8bc8262073a6fcfbba3b469d43203c8 Mon Sep 17 00:00:00 2001 From: Tobias Stadler Date: Fri, 23 Dec 2022 11:55:13 +0100 Subject: [PATCH] Updated netty to 4.1.86.Final --- common/network-common/pom.xml | 23 ++++++++++ .../apache/spark/network/util/NettyUtils.java | 8 ++++ core/pom.xml | 22 ++++++++++ dev/deps/spark-deps-hadoop-2-hive-2.3 | 29 +++++++------ dev/deps/spark-deps-hadoop-3-hive-2.3 | 29 +++++++------ pom.xml | 42 ++++++++++++++++++- 6 files changed, 122 insertions(+), 31 deletions(-) diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index bfadba306c5e..68caa0155ab0 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -42,10 +42,33 @@ + io.netty netty-all + + io.netty + netty-transport-native-epoll + linux-x86_64 + + + io.netty + netty-transport-native-epoll + linux-aarch_64 + + + io.netty + netty-transport-native-kqueue + osx-aarch_64 + + + io.netty + netty-transport-native-kqueue + osx-x86_64 + + + org.apache.commons commons-lang3 diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java index ac61aa39e5f1..cc4657efe39a 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java @@ -160,6 +160,14 @@ public static PooledByteBufAllocator createPooledByteBufAllocator( if (numCores == 0) { numCores = Runtime.getRuntime().availableProcessors(); } + // SPARK-38541: After upgrade to Netty 4.1.75, there are 2 behavior changes of this method: + // 1. `PooledByteBufAllocator.defaultMaxOrder()` change from 11 to 9, this means the default + // `PooledByteBufAllocator` chunk size reduce from 16 MiB to 4 MiB, we need use + // `-Dio.netty.allocator.maxOrder=11` to keep the chunk size of PooledByteBufAllocator + // to 16m. + // 2. `PooledByteBufAllocator.defaultUseCacheForAllThreads()` change from true to false, we need + // to use `-Dio.netty.allocator.useCacheForAllThreads=true` to + // enable `useCacheForAllThreads`. return new PooledByteBufAllocator( allowDirectBufs && PlatformDependent.directBufferPreferred(), Math.min(PooledByteBufAllocator.defaultNumHeapArena(), numCores), diff --git a/core/pom.xml b/core/pom.xml index d5267f3b32d2..13e264560281 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -304,10 +304,32 @@ jersey-test-framework-provider-simple test + io.netty netty-all + + io.netty + netty-transport-native-epoll + linux-x86_64 + + + io.netty + netty-transport-native-epoll + linux-aarch_64 + + + io.netty + netty-transport-native-kqueue + osx-aarch_64 + + + io.netty + netty-transport-native-kqueue + osx-x86_64 + + com.clearspring.analytics stream diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index 00a75528fc85..fc1f80b51c9a 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -200,21 +200,20 @@ metrics-jmx/4.2.7//metrics-jmx-4.2.7.jar metrics-json/4.2.7//metrics-json-4.2.7.jar metrics-jvm/4.2.7//metrics-jvm-4.2.7.jar minlog/1.3.0//minlog-1.3.0.jar -netty-all/4.1.74.Final//netty-all-4.1.74.Final.jar -netty-buffer/4.1.74.Final//netty-buffer-4.1.74.Final.jar -netty-codec/4.1.74.Final//netty-codec-4.1.74.Final.jar -netty-common/4.1.74.Final//netty-common-4.1.74.Final.jar -netty-handler/4.1.74.Final//netty-handler-4.1.74.Final.jar -netty-resolver/4.1.74.Final//netty-resolver-4.1.74.Final.jar -netty-tcnative-classes/2.0.48.Final//netty-tcnative-classes-2.0.48.Final.jar -netty-transport-classes-epoll/4.1.74.Final//netty-transport-classes-epoll-4.1.74.Final.jar -netty-transport-classes-kqueue/4.1.74.Final//netty-transport-classes-kqueue-4.1.74.Final.jar -netty-transport-native-epoll/4.1.74.Final/linux-aarch_64/netty-transport-native-epoll-4.1.74.Final-linux-aarch_64.jar -netty-transport-native-epoll/4.1.74.Final/linux-x86_64/netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar -netty-transport-native-kqueue/4.1.74.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.74.Final-osx-aarch_64.jar -netty-transport-native-kqueue/4.1.74.Final/osx-x86_64/netty-transport-native-kqueue-4.1.74.Final-osx-x86_64.jar -netty-transport-native-unix-common/4.1.74.Final//netty-transport-native-unix-common-4.1.74.Final.jar -netty-transport/4.1.74.Final//netty-transport-4.1.74.Final.jar +netty-all/4.1.86.Final//netty-all-4.1.86.Final.jar +netty-buffer/4.1.86.Final//netty-buffer-4.1.86.Final.jar +netty-codec/4.1.86.Final//netty-codec-4.1.86.Final.jar +netty-common/4.1.86.Final//netty-common-4.1.86.Final.jar +netty-handler/4.1.86.Final//netty-handler-4.1.86.Final.jar +netty-resolver/4.1.86.Final//netty-resolver-4.1.86.Final.jar +netty-transport-classes-epoll/4.1.86.Final//netty-transport-classes-epoll-4.1.86.Final.jar +netty-transport-classes-kqueue/4.1.86.Final//netty-transport-classes-kqueue-4.1.86.Final.jar +netty-transport-native-epoll/4.1.86.Final/linux-aarch_64/netty-transport-native-epoll-4.1.86.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.86.Final/linux-x86_64/netty-transport-native-epoll-4.1.86.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.86.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.86.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.86.Final/osx-x86_64/netty-transport-native-kqueue-4.1.86.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.86.Final//netty-transport-native-unix-common-4.1.86.Final.jar +netty-transport/4.1.86.Final//netty-transport-4.1.86.Final.jar objenesis/3.2//objenesis-3.2.jar okhttp/3.12.12//okhttp-3.12.12.jar okio/1.14.0//okio-1.14.0.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 63b7e07af3b4..15cab2ae7606 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -186,21 +186,20 @@ metrics-jmx/4.2.7//metrics-jmx-4.2.7.jar metrics-json/4.2.7//metrics-json-4.2.7.jar metrics-jvm/4.2.7//metrics-jvm-4.2.7.jar minlog/1.3.0//minlog-1.3.0.jar -netty-all/4.1.74.Final//netty-all-4.1.74.Final.jar -netty-buffer/4.1.74.Final//netty-buffer-4.1.74.Final.jar -netty-codec/4.1.74.Final//netty-codec-4.1.74.Final.jar -netty-common/4.1.74.Final//netty-common-4.1.74.Final.jar -netty-handler/4.1.74.Final//netty-handler-4.1.74.Final.jar -netty-resolver/4.1.74.Final//netty-resolver-4.1.74.Final.jar -netty-tcnative-classes/2.0.48.Final//netty-tcnative-classes-2.0.48.Final.jar -netty-transport-classes-epoll/4.1.74.Final//netty-transport-classes-epoll-4.1.74.Final.jar -netty-transport-classes-kqueue/4.1.74.Final//netty-transport-classes-kqueue-4.1.74.Final.jar -netty-transport-native-epoll/4.1.74.Final/linux-aarch_64/netty-transport-native-epoll-4.1.74.Final-linux-aarch_64.jar -netty-transport-native-epoll/4.1.74.Final/linux-x86_64/netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar -netty-transport-native-kqueue/4.1.74.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.74.Final-osx-aarch_64.jar -netty-transport-native-kqueue/4.1.74.Final/osx-x86_64/netty-transport-native-kqueue-4.1.74.Final-osx-x86_64.jar -netty-transport-native-unix-common/4.1.74.Final//netty-transport-native-unix-common-4.1.74.Final.jar -netty-transport/4.1.74.Final//netty-transport-4.1.74.Final.jar +netty-all/4.1.86.Final//netty-all-4.1.86.Final.jar +netty-buffer/4.1.86.Final//netty-buffer-4.1.86.Final.jar +netty-codec/4.1.86.Final//netty-codec-4.1.86.Final.jar +netty-common/4.1.86.Final//netty-common-4.1.86.Final.jar +netty-handler/4.1.86.Final//netty-handler-4.1.86.Final.jar +netty-resolver/4.1.86.Final//netty-resolver-4.1.86.Final.jar +netty-transport-classes-epoll/4.1.86.Final//netty-transport-classes-epoll-4.1.86.Final.jar +netty-transport-classes-kqueue/4.1.86.Final//netty-transport-classes-kqueue-4.1.86.Final.jar +netty-transport-native-epoll/4.1.86.Final/linux-aarch_64/netty-transport-native-epoll-4.1.86.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.86.Final/linux-x86_64/netty-transport-native-epoll-4.1.86.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.86.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.86.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.86.Final/osx-x86_64/netty-transport-native-kqueue-4.1.86.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.86.Final//netty-transport-native-unix-common-4.1.86.Final.jar +netty-transport/4.1.86.Final//netty-transport-4.1.86.Final.jar objenesis/3.2//objenesis-3.2.jar okhttp/3.12.12//okhttp-3.12.12.jar okio/1.14.0//okio-1.14.0.jar diff --git a/pom.xml b/pom.xml index edca7df7004c..ee6f95bce8cf 100644 --- a/pom.xml +++ b/pom.xml @@ -202,6 +202,7 @@ 1.5.0 1.60 1.6.1 + 4.1.86.Final io.netty netty-all - 4.1.74.Final + ${netty.version} io.netty @@ -894,8 +897,45 @@ io.netty netty-transport-udt + + io.netty + netty-handler-ssl-ocsp + + + org.jctools + jctools-core + + + + io.netty + netty-transport-native-epoll + ${netty.version} + linux-x86_64 + + + io.netty + netty-transport-native-epoll + ${netty.version} + linux-aarch_64 + + + io.netty + netty-transport-native-kqueue + ${netty.version} + osx-aarch_64 + + + io.netty + netty-transport-native-kqueue + ${netty.version} + osx-x86_64 + + + org.apache.derby derby