diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 589dfcbefb6ea..cc0f2919568ac 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -108,8 +108,12 @@ public int numConnectionsPerPeer() { return conf.getInt(SPARK_NETWORK_IO_NUMCONNECTIONSPERPEER_KEY, 1); } - /** Requested maximum length of the queue of incoming connections. Default is 64. */ - public int backLog() { return conf.getInt(SPARK_NETWORK_IO_BACKLOG_KEY, 64); } + /** + * Requested maximum length of the queue of incoming connections. If < 1, + * the default Netty value of {@link io.netty.util.NetUtil#SOMAXCONN} will be used. + * Default to -1. + */ + public int backLog() { return conf.getInt(SPARK_NETWORK_IO_BACKLOG_KEY, -1); } /** Number of threads used in the server thread pool. Default to 0, which is 2x#cores. */ public int serverThreads() { return conf.getInt(SPARK_NETWORK_IO_SERVERTHREADS_KEY, 0); } diff --git a/docs/configuration.md b/docs/configuration.md index aebd35aa6e7d4..559c5cdbff1d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -844,13 +844,14 @@ Apart from these, the following properties are also available, and may be useful spark.shuffle.io.backLog - 64 + -1 Length of the accept queue for the shuffle service. For large applications, this value may need to be increased, so that incoming connections are not dropped if the service cannot keep up with a large number of connections arriving in a short period of time. This needs to be configured wherever the shuffle service itself is running, which may be outside of the - application (see spark.shuffle.service.enabled option below). + application (see spark.shuffle.service.enabled option below). If set below 1, + will fallback to OS default defined by Netty's io.netty.util.NetUtil#SOMAXCONN.