diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportServer.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportServer.java index 9b327d5404b47..8396e691e9db1 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportServer.java +++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportServer.java @@ -133,6 +133,8 @@ private void init(String hostToBind, int portToBind) { bootstrap.childHandler(new ChannelInitializer() { @Override protected void initChannel(SocketChannel ch) { + logger.debug("New connection accepted for remote address {}.", ch.remoteAddress()); + RpcHandler rpcHandler = appRpcHandler; for (TransportServerBootstrap bootstrap : bootstraps) { rpcHandler = bootstrap.doBootstrap(ch, rpcHandler); diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 453bff23026d0..589dfcbefb6ea 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -108,8 +108,8 @@ public int numConnectionsPerPeer() { return conf.getInt(SPARK_NETWORK_IO_NUMCONNECTIONSPERPEER_KEY, 1); } - /** Requested maximum length of the queue of incoming connections. Default -1 for no backlog. */ - public int backLog() { return conf.getInt(SPARK_NETWORK_IO_BACKLOG_KEY, -1); } + /** Requested maximum length of the queue of incoming connections. Default is 64. */ + public int backLog() { return conf.getInt(SPARK_NETWORK_IO_BACKLOG_KEY, 64); } /** Number of threads used in the server thread pool. Default to 0, which is 2x#cores. */ public int serverThreads() { return conf.getInt(SPARK_NETWORK_IO_SERVERTHREADS_KEY, 0); } diff --git a/docs/configuration.md b/docs/configuration.md index d20b41671c03b..216995162d034 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -734,6 +734,17 @@ Apart from these, the following properties are also available, and may be useful is 15 seconds by default, calculated as maxRetries * retryWait. + + spark.shuffle.io.backLog + 64 + + Length of the accept queue for the shuffle service. For large applications, this value may + need to be increased, so that incoming connections are not dropped if the service cannot keep + up with a large number of connections arriving in a short period of time. This needs to + be configured wherever the shuffle service itself is running, which may be outside of the + application (see spark.shuffle.service.enabled option below). + + spark.shuffle.service.enabled false @@ -1515,6 +1526,15 @@ Apart from these, the following properties are also available, and may be useful This is used for communicating with the executors and the standalone Master. + + spark.rpc.io.backLog + 64 + + Length of the accept queue for the RPC server. For large applications, this value may + need to be increased, so that incoming connections are not dropped when a large number of + connections arrives in a short period of time. + + spark.network.timeout 120s