Skip to content

Commit ffe114b

Browse files
ywelschywelsch
committed
Set specific keepalive options by default on supported platforms (#59278)
keepalives tell any intermediate devices that the connection remains alive, which helps with overzealous firewalls that are killing idle connections. keepalives are enabled by default in Elasticsearch, but use system defaults for their configuration, which often times do not have reasonable defaults (e.g. 7200s for TCP_KEEP_IDLE) in the context of distributed systems such as Elasticsearch. This PR sets the socket-level keep_alive options for network.tcp.{keep_idle,keep_interval} to 5 minutes on configurations that support it (>= Java 11 & (MacOS || Linux)) and where the system defaults are set to something higher than 5 minutes. This helps keep the connections alive while not interfering with system defaults or user-specified settings unless they are deemed to be set too high by providing better out-of-the-box defaults.
1 parent 981e436 commit ffe114b

File tree

11 files changed

+269
-64
lines changed

11 files changed

+269
-64
lines changed

‎docs/reference/modules/transport.asciidoc‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -80,31 +80,30 @@ this node connects to other nodes in the cluster.
8080
The following parameters can be configured on each transport profile, as in the
8181
example above:
8282

83-
* `port`: The port to bind to
84-
* `bind_host`: The host to bind
85-
* `publish_host`: The host which is published in informational APIs
86-
* `tcp.no_delay`: Configures the `TCP_NO_DELAY` option for this socket
87-
* `tcp.keep_alive`: Configures the `SO_KEEPALIVE` option for this socket
83+
* `port`: The port to which to bind.
84+
* `bind_host`: The host to which to bind.
85+
* `publish_host`: The host which is published in informational APIs.
86+
* `tcp.no_delay`: Configures the `TCP_NO_DELAY` option for this socket.
87+
* `tcp.keep_alive`: Configures the `SO_KEEPALIVE` option for this socket, which
88+
determines whether it sends TCP keepalive probes.
8889
* `tcp.keep_idle`: Configures the `TCP_KEEPIDLE` option for this socket, which
8990
determines the time in seconds that a connection must be idle before
90-
starting to send TCP keepalive probes.
91-
Only applicable on Linux and Mac, and requires JDK 11 or newer.
92-
Defaults to -1, which does not set this option at the socket level, but
93-
uses default system configuration instead.
91+
starting to send TCP keepalive probes. Defaults to `-1` which means to use
92+
the smaller of 300 or the system default. May not be greater than 300. Only
93+
applicable on Linux and macOS, and requires Java 11 or newer.
9494
* `tcp.keep_interval`: Configures the `TCP_KEEPINTVL` option for this socket,
9595
which determines the time in seconds between sending TCP keepalive probes.
96-
Only applicable on Linux and Mac, and requires JDK 11 or newer.
97-
Defaults to -1, which does not set this option at the socket level, but
98-
uses default system configuration instead.
96+
Defaults to `-1` which means to use the smaller of 300 or the system
97+
default. May not be greater than 300. Only applicable on Linux and macOS,
98+
and requires Java 11 or newer.
9999
* `tcp.keep_count`: Configures the `TCP_KEEPCNT` option for this socket, which
100100
determines the number of unacknowledged TCP keepalive probes that may be
101-
sent on a connection before it is dropped.
102-
Only applicable on Linux and Mac, and requires JDK 11 or newer.
103-
Defaults to -1, which does not set this option at the socket level, but
104-
uses default system configuration instead.
105-
* `tcp.reuse_address`: Configures the `SO_REUSEADDR` option for this socket
106-
* `tcp.send_buffer_size`: Configures the send buffer size of the socket
107-
* `tcp.receive_buffer_size`: Configures the receive buffer size of the socket
101+
sent on a connection before it is dropped. Defaults to `-1` which means to
102+
use the system default. Only applicable on Linux and macOS, and requires
103+
Java 11 or newer.
104+
* `tcp.reuse_address`: Configures the `SO_REUSEADDR` option for this socket.
105+
* `tcp.send_buffer_size`: Configures the send buffer size of the socket.
106+
* `tcp.receive_buffer_size`: Configures the receive buffer size of the socket.
108107

109108
[[long-lived-connections]]
110109
===== Long-lived idle connections

‎libs/core/src/main/java/org/elasticsearch/core/internal/net/NetUtils.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,12 @@
1919

2020
package org.elasticsearch.core.internal.net;
2121

22+
import java.io.IOException;
2223
import java.lang.reflect.Field;
2324
import java.net.SocketOption;
25+
import java.net.StandardSocketOptions;
26+
import java.nio.channels.NetworkChannel;
27+
import java.util.Arrays;
2428

2529
/**
2630
* Utilities for network-related methods.
@@ -59,4 +63,47 @@ private static <T> SocketOption<T> getExtendedSocketOptionOrNull(String fieldNam
5963
return null;
6064
}
6165
}
66+
67+
/**
68+
* If SO_KEEPALIVE is enabled (default), this method ensures sane default values for the extended socket options
69+
* TCP_KEEPIDLE and TCP_KEEPINTERVAL. The default value for TCP_KEEPIDLE is system dependent, but is typically 2 hours.
70+
* Such a high value can result in firewalls eagerly closing these connections. To tell any intermediate devices that
71+
* the connection remains alive, we explicitly set these options to 5 minutes if the defaults are higher than that.
72+
*/
73+
public static void tryEnsureReasonableKeepAliveConfig(NetworkChannel socketChannel) {
74+
assert socketChannel != null;
75+
try {
76+
if (socketChannel.supportedOptions().contains(StandardSocketOptions.SO_KEEPALIVE)) {
77+
final Boolean keepalive = socketChannel.getOption(StandardSocketOptions.SO_KEEPALIVE);
78+
assert keepalive != null;
79+
if (keepalive.booleanValue()) {
80+
for (SocketOption<Integer> option : Arrays.asList(
81+
NetUtils.getTcpKeepIdleSocketOptionOrNull(),
82+
NetUtils.getTcpKeepIntervalSocketOptionOrNull())) {
83+
setMinValueForSocketOption(socketChannel, option, 300);
84+
}
85+
}
86+
}
87+
} catch (Exception e) {
88+
// Getting an exception here should be ok when concurrently closing the channel
89+
// An UnsupportedOperationException or IllegalArgumentException, however, should not happen
90+
assert e instanceof IOException : e;
91+
}
92+
}
93+
94+
private static void setMinValueForSocketOption(NetworkChannel socketChannel, SocketOption<Integer> option, int minValue) {
95+
if (option != null && socketChannel.supportedOptions().contains(option)) {
96+
try {
97+
final Integer currentIdleVal = socketChannel.getOption(option);
98+
assert currentIdleVal != null;
99+
if (currentIdleVal.intValue() > minValue) {
100+
socketChannel.setOption(option, minValue);
101+
}
102+
} catch (Exception e) {
103+
// Getting an exception here should be ok when concurrently closing the channel
104+
// An UnsupportedOperationException or IllegalArgumentException, however, should not happen
105+
assert e instanceof IOException : e;
106+
}
107+
}
108+
}
62109
}

‎libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ private void configureSocket(Socket socket, boolean isConnectComplete) throws IO
350350
}
351351
}
352352
}
353+
NetUtils.tryEnsureReasonableKeepAliveConfig(socket.getChannel());
353354
socket.setTcpNoDelay(socketConfig.tcpNoDelay());
354355
int tcpSendBufferSize = socketConfig.tcpSendBufferSize();
355356
if (tcpSendBufferSize > 0) {

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/CopyBytesSocketChannel.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
* local buffer with a defined size.
6060
*/
6161
@SuppressForbidden(reason = "Channel#write")
62-
public class CopyBytesSocketChannel extends NioSocketChannel {
62+
public class CopyBytesSocketChannel extends Netty4NioSocketChannel {
6363

6464
private static final int MAX_BYTES_PER_WRITE = StrictMath.toIntExact(ByteSizeValue.parseBytesSizeValue(
6565
System.getProperty("es.transport.buffer.size", "1m"), "es.transport.buffer.size").getBytes());

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4NioSocketChannel.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.transport;
21+
22+
import io.netty.channel.Channel;
23+
import io.netty.channel.socket.nio.NioSocketChannel;
24+
25+
import java.nio.channels.SocketChannel;
26+
27+
/**
28+
* Helper class to expose {@link #javaChannel()} method
29+
*/
30+
public class Netty4NioSocketChannel extends NioSocketChannel {
31+
32+
public Netty4NioSocketChannel() {
33+
super();
34+
}
35+
36+
public Netty4NioSocketChannel(Channel parent, SocketChannel socket) {
37+
super(parent, socket);
38+
}
39+
40+
@Override
41+
public SocketChannel javaChannel() {
42+
return super.javaChannel();
43+
}
44+
45+
}

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import io.netty.channel.Channel;
2828
import io.netty.channel.ServerChannel;
2929
import io.netty.channel.socket.nio.NioServerSocketChannel;
30-
import io.netty.channel.socket.nio.NioSocketChannel;
3130
import org.elasticsearch.common.Booleans;
3231
import org.elasticsearch.monitor.jvm.JvmInfo;
3332

@@ -68,7 +67,7 @@ public static Class<? extends Channel> getChannelType() {
6867
if (ALLOCATOR instanceof NoDirectBuffers) {
6968
return CopyBytesSocketChannel.class;
7069
} else {
71-
return NioSocketChannel.class;
70+
return Netty4NioSocketChannel.class;
7271
}
7372
}
7473

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java‎

Lines changed: 39 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,12 @@
4949
import org.elasticsearch.common.unit.ByteSizeValue;
5050
import org.elasticsearch.common.util.PageCacheRecycler;
5151
import org.elasticsearch.common.util.concurrent.EsExecutors;
52-
import org.elasticsearch.core.internal.io.IOUtils;
5352
import org.elasticsearch.core.internal.net.NetUtils;
5453
import org.elasticsearch.indices.breaker.CircuitBreakerService;
5554
import org.elasticsearch.threadpool.ThreadPool;
56-
import org.elasticsearch.transport.SharedGroupFactory;
55+
import org.elasticsearch.transport.Netty4NioSocketChannel;
5756
import org.elasticsearch.transport.NettyAllocator;
57+
import org.elasticsearch.transport.SharedGroupFactory;
5858
import org.elasticsearch.transport.TcpTransport;
5959
import org.elasticsearch.transport.TransportSettings;
6060

@@ -143,31 +143,30 @@ private Bootstrap createClientBootstrap(SharedGroupFactory.SharedGroup sharedGro
143143
bootstrap.group(sharedGroup.getLowLevelGroup());
144144

145145
// NettyAllocator will return the channel type designed to work with the configured allocator
146+
assert Netty4NioSocketChannel.class.isAssignableFrom(NettyAllocator.getChannelType());
146147
bootstrap.channel(NettyAllocator.getChannelType());
147148
bootstrap.option(ChannelOption.ALLOCATOR, NettyAllocator.getAllocator());
148149

149150
bootstrap.option(ChannelOption.TCP_NODELAY, TransportSettings.TCP_NO_DELAY.get(settings));
150151
bootstrap.option(ChannelOption.SO_KEEPALIVE, TransportSettings.TCP_KEEP_ALIVE.get(settings));
151152
if (TransportSettings.TCP_KEEP_ALIVE.get(settings)) {
152-
// Netty logs a warning if it can't set the option, so try this only on supported platforms
153-
if (IOUtils.LINUX || IOUtils.MAC_OS_X) {
154-
if (TransportSettings.TCP_KEEP_IDLE.get(settings) >= 0) {
155-
final SocketOption<Integer> keepIdleOption = NetUtils.getTcpKeepIdleSocketOptionOrNull();
156-
if (keepIdleOption != null) {
157-
bootstrap.option(NioChannelOption.of(keepIdleOption), TransportSettings.TCP_KEEP_IDLE.get(settings));
158-
}
153+
// Note that Netty logs a warning if it can't set the option
154+
if (TransportSettings.TCP_KEEP_IDLE.get(settings) >= 0) {
155+
final SocketOption<Integer> keepIdleOption = NetUtils.getTcpKeepIdleSocketOptionOrNull();
156+
if (keepIdleOption != null) {
157+
bootstrap.option(NioChannelOption.of(keepIdleOption), TransportSettings.TCP_KEEP_IDLE.get(settings));
159158
}
160-
if (TransportSettings.TCP_KEEP_INTERVAL.get(settings) >= 0) {
161-
final SocketOption<Integer> keepIntervalOption = NetUtils.getTcpKeepIntervalSocketOptionOrNull();
162-
if (keepIntervalOption != null) {
163-
bootstrap.option(NioChannelOption.of(keepIntervalOption), TransportSettings.TCP_KEEP_INTERVAL.get(settings));
164-
}
159+
}
160+
if (TransportSettings.TCP_KEEP_INTERVAL.get(settings) >= 0) {
161+
final SocketOption<Integer> keepIntervalOption = NetUtils.getTcpKeepIntervalSocketOptionOrNull();
162+
if (keepIntervalOption != null) {
163+
bootstrap.option(NioChannelOption.of(keepIntervalOption), TransportSettings.TCP_KEEP_INTERVAL.get(settings));
165164
}
166-
if (TransportSettings.TCP_KEEP_COUNT.get(settings) >= 0) {
167-
final SocketOption<Integer> keepCountOption = NetUtils.getTcpKeepCountSocketOptionOrNull();
168-
if (keepCountOption != null) {
169-
bootstrap.option(NioChannelOption.of(keepCountOption), TransportSettings.TCP_KEEP_COUNT.get(settings));
170-
}
165+
}
166+
if (TransportSettings.TCP_KEEP_COUNT.get(settings) >= 0) {
167+
final SocketOption<Integer> keepCountOption = NetUtils.getTcpKeepCountSocketOptionOrNull();
168+
if (keepCountOption != null) {
169+
bootstrap.option(NioChannelOption.of(keepCountOption), TransportSettings.TCP_KEEP_COUNT.get(settings));
171170
}
172171
}
173172
}
@@ -215,26 +214,24 @@ private void createServerBootstrap(ProfileSettings profileSettings, SharedGroupF
215214
serverBootstrap.childOption(ChannelOption.TCP_NODELAY, profileSettings.tcpNoDelay);
216215
serverBootstrap.childOption(ChannelOption.SO_KEEPALIVE, profileSettings.tcpKeepAlive);
217216
if (profileSettings.tcpKeepAlive) {
218-
// Netty logs a warning if it can't set the option, so try this only on supported platforms
219-
if (IOUtils.LINUX || IOUtils.MAC_OS_X) {
220-
if (profileSettings.tcpKeepIdle >= 0) {
221-
final SocketOption<Integer> keepIdleOption = NetUtils.getTcpKeepIdleSocketOptionOrNull();
222-
if (keepIdleOption != null) {
223-
serverBootstrap.childOption(NioChannelOption.of(keepIdleOption), profileSettings.tcpKeepIdle);
224-
}
217+
// Note that Netty logs a warning if it can't set the option
218+
if (profileSettings.tcpKeepIdle >= 0) {
219+
final SocketOption<Integer> keepIdleOption = NetUtils.getTcpKeepIdleSocketOptionOrNull();
220+
if (keepIdleOption != null) {
221+
serverBootstrap.childOption(NioChannelOption.of(keepIdleOption), profileSettings.tcpKeepIdle);
225222
}
226-
if (profileSettings.tcpKeepInterval >= 0) {
227-
final SocketOption<Integer> keepIntervalOption = NetUtils.getTcpKeepIntervalSocketOptionOrNull();
228-
if (keepIntervalOption != null) {
229-
serverBootstrap.childOption(NioChannelOption.of(keepIntervalOption), profileSettings.tcpKeepInterval);
230-
}
231-
223+
}
224+
if (profileSettings.tcpKeepInterval >= 0) {
225+
final SocketOption<Integer> keepIntervalOption = NetUtils.getTcpKeepIntervalSocketOptionOrNull();
226+
if (keepIntervalOption != null) {
227+
serverBootstrap.childOption(NioChannelOption.of(keepIntervalOption), profileSettings.tcpKeepInterval);
232228
}
233-
if (profileSettings.tcpKeepCount >= 0) {
234-
final SocketOption<Integer> keepCountOption = NetUtils.getTcpKeepCountSocketOptionOrNull();
235-
if (keepCountOption != null) {
236-
serverBootstrap.childOption(NioChannelOption.of(keepCountOption), profileSettings.tcpKeepCount);
237-
}
229+
230+
}
231+
if (profileSettings.tcpKeepCount >= 0) {
232+
final SocketOption<Integer> keepCountOption = NetUtils.getTcpKeepCountSocketOptionOrNull();
233+
if (keepCountOption != null) {
234+
serverBootstrap.childOption(NioChannelOption.of(keepCountOption), profileSettings.tcpKeepCount);
238235
}
239236
}
240237
}
@@ -281,7 +278,6 @@ protected Netty4TcpChannel initiateChannel(DiscoveryNode node) throws IOExceptio
281278
ExceptionsHelper.maybeDieOnAnotherThread(connectFuture.cause());
282279
throw new IOException(connectFuture.cause());
283280
}
284-
addClosedExceptionLogger(channel);
285281

286282
Netty4TcpChannel nettyChannel = new Netty4TcpChannel(channel, false, "default", connectFuture);
287283
channel.attr(CHANNEL_KEY).set(nettyChannel);
@@ -311,6 +307,9 @@ protected class ClientChannelInitializer extends ChannelInitializer<Channel> {
311307

312308
@Override
313309
protected void initChannel(Channel ch) throws Exception {
310+
addClosedExceptionLogger(ch);
311+
assert ch instanceof Netty4NioSocketChannel;
312+
NetUtils.tryEnsureReasonableKeepAliveConfig(((Netty4NioSocketChannel) ch).javaChannel());
314313
ch.pipeline().addLast("logging", new ESLoggingHandler());
315314
// using a dot as a prefix means this cannot come from any settings parsed
316315
ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(pageCacheRecycler, Netty4Transport.this));
@@ -334,6 +333,8 @@ protected ServerChannelInitializer(String name) {
334333
@Override
335334
protected void initChannel(Channel ch) throws Exception {
336335
addClosedExceptionLogger(ch);
336+
assert ch instanceof Netty4NioSocketChannel;
337+
NetUtils.tryEnsureReasonableKeepAliveConfig(((Netty4NioSocketChannel) ch).javaChannel());
337338
Netty4TcpChannel nettyTcpChannel = new Netty4TcpChannel(ch, true, name, ch.newSucceededFuture());
338339
ch.attr(CHANNEL_KEY).set(nettyTcpChannel);
339340
ch.pipeline().addLast("logging", new ESLoggingHandler());

0 commit comments

Comments
 (0)