Allow adjustment of transport TLS handshake timeout

docs/changelog/130909.yaml

@@ -0,0 +1,5 @@
+pr: 130909
+summary: Allow adjustment of transport TLS handshake timeout
+area: Network
+type: enhancement
+issues: []

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -1933,6 +1933,8 @@ You can configure the following TLS/SSL settings.
 `xpack.security.transport.ssl.trust_restrictions.x509_fields` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
 :   Specifies which field(s) from the TLS certificate is used to match for the restricted trust management that is used for remote clusters connections. This should only be set when a self managed cluster can not create certificates that follow the Elastic Cloud pattern. The default value is ["subjectAltName.otherName.commonName"], the Elastic Cloud pattern. "subjectAltName.dnsName" is also supported and can be configured in addition to or in replacement of the default.
 
+`xpack.security.transport.ssl.handshake_timeout`
+:   Specifies the timeout for a TLS handshake when opening a transport connection. Defaults to `10s`.
 
 ### Transport TLS/SSL key and trusted certificate settings [security-transport-tls-ssl-key-trusted-certificate-settings]
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.common.ssl.SslTrustConfig;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
@@ -118,9 +119,16 @@ public class SSLService {
         Setting.Property.NodeScope
     );
 
+    private static final Setting<TimeValue> TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING = Setting.positiveTimeSetting(
+        "xpack.security.transport.ssl.handshake_timeout",
+        TimeValue.timeValueSeconds(10),
+        Setting.Property.NodeScope
+    );
+
     private final Environment env;
     private final Settings settings;
     private final boolean diagnoseTrustExceptions;
+    private final long handshakeTimeoutMillis;
 
     /**
      * This is a mapping from "context name" (in general use, the name of a setting key)
@@ -156,6 +164,7 @@ public SSLService(Environment environment, Map<String, SslConfiguration> sslConf
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(environment.settings());
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(environment.settings()).millis();
         this.sslConfigurations = sslConfigurations;
         this.sslContexts = loadSslConfigurations(this.sslConfigurations);
     }
@@ -166,6 +175,7 @@ public SSLService(Settings settings, Environment environment) {
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(settings);
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(settings).millis();
         this.sslConfigurations = getSSLConfigurations(env, this.settings);
         this.sslContexts = loadSslConfigurations(this.sslConfigurations);
     }
@@ -178,6 +188,7 @@ private SSLService(
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(environment.settings());
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(environment.settings()).millis();
         this.sslConfigurations = sslConfigurations;
         this.sslContexts = sslContexts;
     }
@@ -214,6 +225,7 @@ SSLContextHolder sslContextHolder(SslConfiguration sslConfiguration) {
 
     public static void registerSettings(List<Setting<?>> settingList) {
         settingList.add(DIAGNOSE_TRUST_EXCEPTIONS_SETTING);
+        settingList.add(TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING);
     }
 
     /**
@@ -979,4 +991,8 @@ private static String sslContextAlgorithm(List<String> supportedProtocols) {
             "no supported SSL/TLS protocol was found in the configured supported protocols: " + supportedProtocols
         );
     }
+
+    public long getHandshakeTimeoutMillis() {
+        return handshakeTimeoutMillis;
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java

@@ -240,6 +240,7 @@ protected void initChannel(Channel ch) throws Exception {
             SSLEngine serverEngine = sslService.createSSLEngine(configuration, null, -1);
             serverEngine.setUseClientMode(false);
             final SslHandler sslHandler = new SslHandler(serverEngine);
+            sslHandler.setHandshakeTimeoutMillis(sslService.getHandshakeTimeoutMillis());
             ch.pipeline().addFirst("sslhandler", sslHandler);
             super.initChannel(ch);
             assert ch.pipeline().first() == sslHandler : "SSL handler must be first handler in pipeline";
@@ -340,6 +341,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock
             }
             final ChannelPromise connectPromise = ctx.newPromise();
             final SslHandler sslHandler = new SslHandler(sslEngine);
+            sslHandler.setHandshakeTimeoutMillis(sslService.getHandshakeTimeoutMillis());
             ctx.pipeline().replace(this, "ssl", sslHandler);
             final Future<?> handshakePromise = sslHandler.handshakeFuture();
             Netty4Utils.addListener(connectPromise, result -> {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java

@@ -11,6 +11,7 @@
 import io.netty.channel.socket.nio.NioChannelOption;
 import io.netty.handler.ssl.SslHandshakeTimeoutException;
 
+import org.apache.logging.log4j.Level;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
@@ -35,13 +36,17 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.mocksocket.MockServerSocket;
+import org.elasticsearch.mocksocket.MockSocket;
+import org.elasticsearch.test.MockLog;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -65,6 +70,7 @@
 import org.elasticsearch.xpack.security.transport.SSLEngineUtils;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
@@ -902,7 +908,15 @@ public void testTcpHandshakeTimeout() throws IOException {
         }
     }
 
+    @TestLogging(reason = "inbound timeout is reported at TRACE", value = "org.elasticsearch.transport.netty4.ESLoggingHandler:TRACE")
     public void testTlsHandshakeTimeout() throws IOException {
+        runOutboundTlsHandshakeTimeoutTest(null);
+        runOutboundTlsHandshakeTimeoutTest(randomLongBetween(1, 500));
+        runInboundTlsHandshakeTimeoutTest(null);
+        runInboundTlsHandshakeTimeoutTest(randomLongBetween(1, 500));
+    }
+
+    private void runOutboundTlsHandshakeTimeoutTest(@Nullable /* to use the default */ Long handshakeTimeoutMillis) throws IOException {
         final CountDownLatch doneLatch = new CountDownLatch(1);
         try (ServerSocket socket = new MockServerSocket()) {
             socket.bind(getLocalEphemeral(), 1);
@@ -928,16 +942,56 @@ public void testTlsHandshakeTimeout() throws IOException {
                 TransportRequestOptions.Type.REG,
                 TransportRequestOptions.Type.STATE
             );
-            final var future = new TestPlainActionFuture<Releasable>();
-            serviceA.connectToNode(dummy, builder.build(), future);
-            final var ex = expectThrows(ExecutionException.class, ConnectTransportException.class, future::get); // long wait
-            assertEquals("[][" + dummy.getAddress() + "] connect_exception", ex.getMessage());
-            assertNotNull(ExceptionsHelper.unwrap(ex, SslHandshakeTimeoutException.class));
+            final ConnectTransportException exception;
+            final var transportSettings = Settings.builder();
+            if (handshakeTimeoutMillis == null) {
+                handshakeTimeoutMillis = 10000L; // default
+            } else {
+                transportSettings.put("xpack.security.transport.ssl.handshake_timeout", TimeValue.timeValueMillis(handshakeTimeoutMillis));
+            }
+            try (var service = buildService(getTestName(), version0, transportVersion0, transportSettings.build())) {
+                final var future = new TestPlainActionFuture<Releasable>();
+                service.connectToNode(dummy, builder.build(), future);
+                exception = expectThrows(ExecutionException.class, ConnectTransportException.class, future::get); // long wait
+                assertEquals("[][" + dummy.getAddress() + "] connect_exception", exception.getMessage());
+                assertThat(
+                    asInstanceOf(SslHandshakeTimeoutException.class, exception.getCause()).getMessage(),
+                    equalTo("handshake timed out after " + handshakeTimeoutMillis + "ms")
+                );
+            }
         } finally {
             doneLatch.countDown();
         }
     }
 
+    @SuppressForbidden(reason = "test needs a simple TCP connection")
+    private void runInboundTlsHandshakeTimeoutTest(@Nullable /* to use the default */ Long handshakeTimeoutMillis) throws IOException {
+        final var transportSettings = Settings.builder();
+        if (handshakeTimeoutMillis == null) {
+            handshakeTimeoutMillis = 10000L; // default
+        } else {
+            transportSettings.put("xpack.security.transport.ssl.handshake_timeout", TimeValue.timeValueMillis(handshakeTimeoutMillis));
+        }
+        try (
+            var service = buildService(getTestName(), version0, transportVersion0, transportSettings.build());
+            Socket clientSocket = new MockSocket();
+            MockLog mockLog = MockLog.capture("org.elasticsearch.transport.netty4.ESLoggingHandler")
+        ) {
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "timeout event message",
+                    "org.elasticsearch.transport.netty4.ESLoggingHandler",
+                    Level.TRACE,
+                    "SslHandshakeTimeoutException: handshake timed out after " + handshakeTimeoutMillis + "ms"
+                )
+            );
+
+            clientSocket.connect(service.boundAddress().boundAddresses()[0].address());
+            expectThrows(EOFException.class, () -> clientSocket.getInputStream().skipNBytes(Long.MAX_VALUE));
+            mockLog.assertAllExpectationsMatched();
+        }
+    }
+
     public void testTcpHandshakeConnectionReset() throws IOException, InterruptedException {
         assumeFalse("Can't run in a FIPS JVM, TrustAllConfig is not a SunJSSE TrustManagers", inFipsJvm());
         SSLService sslService = createSSLService();