diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index fe0a73ff11b12..7f03351093329 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -9,6 +9,7 @@ package org.elasticsearch.transport; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.AlreadyClosedException; @@ -241,8 +242,9 @@ private void connectionAttemptCompleted(@Nullable Exception e) { if (e == null) { logger.debug(msgSupplier); } else { - logger.warn(msgSupplier, e); - if (connectionAttemptFailures != null) { + final var isClosed = isClosed(); + logger.log(isClosed ? Level.DEBUG : Level.WARN, msgSupplier, e); + if (isClosed == false && connectionAttemptFailures != null) { connectionAttemptFailures.add( 1, Map.of( diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java index f0504772bcf5e..577bd6f67e065 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java @@ -33,8 +33,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; import static org.elasticsearch.test.MockLog.assertThatLogger; +import static org.elasticsearch.test.MockLog.awaitLogger; import static org.elasticsearch.transport.RemoteClusterSettings.ProxyConnectionStrategySettings.PROXY_ADDRESS; import static org.elasticsearch.transport.RemoteClusterSettings.REMOTE_CONNECTION_MODE; import static org.elasticsearch.transport.RemoteClusterSettings.SniffConnectionStrategySettings.REMOTE_CLUSTER_SEEDS; @@ -286,6 +288,43 @@ public void testConnectionAttemptMetricsAndLogging() { } } } + + // Now verify connection errors when closing (node shutting down) are logged at debug and not warn. + final var strategy = new FakeConnectionStrategy(originProjectId, linkedProjectId, alias, transportService, connectionManager); + waitForConnect(strategy); + metricRecorder.resetCalls(); + strategy.setShouldConnectFail(true); + strategy.setWaitInConnect(true); + final var expectedLogLevel = Level.DEBUG; + final var expectedLogMessage = Strings.format( + "Origin project [%s] failed to connect to linked project [%s] with alias [%s] on a reconnection attempt", + originProjectId, + linkedProjectId, + alias + ); + awaitLogger(() -> assertThrows(RuntimeException.class, () -> { + PlainActionFuture connectFuture = new PlainActionFuture<>(); + // Initiate the connection, this will block the connecting thread. + strategy.connect(connectFuture); + // Close the strategy and manager (similar to RemoteClusterConnection.close()), then let the connection attempt complete. + strategy.close(); + connectionManager.close(); + strategy.releaseWaitingConnect(); + connectFuture.actionGet(); + }), + strategy.getClass(), + new MockLog.SeenEventExpectation( + "connection strategy should log at " + expectedLogLevel + " after a failed reconnection attempt", + strategy.getClass().getCanonicalName(), + expectedLogLevel, + expectedLogMessage + ) + ); + // Expect no metric change if the strategy has been closed. + metricRecorder.collect(); + final var counterName = RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME; + final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_UP_DOWN_COUNTER, counterName); + assertTrue(measurements.isEmpty()); } } @@ -320,6 +359,8 @@ private static class FakeConnectionStrategy extends RemoteConnectionStrategy { private final ConnectionStrategy strategy; private boolean shouldConnectFail; + private boolean waitInConnect; + private final CountDownLatch waitLatch; FakeConnectionStrategy( ProjectId originProjectId, @@ -365,12 +406,22 @@ private static class FakeConnectionStrategy extends RemoteConnectionStrategy { }, transportService, connectionManager); this.strategy = strategy; this.shouldConnectFail = false; + this.waitInConnect = false; + this.waitLatch = new CountDownLatch(1); } void setShouldConnectFail(boolean shouldConnectFail) { this.shouldConnectFail = shouldConnectFail; } + void setWaitInConnect(boolean waitInConnect) { + this.waitInConnect = waitInConnect; + } + + void releaseWaitingConnect() { + this.waitLatch.countDown(); + } + @Override protected boolean strategyMustBeRebuilt(LinkedProjectConfig config) { return false; @@ -388,6 +439,9 @@ protected boolean shouldOpenMoreConnections() { @Override protected void connectImpl(ActionListener listener) { + if (waitInConnect) { + safeAwait(waitLatch); + } if (shouldConnectFail) { listener.onFailure(new RuntimeException("simulated failure")); } else {