Log linked project connection errors at debug during shutdown

server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.transport;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -241,8 +242,9 @@ private void connectionAttemptCompleted(@Nullable Exception e) {
         if (e == null) {
             logger.debug(msgSupplier);
         } else {
-            logger.warn(msgSupplier, e);
-            if (connectionAttemptFailures != null) {
+            final var isClosed = isClosed();
+            logger.log(isClosed ? Level.DEBUG : Level.WARN, msgSupplier, e);
+            if (isClosed == false && connectionAttemptFailures != null) {
                 connectionAttemptFailures.add(
                     1,
                     Map.of(

server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java

@@ -33,8 +33,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.test.MockLog.assertThatLogger;
+import static org.elasticsearch.test.MockLog.awaitLogger;
 import static org.elasticsearch.transport.RemoteClusterSettings.ProxyConnectionStrategySettings.PROXY_ADDRESS;
 import static org.elasticsearch.transport.RemoteClusterSettings.REMOTE_CONNECTION_MODE;
 import static org.elasticsearch.transport.RemoteClusterSettings.SniffConnectionStrategySettings.REMOTE_CLUSTER_SEEDS;
@@ -286,6 +288,43 @@ public void testConnectionAttemptMetricsAndLogging() {
                     }
                 }
             }
+
+            // Now verify connection errors when closing (node shutting down) are logged at debug and not warn.
+            final var strategy = new FakeConnectionStrategy(originProjectId, linkedProjectId, alias, transportService, connectionManager);
+            waitForConnect(strategy);
+            metricRecorder.resetCalls();
+            strategy.setShouldConnectFail(true);
+            strategy.setWaitInConnect(true);
+            final var expectedLogLevel = Level.DEBUG;
+            final var expectedLogMessage = Strings.format(
+                "Origin project [%s] failed to connect to linked project [%s] with alias [%s] on a reconnection attempt",
+                originProjectId,
+                linkedProjectId,
+                alias
+            );
+            awaitLogger(() -> assertThrows(RuntimeException.class, () -> {
+                PlainActionFuture<Void> connectFuture = new PlainActionFuture<>();
+                // Initiate the connection, this will block the connecting thread.
+                strategy.connect(connectFuture);
+                // Close the strategy and manager (similar to RemoteClusterConnection.close()), then let the connection attempt complete.
+                strategy.close();
+                connectionManager.close();
+                strategy.releaseWaitingConnect();
+                connectFuture.actionGet();
+            }),
+                strategy.getClass(),
+                new MockLog.SeenEventExpectation(
+                    "connection strategy should log at " + expectedLogLevel + " after a failed reconnection attempt",
+                    strategy.getClass().getCanonicalName(),
+                    expectedLogLevel,
+                    expectedLogMessage
+                )
+            );
+            // Expect no metric change if the strategy has been closed.
+            metricRecorder.collect();
+            final var counterName = RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME;
+            final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_UP_DOWN_COUNTER, counterName);
+            assertTrue(measurements.isEmpty());
         }
     }
 
@@ -320,6 +359,8 @@ private static class FakeConnectionStrategy extends RemoteConnectionStrategy {
 
         private final ConnectionStrategy strategy;
         private boolean shouldConnectFail;
+        private boolean waitInConnect;
+        private final CountDownLatch waitLatch;
 
         FakeConnectionStrategy(
             ProjectId originProjectId,
@@ -365,12 +406,22 @@ private static class FakeConnectionStrategy extends RemoteConnectionStrategy {
             }, transportService, connectionManager);
             this.strategy = strategy;
             this.shouldConnectFail = false;
+            this.waitInConnect = false;
+            this.waitLatch = new CountDownLatch(1);
         }
 
         void setShouldConnectFail(boolean shouldConnectFail) {
             this.shouldConnectFail = shouldConnectFail;
         }
 
+        void setWaitInConnect(boolean waitInConnect) {
+            this.waitInConnect = waitInConnect;
+        }
+
+        void releaseWaitingConnect() {
+            this.waitLatch.countDown();
+        }
+
         @Override
         protected boolean strategyMustBeRebuilt(LinkedProjectConfig config) {
             return false;
@@ -388,6 +439,9 @@ protected boolean shouldOpenMoreConnections() {
 
         @Override
         protected void connectImpl(ActionListener<Void> listener) {
+            if (waitInConnect) {
+                safeAwait(waitLatch);
+            }
             if (shouldConnectFail) {
                 listener.onFailure(new RuntimeException("simulated failure"));
             } else {