[Socket Handler]: Adds HTTP/2 PING keep-alive to detect broken connections in pool.

Microsoft.Azure.Cosmos.Samples/Usage/HttpClientFactory/Program.cs

@@ -193,7 +193,10 @@ private static void ShareHttpHandlers(
             {
                 PooledConnectionLifetime = TimeSpan.FromMinutes(10), // Customize this value based on desired DNS refresh timer
                 MaxConnectionsPerServer = 20, // Customize the maximum number of allowed connections
-                EnableMultipleHttp2Connections = true // Recommended for thin client (HTTP/2) mode to open additional connections when stream limits are reached
+                EnableMultipleHttp2Connections = true, // Recommended for thin client (HTTP/2) mode to open additional connections when stream limits are reached
+                KeepAlivePingDelay = TimeSpan.FromSeconds(1), // Send HTTP/2 PING after 1s of inactivity to detect broken connections
+                KeepAlivePingTimeout = TimeSpan.FromSeconds(2), // Mark connection dead if no PONG within 2s
+                KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always 
             };
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -244,11 +244,19 @@ public string ApplicationName
         /// </summary>
         /// <remarks>
         /// This setting is only applicable in Gateway mode.
-        /// The SDK sets EnableMultipleHttp2Connections = true on the underlying SocketsHttpHandler,
-        /// allowing additional HTTP/2 TCP connections to be opened when the maximum concurrent streams
-        /// limit on an existing connection is reached. This property controls the upper bound on the
-        /// total number of connections per server endpoint.
-        /// When using a custom <see cref="HttpClientFactory"/>, set EnableMultipleHttp2Connections
+        /// The SDK sets the following on the underlying SocketsHttpHandler:
+        /// <list type="bullet">
+        /// <item><description>EnableMultipleHttp2Connections = true — allows additional HTTP/2 TCP connections
+        /// to be opened when the maximum concurrent streams limit on an existing connection is reached.</description></item>
+        /// <item><description>KeepAlivePingDelay = 1 second — sends HTTP/2 PING frames after 1 second
+        /// of inactivity to detect broken connections in the pool.</description></item>
+        /// <item><description>KeepAlivePingTimeout = 2 seconds — marks a connection as dead if no PONG
+        /// response is received within 2 seconds.</description></item>
+        /// <item><description>KeepAlivePingPolicy = Always — sends pings even for idle connections, which
+        /// is critical for detecting broken connections that remain in the pool.</description></item>
+        /// </list>
+        /// This property controls the upper bound on the total number of connections per server endpoint.
+        /// When using a custom <see cref="HttpClientFactory"/>, configure these properties
         /// directly on your SocketsHttpHandler for equivalent behavior.
         /// </remarks>
         /// <example>
@@ -268,7 +276,10 @@ public string ApplicationName
         /// SocketsHttpHandler handler = new SocketsHttpHandler
         /// {
         ///     MaxConnectionsPerServer = 100,
-        ///     EnableMultipleHttp2Connections = true
+        ///     EnableMultipleHttp2Connections = true,
+        ///     KeepAlivePingDelay = TimeSpan.FromSeconds(1),
+        ///     KeepAlivePingTimeout = TimeSpan.FromSeconds(2),
+        ///     KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always
         /// };
         /// CosmosClientOptions options = new CosmosClientOptions()
         /// {

Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs

@@ -183,6 +183,42 @@ public static HttpMessageHandler CreateSocketsHttpHandlerHelper(
                 DefaultTrace.TraceWarning("Failed to set EnableMultipleHttp2Connections on SocketsHttpHandler: {0}", ex.Message);
             }
 
+            // Enable HTTP/2 PING keep-alive to detect broken connections.
+            // Without this, a broken HTTP/2 connection (e.g. after a network blip or load balancer
+            // reset) can remain in the pool indefinitely, causing persistent request failures
+            // that only resolve after application restart.
+            // KeepAlivePingDelay/Timeout/Policy are available on SocketsHttpHandler in .NET 5.0+.
+            try
+            {
+                int pingDelayInSeconds = ConfigurationManager.GetEnvironmentVariable<int>(
+                    ConfigurationManager.Http2KeepAlivePingDelayInSeconds,
+                    defaultValue: 1);
+
+                int pingTimeoutInSeconds = ConfigurationManager.GetEnvironmentVariable<int>(
+                    ConfigurationManager.Http2KeepAlivePingTimeoutInSeconds,
+                    defaultValue: 2);
+
+                PropertyInfo keepAlivePingDelayInfo = socketHandlerType.GetProperty("KeepAlivePingDelay");
+                keepAlivePingDelayInfo?.SetValue(socketHttpHandler, TimeSpan.FromSeconds(pingDelayInSeconds));
+
+                PropertyInfo keepAlivePingTimeoutInfo = socketHandlerType.GetProperty("KeepAlivePingTimeout");
+                keepAlivePingTimeoutInfo?.SetValue(socketHttpHandler, TimeSpan.FromSeconds(pingTimeoutInSeconds));
+
+                // HttpKeepAlivePingPolicy.Always = 1: send pings even for idle connections,
+                // which is critical for detecting broken connections lingering in the pool.
+                PropertyInfo keepAlivePingPolicyInfo = socketHandlerType.GetProperty("KeepAlivePingPolicy");
+                if (keepAlivePingPolicyInfo != null)
+                {
+                    Type pingPolicyType = keepAlivePingPolicyInfo.PropertyType;
+                    object alwaysValue = Enum.ToObject(pingPolicyType, 1);
+                    keepAlivePingPolicyInfo.SetValue(socketHttpHandler, alwaysValue);
+                }
+            }
+            catch (Exception ex)
+            {
+                DefaultTrace.TraceWarning("Failed to configure HTTP/2 keep-alive ping on SocketsHttpHandler: {0}", ex.Message);
+            }
+
             if (serverCertificateCustomValidationCallback != null)
             {
                 //Get SslOptions Property

Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs

@@ -133,6 +133,19 @@ internal static class ConfigurationManager
         /// </summary>
         internal static readonly string TcpDnsDotSuffixEnabled = "AZURE_COSMOS_TCP_DNS_DOT_SUFFIX_ENABLED";
 
+        /// <summary>
+        /// Environment variable to override the HTTP/2 PING keep-alive delay (in seconds).
+        /// After this many seconds of inactivity on an HTTP/2 connection, a PING frame is sent
+        /// to detect broken connections in the pool. Default: 1 second.
+        /// </summary>
+        internal static readonly string Http2KeepAlivePingDelayInSeconds = "AZURE_COSMOS_HTTP2_KEEPALIVE_PING_DELAY_IN_SECONDS";
+
+        /// <summary>
+        /// Environment variable to override the HTTP/2 PING keep-alive timeout (in seconds).
+        /// If no PONG response is received within this time, the connection is marked dead. Default: 2 seconds.
+        /// </summary>
+        internal static readonly string Http2KeepAlivePingTimeoutInSeconds = "AZURE_COSMOS_HTTP2_KEEPALIVE_PING_TIMEOUT_IN_SECONDS";
+
         public static T GetEnvironmentVariable<T>(string variable, T defaultValue)
         {
             string value = Environment.GetEnvironmentVariable(variable);

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs

@@ -829,6 +829,10 @@ public void VerifyHttpClientHandlerIsSet()
             SocketsHttpHandler handler = (SocketsHttpHandler)cosmosHttpClient.HttpMessageHandler;
 
             Assert.IsTrue(object.ReferenceEquals(webProxy, handler.Proxy));
+            Assert.IsTrue(handler.EnableMultipleHttp2Connections, "EnableMultipleHttp2Connections should be set through the builder pipeline");
+            Assert.AreEqual(TimeSpan.FromSeconds(1), handler.KeepAlivePingDelay, "KeepAlivePingDelay should be set through the builder pipeline");
+            Assert.AreEqual(TimeSpan.FromSeconds(2), handler.KeepAlivePingTimeout, "KeepAlivePingTimeout should be set through the builder pipeline");
+            Assert.AreEqual(HttpKeepAlivePingPolicy.Always, handler.KeepAlivePingPolicy, "KeepAlivePingPolicy should be set through the builder pipeline");
         }
 
         [TestMethod]

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosHttpClientCoreTests.cs

@@ -466,6 +466,11 @@ public void CreateSocketsHttpHandlerCreatesCorrectValueType()
             Assert.AreEqual(gatewayLimit, socketsHandler.MaxConnectionsPerServer);
             Assert.IsTrue(socketsHandler.EnableMultipleHttp2Connections, "EnableMultipleHttp2Connections should be true for HTTP/2 thin client support");
 
+            // HTTP/2 PING keep-alive: detects broken connections lingering in the pool
+            Assert.AreEqual(TimeSpan.FromSeconds(1), socketsHandler.KeepAlivePingDelay, "KeepAlivePingDelay should be 1 second for HTTP/2 connection health monitoring");
+            Assert.AreEqual(TimeSpan.FromSeconds(2), socketsHandler.KeepAlivePingTimeout, "KeepAlivePingTimeout should be 2 seconds");
+            Assert.AreEqual(HttpKeepAlivePingPolicy.Always, socketsHandler.KeepAlivePingPolicy, "KeepAlivePingPolicy should be Always to detect broken idle connections");
+
             //Create cert for test
             X509Certificate2 x509Certificate2 = new CertificateRequest("cn=www.test", ECDsa.Create(), HashAlgorithmName.SHA256).CreateSelfSigned(DateTime.Now, DateTime.Now.AddYears(1));
             X509Chain x509Chain = new X509Chain();
@@ -495,6 +500,46 @@ public void CreateHttpClientHandlerCreatesCorrectValueType()
             Assert.IsFalse(clientHandler.ServerCertificateCustomValidationCallback.Invoke(new HttpRequestMessage(), x509Certificate2, x509Chain, sslPolicyErrors));
         }
 
+        [TestMethod]
+        public void CreateSocketsHttpHandlerRespectsEnvironmentVariableOverrides()
+        {
+            int customPingDelay = 60;
+            int customPingTimeout = 10;
+
+            try
+            {
+                Environment.SetEnvironmentVariable(
+                    ConfigurationManager.Http2KeepAlivePingDelayInSeconds,
+                    customPingDelay.ToString());
+                Environment.SetEnvironmentVariable(
+                    ConfigurationManager.Http2KeepAlivePingTimeoutInSeconds,
+                    customPingTimeout.ToString());
+
+                HttpMessageHandler handler = CosmosHttpClientCore.CreateSocketsHttpHandlerHelper(
+                    gatewayModeMaxConnectionLimit: 10,
+                    webProxy: null,
+                    serverCertificateCustomValidationCallback: null);
+
+                SocketsHttpHandler socketsHandler = (SocketsHttpHandler)handler;
+
+                Assert.AreEqual(TimeSpan.FromSeconds(customPingDelay), socketsHandler.KeepAlivePingDelay,
+                    "KeepAlivePingDelay should respect environment variable override");
+                Assert.AreEqual(TimeSpan.FromSeconds(customPingTimeout), socketsHandler.KeepAlivePingTimeout,
+                    "KeepAlivePingTimeout should respect environment variable override");
+                Assert.AreEqual(HttpKeepAlivePingPolicy.Always, socketsHandler.KeepAlivePingPolicy,
+                    "KeepAlivePingPolicy should always be Always regardless of environment variables");
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(
+                    ConfigurationManager.Http2KeepAlivePingDelayInSeconds,
+                    null);
+                Environment.SetEnvironmentVariable(
+                    ConfigurationManager.Http2KeepAlivePingTimeoutInSeconds,
+                    null);
+            }
+        }
+
         [TestMethod]
         public async Task HttpTimeoutPolicyForThinClientOn503TestAsync()
         {