GlobalEndpointManager: Fixes Unobserved and Unhandled Exception from Getting Thrown

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -584,7 +584,12 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
             {
                 this.LastBackgroundRefreshUtc = DateTime.UtcNow;
                 this.locationCache.OnDatabaseAccountRead(await this.GetDatabaseAccountAsync(true));
-
+            }
+            catch (Exception ex)
+            {
+                DefaultTrace.TraceWarning("Failed to refresh database account with exception: {0}. Activity Id: '{1}'",
+                    ex,
+                    System.Diagnostics.Trace.CorrelationManager.ActivityId);
             }
             finally
             {
@@ -596,16 +601,16 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
         }
         internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh = false)
         {
-#nullable disable  // Needed because AsyncCache does not have nullable enabled
-            return await this.databaseAccountCache.GetAsync(
-                              key: string.Empty,
-                              obsoleteValue: null,
-                              singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
-                                  this.defaultEndpoint,
-                                  this.connectionPolicy.PreferredLocations,
-                                  this.GetDatabaseAccountAsync,
-                                  this.cancellationTokenSource.Token),
-                              cancellationToken: this.cancellationTokenSource.Token,
+#nullable disable  // Needed because AsyncCache does not have nullable enabled
+            return await this.databaseAccountCache.GetAsync(
+                              key: string.Empty,
+                              obsoleteValue: null,
+                              singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
+                                  this.defaultEndpoint,
+                                  this.connectionPolicy.PreferredLocations,
+                                  this.GetDatabaseAccountAsync,
+                                  this.cancellationTokenSource.Token),
+                              cancellationToken: this.cancellationTokenSource.Token,
                               forceRefresh: forceRefresh);
 #nullable enable
         }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs

@@ -407,6 +407,45 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync()
             }
         }
 
+        /// <summary>
+        /// Test to validate that when an exception is thrown during a RefreshLocationAsync call
+        /// the exception should not be bubbled up and remain unobserved. The exception should be
+        /// handled gracefully and logged as a warning trace event.
+        /// </summary>
+        [TestMethod]
+        public async Task RefreshLocationAsync_WhenGetDatabaseThrowsException_ShouldNotBubbleUpAsUnobservedException()
+        {
+            // Arrange.
+            Mock<IDocumentClientInternal> mockOwner = new Mock<IDocumentClientInternal>();
+            mockOwner.Setup(owner => owner.ServiceEndpoint).Returns(new Uri("https://defaultendpoint.net/"));
+            mockOwner.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>())).ThrowsAsync(new TaskCanceledException());
+
+            //Create connection policy and populate preferred locations
+            ConnectionPolicy connectionPolicy = new ConnectionPolicy();
+            connectionPolicy.PreferredLocations.Add("ReadLocation1");
+            connectionPolicy.PreferredLocations.Add("ReadLocation2");
+
+            bool isExceptionLogged = false;
+            void TraceHandler(string message)
+            {
+                if (message.Contains("Failed to refresh database account with exception:"))
+                {
+                    isExceptionLogged = true;
+                }
+            }
+
+            DefaultTrace.TraceSource.Listeners.Add(new TestTraceListener { Callback = TraceHandler });
+            DefaultTrace.InitEventListener();
+
+            using GlobalEndpointManager globalEndpointManager = new (mockOwner.Object, connectionPolicy);
+
+            // Act.
+            await globalEndpointManager.RefreshLocationAsync(forceRefresh: false);
+
+            // Assert.
+            Assert.IsTrue(isExceptionLogged, "The exception was logged as a warning trace event.");
+        }
+
         private sealed class GetAccountRequestInjector
         {
             public Func<Uri, bool> ShouldFailRequest { get; set; }