LocationCache: Fixes read fallback to use WriteEndpoints[0] when PPAF enabled and all regions excluded

Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs

@@ -74,12 +74,12 @@ public class RequestOptions
         /// Threshold values for Distributed Tracing. 
         /// These values decides whether to generate operation level <see cref="System.Diagnostics.Tracing.EventSource"/> with request diagnostics or not.
         /// </summary>
-        public CosmosThresholdOptions CosmosThresholdOptions { get; set; }
-
+        public CosmosThresholdOptions CosmosThresholdOptions { get; set; }
+
         /// <summary>
         /// List of regions to be excluded routing the request to.
-        /// This can be used to route a request to a specific region by excluding all other regions.
-        /// If all regions are excluded, then the request will be routed to the primary/hub region.
+        /// This can be used to route a request to a specific region by excluding all other regions.
+        /// If all regions are excluded, the SDK will route the request on a best-effort basis to maintain availability.
         /// </summary>
         public List<string> ExcludeRegions { get; set; }
 

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -58,7 +58,8 @@ public GlobalEndpointManager(
                 owner.ServiceEndpoint,
                 connectionPolicy.EnableEndpointDiscovery,
                 connectionPolicy.MaxConnectionLimit,
-                connectionPolicy.UseMultipleWriteLocations);
+                connectionPolicy.UseMultipleWriteLocations,
+                isPartitionLevelFailoverEnabled: () => connectionPolicy.EnablePartitionLevelFailover);
 
             this.owner = owner;
             this.defaultEndpoint = owner.ServiceEndpoint;

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -29,6 +29,7 @@ internal sealed class LocationCache
         private readonly int connectionLimit;
         private readonly ConcurrentDictionary<Uri, LocationUnavailabilityInfo> locationUnavailablityInfoByEndpoint;
         private readonly RegionNameMapper regionNameMapper;
+        private readonly Func<bool> isPartitionLevelFailoverEnabled;
 
         private DatabaseAccountLocationsInfo locationInfo;
         private DateTime lastCacheUpdateTimestamp;
@@ -39,13 +40,15 @@ public LocationCache(
             Uri defaultEndpoint,
             bool enableEndpointDiscovery,
             int connectionLimit,
-            bool useMultipleWriteLocations)
+            bool useMultipleWriteLocations,
+            Func<bool> isPartitionLevelFailoverEnabled = null)
         {
             this.locationInfo = new DatabaseAccountLocationsInfo(preferredLocations, defaultEndpoint);
             this.defaultEndpoint = defaultEndpoint;
             this.enableEndpointDiscovery = enableEndpointDiscovery;
             this.useMultipleWriteLocations = useMultipleWriteLocations;
             this.connectionLimit = connectionLimit;
+            this.isPartitionLevelFailoverEnabled = isPartitionLevelFailoverEnabled;
 
             this.lockObject = new object();
             this.locationUnavailablityInfoByEndpoint = new ConcurrentDictionary<Uri, LocationUnavailabilityInfo>();
@@ -380,10 +383,18 @@ public ReadOnlyCollection<Uri> GetApplicableEndpoints(DocumentServiceRequest req
 
             ReadOnlyCollection<string> effectivePreferredLocations = databaseAccountLocationsInfoSnapshot.EffectivePreferredLocations;
 
+            // For reads when PPAF is enabled, use WriteEndpoints[0] as fallback (dynamic,
+            // tracks current write region) instead of this.defaultEndpoint (static, region-agnostic,
+            // never updated after init). This aligns with UpdateLocationCache which already uses
+            // WriteEndpoints[0] as the ReadEndpoints fallback, and matches Java/Python SDK behavior.
+            Uri fallbackEndpoint = (isReadRequest && this.isPartitionLevelFailoverEnabled?.Invoke() == true)
+                ? databaseAccountLocationsInfoSnapshot.WriteEndpoints[0]
+                : this.defaultEndpoint;
+
             return GetApplicableEndpoints(
-                isReadRequest ? this.locationInfo.AvailableReadEndpointByLocation : this.locationInfo.AvailableWriteEndpointByLocation,
+                isReadRequest ? databaseAccountLocationsInfoSnapshot.AvailableReadEndpointByLocation : databaseAccountLocationsInfoSnapshot.AvailableWriteEndpointByLocation,
                 effectivePreferredLocations,
-                this.defaultEndpoint,
+                fallbackEndpoint,
                 request.RequestContext.ExcludeRegions);
         }
 

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs

@@ -1439,14 +1439,291 @@ public void VerifyRegionExcludedTest(
             }
 
         }
-
-        [TestMethod]
-        public void ValidateThinClientReadFallbackToWriteEndpointTest()
-        {
-            // Arrange:
-            Collection<AccountRegion> normalReads = new Collection<AccountRegion>()
-            {
-                new AccountRegion { Name = "ReadLocation", Endpoint = "https://readlocation.documents.azure.com" }
+
+        [TestMethod]
+        [Description("Validates that read fallback uses WriteEndpoints[0] when PPAF is enabled, and defaultEndpoint when PPAF is disabled. Regression test for issue #5821.")]
+        public void ValidateReadFallbackUsesWriteEndpointAfterHubSwitch()
+        {
+            // Arrange: Single-master account with two regions.
+            // Hub region (write) starts at "location1", read available at both "location1" and "location2".
+            Collection<AccountRegion> writeLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+            };
+
+            Collection<AccountRegion> readLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+            };
+
+            AccountProperties initialAccount = new AccountProperties
+            {
+                ReadLocationsInternal = readLocations,
+                WriteLocationsInternal = writeLocations,
+                EnableMultipleWriteLocations = false,
+            };
+
+            // defaultEndpoint is region-agnostic (static, never updated)
+            Uri defaultEndpoint = new Uri("https://myaccount.documents.azure.com");
+
+            // PPAF enabled — read fallback should use WriteEndpoints[0]
+            LocationCache cache = new LocationCache(
+                preferredLocations: new List<string> { "location1" }.AsReadOnly(),
+                defaultEndpoint: defaultEndpoint,
+                enableEndpointDiscovery: true,
+                connectionLimit: 10,
+                useMultipleWriteLocations: false,
+                isPartitionLevelFailoverEnabled: () => true);
+
+            cache.OnDatabaseAccountRead(initialAccount);
+
+            // Act 1: Read with ExcludeRegions == preferred regions → all excluded → fallback to WriteEndpoints[0]
+            using (DocumentServiceRequest readRequest = DocumentServiceRequest.Create(
+                OperationType.Read,
+                ResourceType.Document,
+                AuthorizationTokenType.PrimaryMasterKey))
+            {
+                readRequest.RequestContext.ExcludeRegions = new List<string> { "location1" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(readRequest, isReadRequest: true);
+
+                Assert.AreEqual(1, endpoints.Count);
+                Assert.AreEqual(
+                    LocationCacheTests.Location1Endpoint,
+                    endpoints[0],
+                    "With PPAF enabled, read fallback should use WriteEndpoints[0], not defaultEndpoint.");
+            }
+
+            // Act 2: Simulate hub switch — write region moves from location1 to location2
+            Collection<AccountRegion> newWriteLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+            };
+
+            Collection<AccountRegion> newReadLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+            };
+
+            AccountProperties updatedAccount = new AccountProperties
+            {
+                ReadLocationsInternal = newReadLocations,
+                WriteLocationsInternal = newWriteLocations,
+                EnableMultipleWriteLocations = false,
+            };
+
+            cache.OnDatabaseAccountRead(updatedAccount);
+
+            // Act 3: Same read after hub switch — WriteEndpoints[0] should now be location2
+            using (DocumentServiceRequest readRequest2 = DocumentServiceRequest.Create(
+                OperationType.Read,
+                ResourceType.Document,
+                AuthorizationTokenType.PrimaryMasterKey))
+            {
+                readRequest2.RequestContext.ExcludeRegions = new List<string> { "location1" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(readRequest2, isReadRequest: true);
+
+                Assert.AreEqual(1, endpoints.Count);
+                Assert.AreEqual(
+                    LocationCacheTests.Location2Endpoint,
+                    endpoints[0],
+                    "After hub switch, read fallback should track the new write region (location2).");
+            }
+
+            // Act 4: Verify write requests still use defaultEndpoint as fallback (unchanged)
+            using (DocumentServiceRequest writeRequest = DocumentServiceRequest.Create(
+                OperationType.Create,
+                ResourceType.Document,
+                AuthorizationTokenType.PrimaryMasterKey))
+            {
+                writeRequest.RequestContext.ExcludeRegions = new List<string> { "location1", "location2" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(writeRequest, isReadRequest: false);
+
+                Assert.AreEqual(1, endpoints.Count);
+                Assert.AreEqual(
+                    defaultEndpoint,
+                    endpoints[0],
+                    "Write fallback should still use defaultEndpoint.");
+            }
+        }
+
+        [TestMethod]
+        [Description("Validates that when PPAF is disabled, read fallback uses defaultEndpoint (original behavior).")]
+        public void ValidateReadFallbackUsesDefaultEndpointWhenPpafDisabled()
+        {
+            Collection<AccountRegion> writeLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+            };
+
+            Collection<AccountRegion> readLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+            };
+
+            AccountProperties account = new AccountProperties
+            {
+                ReadLocationsInternal = readLocations,
+                WriteLocationsInternal = writeLocations,
+                EnableMultipleWriteLocations = false,
+            };
+
+            Uri defaultEndpoint = new Uri("https://myaccount.documents.azure.com");
+
+            // PPAF disabled — read fallback should use defaultEndpoint (original behavior)
+            LocationCache cache = new LocationCache(
+                preferredLocations: new List<string> { "location1" }.AsReadOnly(),
+                defaultEndpoint: defaultEndpoint,
+                enableEndpointDiscovery: true,
+                connectionLimit: 10,
+                useMultipleWriteLocations: false,
+                isPartitionLevelFailoverEnabled: () => false);
+
+            cache.OnDatabaseAccountRead(account);
+
+            using (DocumentServiceRequest readRequest = DocumentServiceRequest.Create(
+                OperationType.Read,
+                ResourceType.Document,
+                AuthorizationTokenType.PrimaryMasterKey))
+            {
+                readRequest.RequestContext.ExcludeRegions = new List<string> { "location1" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(readRequest, isReadRequest: true);
+
+                Assert.AreEqual(1, endpoints.Count);
+                Assert.AreEqual(
+                    defaultEndpoint,
+                    endpoints[0],
+                    "With PPAF disabled, read fallback should use defaultEndpoint.");
+            }
+        }
+
+        [TestMethod]
+        [Description("Validates dynamic PPAF toggle: behavior changes when PPAF is enabled/disabled at runtime.")]
+        public void ValidateReadFallbackReactsToDynamicPpafToggle()
+        {
+            Collection<AccountRegion> writeLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+            };
+
+            Collection<AccountRegion> readLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+            };
+
+            AccountProperties account = new AccountProperties
+            {
+                ReadLocationsInternal = readLocations,
+                WriteLocationsInternal = writeLocations,
+                EnableMultipleWriteLocations = false,
+            };
+
+            Uri defaultEndpoint = new Uri("https://myaccount.documents.azure.com");
+
+            // Start with PPAF disabled, toggle dynamically
+            bool ppafEnabled = false;
+            LocationCache cache = new LocationCache(
+                preferredLocations: new List<string> { "location1" }.AsReadOnly(),
+                defaultEndpoint: defaultEndpoint,
+                enableEndpointDiscovery: true,
+                connectionLimit: 10,
+                useMultipleWriteLocations: false,
+                isPartitionLevelFailoverEnabled: () => ppafEnabled);
+
+            cache.OnDatabaseAccountRead(account);
+
+            // PPAF off → defaultEndpoint
+            using (DocumentServiceRequest req = DocumentServiceRequest.Create(
+                OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey))
+            {
+                req.RequestContext.ExcludeRegions = new List<string> { "location1" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(req, isReadRequest: true);
+                Assert.AreEqual(defaultEndpoint, endpoints[0], "PPAF off: should use defaultEndpoint.");
+            }
+
+            // Toggle PPAF on → WriteEndpoints[0]
+            ppafEnabled = true;
+            using (DocumentServiceRequest req = DocumentServiceRequest.Create(
+                OperationType.Read, ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey))
+            {
+                req.RequestContext.ExcludeRegions = new List<string> { "location1" };
+                ReadOnlyCollection<Uri> endpoints = cache.GetApplicableEndpoints(req, isReadRequest: true);
+                Assert.AreEqual(LocationCacheTests.Location1Endpoint, endpoints[0], "PPAF on: should use WriteEndpoints[0].");
+            }
+        }
+
+        [TestMethod]
+        [Description("Validates that when a PPAF partition-level override (LocationEndpointToRoute) is set, " +
+            "ResolveServiceEndpoint returns it directly, bypassing ExcludeRegions filtering entirely.")]
+        public void ValidateResolveServiceEndpoint_PPAFOverride_WinsOverExcludeRegions()
+        {
+            // Arrange: PPAF enabled, single preferred region "location1"
+            Collection<AccountRegion> writeLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+            };
+
+            Collection<AccountRegion> readLocations = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() },
+                new AccountRegion { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() },
+            };
+
+            AccountProperties account = new AccountProperties
+            {
+                ReadLocationsInternal = readLocations,
+                WriteLocationsInternal = writeLocations,
+                EnableMultipleWriteLocations = false,
+            };
+
+            Uri defaultEndpoint = new Uri("https://myaccount.documents.azure.com");
+
+            LocationCache cache = new LocationCache(
+                preferredLocations: new List<string> { "location1" }.AsReadOnly(),
+                defaultEndpoint: defaultEndpoint,
+                enableEndpointDiscovery: true,
+                connectionLimit: 10,
+                useMultipleWriteLocations: false,
+                isPartitionLevelFailoverEnabled: () => true);
+
+            cache.OnDatabaseAccountRead(account);
+
+            // Simulate PPAF partition-level override: partition failed over to location2
+            Uri ppafOverrideEndpoint = LocationCacheTests.Location2Endpoint;
+
+            using (DocumentServiceRequest readRequest = DocumentServiceRequest.Create(
+                OperationType.Read,
+                ResourceType.Document,
+                AuthorizationTokenType.PrimaryMasterKey))
+            {
+                // ExcludeRegions == PreferredRegions → would normally trigger fallback
+                readRequest.RequestContext.ExcludeRegions = new List<string> { "location1" };
+
+                // Set PPAF override (as GlobalPartitionEndpointManagerCore would do)
+                readRequest.RequestContext.RouteToLocation(ppafOverrideEndpoint);
+
+                // Act
+                Uri resolved = cache.ResolveServiceEndpoint(readRequest);
+
+                // Assert: PPAF override wins at L341 — ExcludeRegions is never evaluated
+                Assert.AreEqual(
+                    ppafOverrideEndpoint,
+                    resolved,
+                    "When a PPAF partition-level override (LocationEndpointToRoute) is present, " +
+                    "ResolveServiceEndpoint should short-circuit and return it, ignoring ExcludeRegions.");
+            }
+        }
+
+        [TestMethod]
+        public void ValidateThinClientReadFallbackToWriteEndpointTest()
+        {
+            // Arrange:
+            Collection<AccountRegion> normalReads = new Collection<AccountRegion>()
+            {
+                new AccountRegion { Name = "ReadLocation", Endpoint = "https://readlocation.documents.azure.com" }
             };
 
             Collection<AccountRegion> normalWrites = new Collection<AccountRegion>()