From daab65d7f17c8318c4ab8396dbf958bdaea11d6e Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 12:07:33 -0400 Subject: [PATCH 1/9] initial commit --- .../CrossRegionHedgingAvailabilityStrategy.cs | 216 +++++++++--------- .../CosmosAvailabilityStrategyTests.cs | 212 ++++++++++++++--- 2 files changed, 287 insertions(+), 141 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs index 01d7a513fa..5456443206 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs @@ -24,7 +24,7 @@ namespace Microsoft.Azure.Cosmos internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInternal { private const string HedgeContext = "Hedge Context"; - private const string ResponseRegion = "Response Region"; + private const string HedgeConfig = "Hedge Config"; /// /// Latency threshold which activates the first region hedging @@ -44,6 +44,8 @@ internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInte /// public bool EnableMultiWriteRegionHedge { get; private set; } + private readonly string HedgeConfigText; + /// /// Constructor for hedging availability strategy /// @@ -68,6 +70,8 @@ public CrossRegionHedgingAvailabilityStrategy( this.Threshold = threshold; this.ThresholdStep = thresholdStep ?? TimeSpan.FromMilliseconds(-1); this.EnableMultiWriteRegionHedge = enableMultiWriteRegionHedge; + + this.HedgeConfigText = $"t:{this.Threshold.TotalMilliseconds}ms, s:{this.ThresholdStep.TotalMilliseconds}ms, w:{this.EnableMultiWriteRegionHedge}"; } /// @@ -133,121 +137,125 @@ internal override async Task ExecuteAvailabilityStrategyAsync( ? null : await StreamExtension.AsClonableStreamAsync(request.Content))) { - IReadOnlyCollection hedgeRegions = client.DocumentClient.GlobalEndpointManager - .GetApplicableRegions( - request.RequestOptions?.ExcludeRegions, - OperationTypeExtensions.IsReadOperation(request.OperationType)); + using (RequestMessage nonModifiedRequestClone = request.Clone(trace, clonedBody)) + { + IReadOnlyCollection hedgeRegions = client.DocumentClient.GlobalEndpointManager + .GetApplicableRegions( + request.RequestOptions?.ExcludeRegions, + OperationTypeExtensions.IsReadOperation(request.OperationType)); - List requestTasks = new List(hedgeRegions.Count + 1); + List requestTasks = new List(hedgeRegions.Count + 1); - Task primaryRequest = null; - HedgingResponse hedgeResponse = null; - - //Send out hedged requests - for (int requestNumber = 0; requestNumber < hedgeRegions.Count; requestNumber++) - { - TimeSpan awaitTime = requestNumber == 0 ? this.Threshold : this.ThresholdStep; + Task primaryRequest = null; + HedgingResponse hedgeResponse = null; - using (CancellationTokenSource timerTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + //Send out hedged requests + for (int requestNumber = 0; requestNumber < hedgeRegions.Count; requestNumber++) { - CancellationToken timerToken = timerTokenSource.Token; - using (Task hedgeTimer = Task.Delay(awaitTime, timerToken)) - { - if (requestNumber == 0) - { - primaryRequest = this.RequestSenderAndResultCheckAsync( - sender, - request, - hedgeRegions.ElementAt(requestNumber), - cancellationToken, - cancellationTokenSource, - trace); - - requestTasks.Add(primaryRequest); - } - else - { - Task requestTask = this.CloneAndSendAsync( - sender: sender, - request: request, - clonedBody: clonedBody, - hedgeRegions: hedgeRegions, - requestNumber: requestNumber, - trace: trace, - cancellationToken: cancellationToken, - cancellationTokenSource: cancellationTokenSource); - - requestTasks.Add(requestTask); - } - - requestTasks.Add(hedgeTimer); - - Task completedTask = await Task.WhenAny(requestTasks); - requestTasks.Remove(completedTask); - - if (completedTask == hedgeTimer) - { - continue; - } - - timerTokenSource.Cancel(); - requestTasks.Remove(hedgeTimer); + TimeSpan awaitTime = requestNumber == 0 ? this.Threshold : this.ThresholdStep; - if (completedTask.IsFaulted) - { - AggregateException innerExceptions = completedTask.Exception.Flatten(); - } - - hedgeResponse = await (Task)completedTask; - if (hedgeResponse.IsNonTransient) + using (CancellationTokenSource timerTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + { + CancellationToken timerToken = timerTokenSource.Token; + using (Task hedgeTimer = Task.Delay(awaitTime, timerToken)) { - cancellationTokenSource.Cancel(); - //Take is not inclusive, so we need to add 1 to the request number which starts at 0 - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeContext, - hedgeRegions.Take(requestNumber + 1)); - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - ResponseRegion, - hedgeResponse.ResponseRegion); - return hedgeResponse.ResponseMessage; + if (requestNumber == 0) + { + primaryRequest = this.RequestSenderAndResultCheckAsync( + sender, + request, + hedgeRegions.ElementAt(requestNumber), + cancellationToken, + cancellationTokenSource, + trace); + + requestTasks.Add(primaryRequest); + } + else + { + Task requestTask = this.CloneAndSendAsync( + sender: sender, + request: nonModifiedRequestClone, + clonedBody: clonedBody, + hedgeRegions: hedgeRegions, + requestNumber: requestNumber, + trace: trace, + cancellationToken: cancellationToken, + cancellationTokenSource: cancellationTokenSource); + + requestTasks.Add(requestTask); + } + + requestTasks.Add(hedgeTimer); + + Task completedTask = await Task.WhenAny(requestTasks); + requestTasks.Remove(completedTask); + + if (completedTask == hedgeTimer) + { + continue; + } + + timerTokenSource.Cancel(); + requestTasks.Remove(hedgeTimer); + + if (completedTask.IsFaulted) + { + AggregateException innerExceptions = completedTask.Exception.Flatten(); + } + + hedgeResponse = await (Task)completedTask; + if (hedgeResponse.IsNonTransient) + { + cancellationTokenSource.Cancel(); + + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeConfig, + this.HedgeConfigText); + //Take is not inclusive, so we need to add 1 to the request number which starts at 0 + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeContext, + hedgeRegions.Take(requestNumber + 1)); + return hedgeResponse.ResponseMessage; + } } } } - } - //Wait for a good response from the hedged requests/primary request - Exception lastException = null; - while (requestTasks.Any()) - { - Task completedTask = await Task.WhenAny(requestTasks); - requestTasks.Remove(completedTask); - if (completedTask.IsFaulted) + //Wait for a good response from the hedged requests/primary request + Exception lastException = null; + while (requestTasks.Any()) { - AggregateException innerExceptions = completedTask.Exception.Flatten(); - lastException = innerExceptions.InnerExceptions.FirstOrDefault(); + Task completedTask = await Task.WhenAny(requestTasks); + requestTasks.Remove(completedTask); + if (completedTask.IsFaulted) + { + AggregateException innerExceptions = completedTask.Exception.Flatten(); + lastException = innerExceptions.InnerExceptions.FirstOrDefault(); + } + + hedgeResponse = await (Task)completedTask; + if (hedgeResponse.IsNonTransient || requestTasks.Count == 0) + { + cancellationTokenSource.Cancel(); + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeConfig, + this.HedgeConfigText); + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeContext, + hedgeRegions); + return hedgeResponse.ResponseMessage; + } } - hedgeResponse = await (Task)completedTask; - if (hedgeResponse.IsNonTransient || requestTasks.Count == 0) + if (lastException != null) { - cancellationTokenSource.Cancel(); - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeContext, - hedgeRegions); - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - ResponseRegion, - hedgeResponse.ResponseRegion); - return hedgeResponse.ResponseMessage; + throw lastException; } - } - if (lastException != null) - { - throw lastException; + Debug.Assert(hedgeResponse != null); + return hedgeResponse.ResponseMessage; } - - Debug.Assert(hedgeResponse != null); - return hedgeResponse.ResponseMessage; } } } @@ -303,12 +311,12 @@ private async Task RequestSenderAndResultCheckAsync( cancellationTokenSource.Cancel(); } - return new HedgingResponse(true, response, hedgedRegion); + return new HedgingResponse(true, response); } - return new HedgingResponse(false, response, hedgedRegion); + return new HedgingResponse(false, response); } - catch (OperationCanceledException oce ) when (cancellationTokenSource.IsCancellationRequested) + catch (OperationCanceledException oce) when (cancellationTokenSource.IsCancellationRequested) { throw new CosmosOperationCanceledException(oce, trace); } @@ -348,13 +356,11 @@ private sealed class HedgingResponse { public readonly bool IsNonTransient; public readonly ResponseMessage ResponseMessage; - public readonly string ResponseRegion; - public HedgingResponse(bool isNonTransient, ResponseMessage responseMessage, string responseRegion) + public HedgingResponse(bool isNonTransient, ResponseMessage responseMessage) { this.IsNonTransient = isNonTransient; this.ResponseMessage = responseMessage; - this.ResponseRegion = responseRegion; } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index c95073aa69..fb3d7a41cf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -255,14 +255,14 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + responseDelay.Enable(); ItemResponse ir = await container.ReadItemAsync("testId", new PartitionKey("pk")); CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object responseRegion); - Assert.IsNotNull(responseRegion); - Assert.AreEqual(region1, (string)responseRegion); //Should send out hedge request but original should be returned traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); @@ -325,6 +325,9 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + responseDelay.Enable(); ItemRequestOptions requestOptions = new ItemRequestOptions @@ -340,9 +343,11 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + IReadOnlyCollection hedgeContextList; + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); } } @@ -389,6 +394,9 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + responseDelay.Enable(); ItemRequestOptions requestOptions = new ItemRequestOptions { @@ -403,7 +411,7 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - Assert.IsFalse(traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out _)); + Assert.IsFalse(traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object _)); } } @@ -533,8 +541,12 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + CosmosTraceDiagnostics traceDiagnostic; object hedgeContext; + IReadOnlyCollection hedgeContextList; switch (operation) { @@ -556,9 +568,11 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); + break; @@ -588,9 +602,10 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); } break; @@ -619,9 +634,10 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); } break; @@ -649,9 +665,10 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); break; @@ -750,8 +767,12 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + CosmosTraceDiagnostics traceDiagnostic; object hedgeContext; + IReadOnlyCollection hedgeContextList; switch (operation) { @@ -765,9 +786,10 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); break; @@ -792,9 +814,10 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); } break; @@ -813,9 +836,10 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); } break; @@ -835,9 +859,10 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); break; @@ -919,6 +944,9 @@ public async Task AvailabilityStrategyMultiMasterWriteBeforeTest() Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + sendDelay.Enable(); ItemRequestOptions requestOptions = new ItemRequestOptions @@ -944,9 +972,10 @@ public async Task AvailabilityStrategyMultiMasterWriteBeforeTest() CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); } } @@ -987,6 +1016,9 @@ public async Task AvailabilityStrategyMultiMasterWriteAfterTest() Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + responseDelay.Enable(); ItemRequestOptions requestOptions = new ItemRequestOptions @@ -1016,9 +1048,10 @@ public async Task AvailabilityStrategyMultiMasterWriteAfterTest() CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region2, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); } finally { @@ -1079,7 +1112,8 @@ public async Task AvailabilityStrategyMultiMasterWriteBeforeStepTest() Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); - + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); ItemRequestOptions requestOptions = new ItemRequestOptions { @@ -1119,9 +1153,10 @@ await this.container.DeleteItemAsync( CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); } } @@ -1177,6 +1212,9 @@ public async Task AvailabilityStrategyMultiMasterWriteAfterStepTest() Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + ItemRequestOptions requestOptions = new ItemRequestOptions { AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( @@ -1218,9 +1256,10 @@ await this.container.DeleteItemAsync( CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(region3, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); } finally { @@ -1274,14 +1313,114 @@ public async Task AvailabilityStrategyWithCancellationTokenThrowsExceptionTest() Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + //warm up connections read + ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); + CosmosOperationCanceledException cancelledException = await Assert.ThrowsExceptionAsync(() => container.ReadItemAsync( "testId", new PartitionKey("pk"), cancellationToken: cts.Token )); + } + } + [TestMethod] + [TestCategory("MultiMaster")] + public async Task HedgingCancellationTokenHandling() + { + List feedRanges = (List)await this.container.GetFeedRangesAsync(); + Assert.IsTrue(feedRanges.Any()); + + try + { + await this.container.DeleteItemAsync("deleteMe", new PartitionKey("MMWrite")); } + catch (Exception) { } + + + FaultInjectionRule sendDelay = new FaultInjectionRuleBuilder( + id: "sendDelay", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithConnectionType(FaultInjectionConnectionType.Gateway) + .WithEndpoint( + new FaultInjectionEndpointBuilder( + MultiRegionSetupHelpers.dbName, + MultiRegionSetupHelpers.containerName, + feedRanges[0]) + .WithIncludePrimary(true) + .WithReplicaCount(4) + .Build()) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.SendDelay) + .WithDelay(TimeSpan.FromMilliseconds(8000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + List rules = new List() { sendDelay }; + FaultInjector faultInjector = new FaultInjector(rules); + + sendDelay.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + ApplicationPreferredRegions = new List() { region1, region2 }, + Serializer = this.cosmosSystemTextJsonSerializer, + RequestTimeout = TimeSpan.FromMilliseconds(5000) + }; + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); + Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + + sendDelay.Enable(); + + CancellationTokenSource cts = new CancellationTokenSource(); + cts.CancelAfter(TimeSpan.FromSeconds(5)); // Cancellation token expiry time is 5 seconds. + + ItemRequestOptions requestOptions = new ItemRequestOptions + { + AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( + threshold: TimeSpan.FromMilliseconds(100), + thresholdStep: TimeSpan.FromMilliseconds(50), + enableMultiWriteRegionHedge: true) + }; + + CosmosIntegrationTestObject CosmosIntegrationTestObject = new CosmosIntegrationTestObject + { + Id = "deleteMe", + Pk = "MMWrite", + Other = "test" + }; + + try + { + ItemResponse ir = await container.CreateItemAsync( + CosmosIntegrationTestObject, + requestOptions: requestOptions, + cancellationToken: cts.Token); + + CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; + Assert.IsNotNull(traceDiagnostic); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); + Assert.IsNotNull(hedgeContext); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + } + catch (CosmosException ex) + { + Assert.Fail(ex.Message); + } + + + sendDelay.Disable(); + } } private static async Task HandleChangesAsync( @@ -1296,9 +1435,10 @@ private static async Task HandleChangesAsync( CosmosTraceDiagnostics traceDiagnostic = context.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreNotEqual(region1, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region2)); await Task.Delay(1); } @@ -1314,10 +1454,10 @@ private static async Task HandleChangesStepAsync( CosmosTraceDiagnostics traceDiagnostic = context.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreNotEqual(region1, (string)hedgeContext); - Assert.AreNotEqual(region2, (string)hedgeContext); + IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; + Assert.IsTrue(hedgeContextList.Contains(region3)); await Task.Delay(1); } } From 1727fa58b0703a5643c0f6e9e40fee970443252d Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 14:44:25 -0400 Subject: [PATCH 2/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index fb3d7a41cf..fe2c123ab1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -568,6 +568,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); + Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); hedgeContextList = hedgeContext as IReadOnlyCollection; @@ -602,6 +603,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); + Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); hedgeContextList = hedgeContext as IReadOnlyCollection; @@ -634,6 +636,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); + Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); hedgeContextList = hedgeContext as IReadOnlyCollection; @@ -665,6 +668,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); + Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); hedgeContextList = hedgeContext as IReadOnlyCollection; From 5ce2914e4aa4278362e8a0aebb8758c05a020def Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 15:06:13 -0400 Subject: [PATCH 3/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index fe2c123ab1..23cf2b200d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -345,9 +345,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList; - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); } } @@ -546,7 +544,6 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co CosmosTraceDiagnostics traceDiagnostic; object hedgeContext; - IReadOnlyCollection hedgeContextList; switch (operation) { @@ -568,11 +565,9 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); break; @@ -603,11 +598,9 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); } break; @@ -636,11 +629,9 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); } break; @@ -668,11 +659,9 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - Console.WriteLine(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); break; @@ -792,8 +781,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); break; @@ -820,8 +808,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); } break; @@ -842,8 +829,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); } break; @@ -865,8 +851,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); Assert.IsNotNull(hedgeContext); - hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); break; @@ -978,8 +963,7 @@ public async Task AvailabilityStrategyMultiMasterWriteBeforeTest() Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); } } @@ -1054,8 +1038,7 @@ public async Task AvailabilityStrategyMultiMasterWriteAfterTest() Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); } finally { @@ -1159,8 +1142,7 @@ await this.container.DeleteItemAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + IAssert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); } } @@ -1262,8 +1244,7 @@ await this.container.DeleteItemAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); } finally { @@ -1441,8 +1422,7 @@ private static async Task HandleChangesAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); await Task.Delay(1); } @@ -1460,8 +1440,7 @@ private static async Task HandleChangesStepAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList = hedgeContext as IReadOnlyCollection; - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); await Task.Delay(1); } } From 632cfaea60996a3ff41fecda1cb673ae46d3c8b9 Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 15:17:14 -0400 Subject: [PATCH 4/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 23cf2b200d..224f344f5d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -765,7 +765,6 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito CosmosTraceDiagnostics traceDiagnostic; object hedgeContext; - IReadOnlyCollection hedgeContextList; switch (operation) { @@ -1142,7 +1141,7 @@ await this.container.DeleteItemAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); Assert.IsNotNull(hedgeContext); - IAssert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); } } From fbf55062649e7a154d46bdcc51454e02bee6c54b Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 16:08:34 -0400 Subject: [PATCH 5/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 100 +++++++----------- 1 file changed, 38 insertions(+), 62 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 224f344f5d..8a03d3c50d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -264,26 +264,18 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - //Should send out hedge request but original should be returned - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - IReadOnlyCollection hedgeContextList; - hedgeContextList = hedgeContext as IReadOnlyCollection; - if (isPreferredLocationsEmpty) { - Assert.AreEqual(3, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(region1)); - Assert.IsTrue(hedgeContextList.Contains(region2)); - Assert.IsTrue(hedgeContextList.Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); } else { - Assert.AreEqual(2, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(region1)); - Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } - }; + } + ; } [TestMethod] @@ -343,9 +335,8 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } } @@ -565,9 +556,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); break; @@ -598,9 +588,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } break; @@ -629,9 +618,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } break; @@ -659,9 +647,8 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsTrue(rule.GetHitCount() > 0); traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); break; @@ -778,9 +765,8 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); break; @@ -805,9 +791,8 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); } break; @@ -826,9 +811,8 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = feedResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); } break; @@ -848,9 +832,8 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito traceDiagnostic = readManyResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); break; @@ -960,9 +943,8 @@ public async Task AvailabilityStrategyMultiMasterWriteBeforeTest() CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } } @@ -1035,9 +1017,8 @@ public async Task AvailabilityStrategyMultiMasterWriteAfterTest() CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } finally { @@ -1139,9 +1120,8 @@ await this.container.DeleteItemAsync( CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); } } @@ -1241,9 +1221,8 @@ await this.container.DeleteItemAsync( CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); } finally { @@ -1393,9 +1372,8 @@ public async Task HedgingCancellationTokenHandling() CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); } catch (CosmosException ex) { @@ -1419,9 +1397,8 @@ private static async Task HandleChangesAsync( CosmosTraceDiagnostics traceDiagnostic = context.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region2)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); await Task.Delay(1); } @@ -1437,9 +1414,8 @@ private static async Task HandleChangesStepAsync( CosmosTraceDiagnostics traceDiagnostic = context.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); - Assert.IsNotNull(hedgeContext); - Assert.IsTrue(((IReadOnlyCollection)hedgeContext).Contains(region3)); + Assert.IsTrue(traceDiagnostic.ToString() + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\",\"{region3}\"]")); await Task.Delay(1); } } From 40a8694af815326b32f07c429df2f0b9b140e644 Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 16:27:46 -0400 Subject: [PATCH 6/9] use existing clone method --- .../CrossRegionHedgingAvailabilityStrategy.cs | 216 +++++++++--------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs index 5456443206..2051d7b577 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs @@ -137,125 +137,124 @@ internal override async Task ExecuteAvailabilityStrategyAsync( ? null : await StreamExtension.AsClonableStreamAsync(request.Content))) { - using (RequestMessage nonModifiedRequestClone = request.Clone(trace, clonedBody)) - { - IReadOnlyCollection hedgeRegions = client.DocumentClient.GlobalEndpointManager - .GetApplicableRegions( - request.RequestOptions?.ExcludeRegions, - OperationTypeExtensions.IsReadOperation(request.OperationType)); + IReadOnlyCollection hedgeRegions = client.DocumentClient.GlobalEndpointManager + .GetApplicableRegions( + request.RequestOptions?.ExcludeRegions, + OperationTypeExtensions.IsReadOperation(request.OperationType)); - List requestTasks = new List(hedgeRegions.Count + 1); + List requestTasks = new List(hedgeRegions.Count + 1); - Task primaryRequest = null; - HedgingResponse hedgeResponse = null; + Task primaryRequest = null; + HedgingResponse hedgeResponse = null; - //Send out hedged requests - for (int requestNumber = 0; requestNumber < hedgeRegions.Count; requestNumber++) - { - TimeSpan awaitTime = requestNumber == 0 ? this.Threshold : this.ThresholdStep; + //Send out hedged requests + for (int requestNumber = 0; requestNumber < hedgeRegions.Count; requestNumber++) + { + TimeSpan awaitTime = requestNumber == 0 ? this.Threshold : this.ThresholdStep; - using (CancellationTokenSource timerTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + using (CancellationTokenSource timerTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + { + CancellationToken timerToken = timerTokenSource.Token; + using (Task hedgeTimer = Task.Delay(awaitTime, timerToken)) { - CancellationToken timerToken = timerTokenSource.Token; - using (Task hedgeTimer = Task.Delay(awaitTime, timerToken)) + if (requestNumber == 0) + { + primaryRequest = this.CloneAndSendAsync( + sender: sender, + request: request, + clonedBody: clonedBody, + hedgeRegions: hedgeRegions, + requestNumber: requestNumber, + trace: trace, + cancellationToken: cancellationToken, + cancellationTokenSource: cancellationTokenSource); + + requestTasks.Add(primaryRequest); + } + else { - if (requestNumber == 0) - { - primaryRequest = this.RequestSenderAndResultCheckAsync( - sender, - request, - hedgeRegions.ElementAt(requestNumber), - cancellationToken, - cancellationTokenSource, - trace); - - requestTasks.Add(primaryRequest); - } - else - { - Task requestTask = this.CloneAndSendAsync( - sender: sender, - request: nonModifiedRequestClone, - clonedBody: clonedBody, - hedgeRegions: hedgeRegions, - requestNumber: requestNumber, - trace: trace, - cancellationToken: cancellationToken, - cancellationTokenSource: cancellationTokenSource); - - requestTasks.Add(requestTask); - } - - requestTasks.Add(hedgeTimer); - - Task completedTask = await Task.WhenAny(requestTasks); - requestTasks.Remove(completedTask); - - if (completedTask == hedgeTimer) - { - continue; - } - - timerTokenSource.Cancel(); - requestTasks.Remove(hedgeTimer); - - if (completedTask.IsFaulted) - { - AggregateException innerExceptions = completedTask.Exception.Flatten(); - } - - hedgeResponse = await (Task)completedTask; - if (hedgeResponse.IsNonTransient) - { - cancellationTokenSource.Cancel(); - - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeConfig, - this.HedgeConfigText); - //Take is not inclusive, so we need to add 1 to the request number which starts at 0 - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeContext, - hedgeRegions.Take(requestNumber + 1)); - return hedgeResponse.ResponseMessage; - } + Task requestTask = this.CloneAndSendAsync( + sender: sender, + request: request, + clonedBody: clonedBody, + hedgeRegions: hedgeRegions, + requestNumber: requestNumber, + trace: trace, + cancellationToken: cancellationToken, + cancellationTokenSource: cancellationTokenSource); + + requestTasks.Add(requestTask); + } + + requestTasks.Add(hedgeTimer); + + Task completedTask = await Task.WhenAny(requestTasks); + requestTasks.Remove(completedTask); + + if (completedTask == hedgeTimer) + { + continue; + } + + timerTokenSource.Cancel(); + requestTasks.Remove(hedgeTimer); + + if (completedTask.IsFaulted) + { + AggregateException innerExceptions = completedTask.Exception.Flatten(); + } + + hedgeResponse = await (Task)completedTask; + if (hedgeResponse.IsNonTransient) + { + cancellationTokenSource.Cancel(); + + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeConfig, + this.HedgeConfigText); + //Take is not inclusive, so we need to add 1 to the request number which starts at 0 + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeContext, + hedgeRegions.Take(requestNumber + 1)); + return hedgeResponse.ResponseMessage; } } } + } - //Wait for a good response from the hedged requests/primary request - Exception lastException = null; - while (requestTasks.Any()) + //Wait for a good response from the hedged requests/primary request + Exception lastException = null; + while (requestTasks.Any()) + { + Task completedTask = await Task.WhenAny(requestTasks); + requestTasks.Remove(completedTask); + if (completedTask.IsFaulted) { - Task completedTask = await Task.WhenAny(requestTasks); - requestTasks.Remove(completedTask); - if (completedTask.IsFaulted) - { - AggregateException innerExceptions = completedTask.Exception.Flatten(); - lastException = innerExceptions.InnerExceptions.FirstOrDefault(); - } - - hedgeResponse = await (Task)completedTask; - if (hedgeResponse.IsNonTransient || requestTasks.Count == 0) - { - cancellationTokenSource.Cancel(); - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeConfig, - this.HedgeConfigText); - ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( - HedgeContext, - hedgeRegions); - return hedgeResponse.ResponseMessage; - } + AggregateException innerExceptions = completedTask.Exception.Flatten(); + lastException = innerExceptions.InnerExceptions.FirstOrDefault(); } - if (lastException != null) + hedgeResponse = await (Task)completedTask; + if (hedgeResponse.IsNonTransient || requestTasks.Count == 0) { - throw lastException; + cancellationTokenSource.Cancel(); + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeConfig, + this.HedgeConfigText); + ((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum( + HedgeContext, + hedgeRegions); + return hedgeResponse.ResponseMessage; } + } - Debug.Assert(hedgeResponse != null); - return hedgeResponse.ResponseMessage; + if (lastException != null) + { + throw lastException; } + + Debug.Assert(hedgeResponse != null); + return hedgeResponse.ResponseMessage; } } } @@ -278,15 +277,17 @@ private async Task CloneAndSendAsync( { clonedRequest.RequestOptions ??= new RequestOptions(); - List excludeRegions = new List(hedgeRegions); - string region = excludeRegions[requestNumber]; - excludeRegions.RemoveAt(requestNumber); - clonedRequest.RequestOptions.ExcludeRegions = excludeRegions; + //we do not want to exclude any regions for the primary request + if (requestNumber > 0) + { + List excludeRegions = new List(hedgeRegions); + excludeRegions.RemoveAt(requestNumber); + clonedRequest.RequestOptions.ExcludeRegions = excludeRegions; + } return await this.RequestSenderAndResultCheckAsync( sender, clonedRequest, - region, cancellationToken, cancellationTokenSource, trace); @@ -296,7 +297,6 @@ private async Task CloneAndSendAsync( private async Task RequestSenderAndResultCheckAsync( Func> sender, RequestMessage request, - string hedgedRegion, CancellationToken cancellationToken, CancellationTokenSource cancellationTokenSource, ITrace trace) From 90fadb39f98e10920f1235c8c6d9dc21017cc69d Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 16:32:56 -0400 Subject: [PATCH 7/9] removed if-else --- .../CrossRegionHedgingAvailabilityStrategy.cs | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs index 2051d7b577..6a1d7c0910 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs @@ -144,7 +144,6 @@ internal override async Task ExecuteAvailabilityStrategyAsync( List requestTasks = new List(hedgeRegions.Count + 1); - Task primaryRequest = null; HedgingResponse hedgeResponse = null; //Send out hedged requests @@ -157,23 +156,7 @@ internal override async Task ExecuteAvailabilityStrategyAsync( CancellationToken timerToken = timerTokenSource.Token; using (Task hedgeTimer = Task.Delay(awaitTime, timerToken)) { - if (requestNumber == 0) - { - primaryRequest = this.CloneAndSendAsync( - sender: sender, - request: request, - clonedBody: clonedBody, - hedgeRegions: hedgeRegions, - requestNumber: requestNumber, - trace: trace, - cancellationToken: cancellationToken, - cancellationTokenSource: cancellationTokenSource); - - requestTasks.Add(primaryRequest); - } - else - { - Task requestTask = this.CloneAndSendAsync( + Task requestTask = this.CloneAndSendAsync( sender: sender, request: request, clonedBody: clonedBody, @@ -183,9 +166,7 @@ internal override async Task ExecuteAvailabilityStrategyAsync( cancellationToken: cancellationToken, cancellationTokenSource: cancellationTokenSource); - requestTasks.Add(requestTask); - } - + requestTasks.Add(requestTask); requestTasks.Add(hedgeTimer); Task completedTask = await Task.WhenAny(requestTasks); From cd3868369e93f7af8b8d88dd7c1e47abd5a6d54c Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 17:25:11 -0400 Subject: [PATCH 8/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 8a03d3c50d..324fb79639 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -534,7 +534,6 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); CosmosTraceDiagnostics traceDiagnostic; - object hedgeContext; switch (operation) { @@ -751,7 +750,6 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito ItemResponse _ = await container.ReadItemAsync("testId", new PartitionKey("pk")); CosmosTraceDiagnostics traceDiagnostic; - object hedgeContext; switch (operation) { From efd7d523597f2ef1bffe8e4788ea861694a4255a Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Fri, 9 May 2025 17:37:17 -0400 Subject: [PATCH 9/9] Update CosmosAvailabilityStrategyTests.cs --- .../CosmosAvailabilityStrategyTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 324fb79639..b3b6b7aaaa 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -336,7 +336,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); Assert.IsTrue(traceDiagnostic.ToString() - .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"]")); + .Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\"")); } }