Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
5044b6f
Routing: Adds Gateway Account Property Flag for Dynamic Hedging Control
NaluTripician May 5, 2026
725580f
Merge branch 'main' into users/ntripician/ppaf-gateway-hedging-override
NaluTripician May 6, 2026
816e334
Routing: Refactors PPAF Gateway hedging override per code-review feed…
NaluTripician May 8, 2026
024db65
Merge branch 'main' into users/ntripician/ppaf-gateway-hedging-override
NaluTripician May 12, 2026
d9041ca
Routing: Refactors PPAF Gateway hedging override per second-round rev…
NaluTripician May 14, 2026
95938a5
Merge remote-tracking branch 'origin/main' into users/ntripician/ppaf…
NaluTripician May 14, 2026
6433c5a
Routing: Refactors PPAF gateway hedging override per third-round review
NaluTripician May 15, 2026
2213b56
Routing: Fixes unresolved XML doc cref for RequestInvokerHandler.Avai…
NaluTripician May 18, 2026
617e9c3
Merge branch 'main' into users/ntripician/ppaf-gateway-hedging-override
NaluTripician May 18, 2026
403d509
Routing: Refactors disableCrossRegionalHedging to volatile for lock-f…
NaluTripician May 19, 2026
3db536a
Merge remote-tracking branch 'origin/main' into users/ntripician/ppaf…
Copilot May 19, 2026
efea7ac
Routing: Fixes hedging override concurrency, recovery, and observabil…
NaluTripician Jun 9, 2026
f230ee3
Merge branch 'main' into users/ntripician/ppaf-gateway-hedging-override
kundadebdatta Jun 16, 2026
a4afa43
Routing: Adds trace for hedging race re-check skip path
Copilot Jun 18, 2026
ccc57a2
Merge branch 'main' into users/ntripician/ppaf-gateway-hedging-override
kundadebdatta Jun 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
314 changes: 292 additions & 22 deletions Microsoft.Azure.Cosmos/src/DocumentClient.cs

Large diffs are not rendered by default.

14 changes: 13 additions & 1 deletion Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,24 @@ public override async Task<ResponseMessage> SendAsync(

/// <summary>
/// This method determines if there is an availability strategy that the request can use.
/// Note that the request level availability strategy options override the client level options.
/// Note that the request level availability strategy options override the client level options,
/// but the Gateway-driven operator override (<see cref="DocumentClient.IsHedgingDisabledByGateway"/>)
/// takes absolute precedence over both — when the Gateway flag
/// <c>disableCrossRegionalHedging</c> is <c>true</c>, hedging is OFF for every request on this
/// client regardless of where the strategy was configured.
/// </summary>
/// <param name="request"></param>
/// <returns>whether the request should be a parallel hedging request.</returns>
public AvailabilityStrategyInternal AvailabilityStrategy(RequestMessage request)
{
// Gateway-driven operator override has absolute precedence over any request-level or
// client-level AvailabilityStrategy. See spec.md → "Gateway flag disables all hedging
// when true" and tasks.md item 4.1.
if (this.client.DocumentClient.IsHedgingDisabledByGateway)
Comment thread
NaluTripician marked this conversation as resolved.
{
return null;
}

AvailabilityStrategy strategy = request.RequestOptions?.AvailabilityStrategy
?? this.client.DocumentClient.ConnectionPolicy.AvailabilityStrategy;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,23 @@ internal long ProvisionedDocumentStorageInMB
[JsonProperty(PropertyName = Constants.Properties.EnablePerPartitionFailoverBehavior)]
internal bool? EnablePartitionLevelFailover { get; set; }

/// <summary>
/// Gets the gateway-controlled override that disables cross-regional hedging for this account.
/// </summary>
/// <remarks>
/// When this flag is <see langword="true"/>, the SDK disables all hedging (both SDK-default PPAF
/// hedging and any explicit customer-configured <see cref="Cosmos.AvailabilityStrategy"/>) regardless
/// of any other configuration. When the flag is <see langword="false"/> or <see langword="null"/>
/// (absent from the Gateway response), existing hedging behavior is preserved. The flag is intended
/// as an operational escape hatch and is not exposed through any public SDK API surface.
/// </remarks>
// TODO: The JSON property name is hard-coded here because the corresponding constant has not yet been
// published in the Microsoft.Azure.Cosmos.Direct package referenced by this SDK. Once Direct is updated
// to expose this name on Constants.Properties, refactor this attribute to read from
// Constants.Properties.<NewConstant> for consistency with the other AccountProperties JSON bindings.
[JsonProperty(PropertyName = "disableCrossRegionalHedging")]
Comment thread
NaluTripician marked this conversation as resolved.
internal bool? DisableCrossRegionalHedging { get; set; }
Comment thread
NaluTripician marked this conversation as resolved.

private IDictionary<string, object> QueryStringToDictConverter()
{
if (!string.IsNullOrEmpty(this.QueryEngineConfigurationString))
Expand Down
75 changes: 58 additions & 17 deletions Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,23 @@ internal class GlobalEndpointManager : IGlobalEndpointManager
private readonly object isAccountRefreshInProgressLock = new object();
private bool isAccountRefreshInProgress = false;
private bool isBackgroundAccountRefreshActive = false;
private DateTime LastBackgroundRefreshUtc = DateTime.MinValue;

/// <summary>
/// Event that is raised when PPAF (Per Partition Automatic Failover) enablement status changes
/// </summary>
internal event Action<bool>? OnEnablePartitionLevelFailoverConfigChanged;
private DateTime LastBackgroundRefreshUtc = DateTime.MinValue;

// Last observed value of the account-level disableCrossRegionalHedging flag.
// Tracked separately so the change event fires when only this flag toggles.
private bool lastKnownDisableCrossRegionalHedging = false;
Comment thread
NaluTripician marked this conversation as resolved.

/// <summary>
/// Event that is raised when PPAF (Per Partition Automatic Failover) enablement status changes
/// or when the gateway-controlled disableCrossRegionalHedging flag toggles.
/// </summary>
/// <remarks>
/// First argument is the latest <c>EnablePartitionLevelFailover</c> value observed from the
/// Gateway (falls back to the existing connection-policy value when the property is absent).
/// Second argument is the latest <c>disableCrossRegionalHedging</c> value (false when absent
/// from the Gateway response).
/// </remarks>
internal event Action<bool, bool>? OnEnablePartitionLevelFailoverConfigChanged;

public GlobalEndpointManager(
IDocumentClientInternal owner,
Expand Down Expand Up @@ -608,10 +619,14 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
return;
}

if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)
{
this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
}
if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)
{
this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
}

// Capture initial disableCrossRegionalHedging baseline so the change-event only fires on
// subsequent transitions, not on the first observation.
this.lastKnownDisableCrossRegionalHedging = databaseAccount.DisableCrossRegionalHedging ?? false;
Comment thread
NaluTripician marked this conversation as resolved.

GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(databaseAccount);

Expand Down Expand Up @@ -768,13 +783,39 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
try
{
this.LastBackgroundRefreshUtc = DateTime.UtcNow;
AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);

if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride
&& accountProperties.EnablePartitionLevelFailover.HasValue
&& (this.connectionPolicy.EnablePartitionLevelFailover != accountProperties.EnablePartitionLevelFailover.Value))
{
this.OnEnablePartitionLevelFailoverConfigChanged?.Invoke(accountProperties.EnablePartitionLevelFailover.Value);
AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);

bool ignorePpafChanges = this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride;

bool ppafEnablementChanged = !ignorePpafChanges
&& accountProperties.EnablePartitionLevelFailover.HasValue
&& (this.connectionPolicy.EnablePartitionLevelFailover != accountProperties.EnablePartitionLevelFailover.Value);

// Hedging change-detection mirrors the PPAF .HasValue guard above:
// a missing property in the response is "no signal", NOT an implicit false.
// This prevents a transient gateway response that drops the property
// (e.g., partial regional failover, stale gateway version) from being
// interpreted as a true -> false transition that re-enables hedging
// during the very window the operator most wants it disabled.
//
// Runbook contract: on-call disables via an explicit "false" property
// value, not by removing the property override.
bool disableHedgingFlagChanged = !ignorePpafChanges
Comment thread
NaluTripician marked this conversation as resolved.
&& accountProperties.DisableCrossRegionalHedging.HasValue
&& (accountProperties.DisableCrossRegionalHedging.Value != this.lastKnownDisableCrossRegionalHedging);

if (ppafEnablementChanged || disableHedgingFlagChanged)
{
bool latestPpafEnabled = accountProperties.EnablePartitionLevelFailover
Comment thread
NaluTripician marked this conversation as resolved.
?? this.connectionPolicy.EnablePartitionLevelFailover;

// Only advance lastKnown when the gateway emitted an explicit value; otherwise
// preserve the cached value so a later property-restored response diffs against
// the previously-honored state (rather than against an implicit false baseline).
bool latestDisableHedging = accountProperties.DisableCrossRegionalHedging
?? this.lastKnownDisableCrossRegionalHedging;
this.lastKnownDisableCrossRegionalHedging = latestDisableHedging;
Comment thread
NaluTripician marked this conversation as resolved.
this.OnEnablePartitionLevelFailoverConfigChanged?.Invoke(latestPpafEnabled, latestDisableHedging);
}

GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(accountProperties);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void ValidatePropertySerialization()
string id = "testId";
this.TestProperty<AccountProperties>(
id,
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""enableNRegionSynchronousCommit"":false}}");
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""disableCrossRegionalHedging"":null,""enableNRegionSynchronousCommit"":false}}");

this.TestProperty<DatabaseProperties>(
id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public void ValidateCustomSerializerNotUsedForInternalTypes()
this.TestProperty<AccountProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""enableNRegionSynchronousCommit"":false}}");
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""disableCrossRegionalHedging"":null,""enableNRegionSynchronousCommit"":false}}");

this.TestProperty<DatabaseProperties>(
serializerCore,
Expand Down
Loading
Loading