Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
314 changes: 292 additions & 22 deletions Microsoft.Azure.Cosmos/src/DocumentClient.cs

Large diffs are not rendered by default.

14 changes: 13 additions & 1 deletion Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,24 @@ public override async Task<ResponseMessage> SendAsync(

/// <summary>
/// This method determines if there is an availability strategy that the request can use.
/// Note that the request level availability strategy options override the client level options.
/// Note that the request level availability strategy options override the client level options,
/// but the Gateway-driven operator override (<see cref="DocumentClient.IsHedgingDisabledByGateway"/>)
/// takes absolute precedence over both — when the Gateway flag
/// <c>disableCrossRegionalHedging</c> is <c>true</c>, hedging is OFF for every request on this
/// client regardless of where the strategy was configured.
/// </summary>
/// <param name="request"></param>
/// <returns>whether the request should be a parallel hedging request.</returns>
public AvailabilityStrategyInternal AvailabilityStrategy(RequestMessage request)
{
// Gateway-driven operator override has absolute precedence over any request-level or
// client-level AvailabilityStrategy. See spec.md → "Gateway flag disables all hedging
// when true" and tasks.md item 4.1.
if (this.client.DocumentClient.IsHedgingDisabledByGateway)
{
return null;
}

AvailabilityStrategy strategy = request.RequestOptions?.AvailabilityStrategy
?? this.client.DocumentClient.ConnectionPolicy.AvailabilityStrategy;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,23 @@ internal long ProvisionedDocumentStorageInMB
[JsonProperty(PropertyName = Constants.Properties.EnablePerPartitionFailoverBehavior)]
internal bool? EnablePartitionLevelFailover { get; set; }

/// <summary>
/// Gets the gateway-controlled override that disables cross-regional hedging for this account.
/// </summary>
/// <remarks>
/// When this flag is <see langword="true"/>, the SDK disables all hedging (both SDK-default PPAF
/// hedging and any explicit customer-configured <see cref="Cosmos.AvailabilityStrategy"/>) regardless
/// of any other configuration. When the flag is <see langword="false"/> or <see langword="null"/>
/// (absent from the Gateway response), existing hedging behavior is preserved. The flag is intended
/// as an operational escape hatch and is not exposed through any public SDK API surface.
/// </remarks>
// TODO: The JSON property name is hard-coded here because the corresponding constant has not yet been
// published in the Microsoft.Azure.Cosmos.Direct package referenced by this SDK. Once Direct is updated
// to expose this name on Constants.Properties, refactor this attribute to read from
// Constants.Properties.<NewConstant> for consistency with the other AccountProperties JSON bindings.
[JsonProperty(PropertyName = "disableCrossRegionalHedging")]
Comment thread
NaluTripician marked this conversation as resolved.
internal bool? DisableCrossRegionalHedging { get; set; }

private IDictionary<string, object> QueryStringToDictConverter()
{
if (!string.IsNullOrEmpty(this.QueryEngineConfigurationString))
Expand Down
75 changes: 58 additions & 17 deletions Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,23 @@ internal class GlobalEndpointManager : IGlobalEndpointManager
private readonly object isAccountRefreshInProgressLock = new object();
private bool isAccountRefreshInProgress = false;
private bool isBackgroundAccountRefreshActive = false;
private DateTime LastBackgroundRefreshUtc = DateTime.MinValue;

/// <summary>
/// Event that is raised when PPAF (Per Partition Automatic Failover) enablement status changes
/// </summary>
internal event Action<bool>? OnEnablePartitionLevelFailoverConfigChanged;
private DateTime LastBackgroundRefreshUtc = DateTime.MinValue;

// Last observed value of the account-level disableCrossRegionalHedging flag.
// Tracked separately so the change event fires when only this flag toggles.
private bool lastKnownDisableCrossRegionalHedging = false;
Comment thread
NaluTripician marked this conversation as resolved.

/// <summary>
/// Event that is raised when PPAF (Per Partition Automatic Failover) enablement status changes
/// or when the gateway-controlled disableCrossRegionalHedging flag toggles.
/// </summary>
/// <remarks>
/// First argument is the latest <c>EnablePartitionLevelFailover</c> value observed from the
/// Gateway (falls back to the existing connection-policy value when the property is absent).
/// Second argument is the latest <c>disableCrossRegionalHedging</c> value (false when absent
/// from the Gateway response).
/// </remarks>
internal event Action<bool, bool>? OnEnablePartitionLevelFailoverConfigChanged;

public GlobalEndpointManager(
IDocumentClientInternal owner,
Expand Down Expand Up @@ -608,10 +619,14 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
return;
}

if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)
{
this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
}
if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)
{
this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
}

// Capture initial disableCrossRegionalHedging baseline so the change-event only fires on
// subsequent transitions, not on the first observation.
this.lastKnownDisableCrossRegionalHedging = databaseAccount.DisableCrossRegionalHedging ?? false;
Comment thread
NaluTripician marked this conversation as resolved.

GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(databaseAccount);

Expand Down Expand Up @@ -768,13 +783,39 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
try
{
this.LastBackgroundRefreshUtc = DateTime.UtcNow;
AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);

if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride
&& accountProperties.EnablePartitionLevelFailover.HasValue
&& (this.connectionPolicy.EnablePartitionLevelFailover != accountProperties.EnablePartitionLevelFailover.Value))
{
this.OnEnablePartitionLevelFailoverConfigChanged?.Invoke(accountProperties.EnablePartitionLevelFailover.Value);
AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);

bool ignorePpafChanges = this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride;

bool ppafEnablementChanged = !ignorePpafChanges
&& accountProperties.EnablePartitionLevelFailover.HasValue
&& (this.connectionPolicy.EnablePartitionLevelFailover != accountProperties.EnablePartitionLevelFailover.Value);

// Hedging change-detection mirrors the PPAF .HasValue guard above:
// a missing property in the response is "no signal", NOT an implicit false.
// This prevents a transient gateway response that drops the property
// (e.g., partial regional failover, stale gateway version) from being
// interpreted as a true -> false transition that re-enables hedging
// during the very window the operator most wants it disabled.
//
// Runbook contract: on-call disables via an explicit "false" property
// value, not by removing the property override.
bool disableHedgingFlagChanged = !ignorePpafChanges
Comment thread
NaluTripician marked this conversation as resolved.
&& accountProperties.DisableCrossRegionalHedging.HasValue
&& (accountProperties.DisableCrossRegionalHedging.Value != this.lastKnownDisableCrossRegionalHedging);

if (ppafEnablementChanged || disableHedgingFlagChanged)
{
bool latestPpafEnabled = accountProperties.EnablePartitionLevelFailover
?? this.connectionPolicy.EnablePartitionLevelFailover;

// Only advance lastKnown when the gateway emitted an explicit value; otherwise
// preserve the cached value so a later property-restored response diffs against
// the previously-honored state (rather than against an implicit false baseline).
bool latestDisableHedging = accountProperties.DisableCrossRegionalHedging
?? this.lastKnownDisableCrossRegionalHedging;
this.lastKnownDisableCrossRegionalHedging = latestDisableHedging;
this.OnEnablePartitionLevelFailoverConfigChanged?.Invoke(latestPpafEnabled, latestDisableHedging);
}

GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(accountProperties);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void ValidatePropertySerialization()
string id = "testId";
this.TestProperty<AccountProperties>(
id,
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""enableNRegionSynchronousCommit"":false}}");
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""disableCrossRegionalHedging"":null,""enableNRegionSynchronousCommit"":false}}");

this.TestProperty<DatabaseProperties>(
id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public void ValidateCustomSerializerNotUsedForInternalTypes()
this.TestProperty<AccountProperties>(
serializerCore,
id,
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""enableNRegionSynchronousCommit"":false}}");
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""disableCrossRegionalHedging"":null,""enableNRegionSynchronousCommit"":false}}");

this.TestProperty<DatabaseProperties>(
serializerCore,
Expand Down
Loading
Loading