Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 94 additions & 4 deletions src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ internal sealed partial class DistributedGrainDirectory : SystemTarget, IGrainDi
private readonly IServiceProvider _serviceProvider;
private readonly ImmutableArray<GrainDirectoryPartition> _partitions;
private readonly CancellationTokenSource _stoppedCts = new();
private readonly ClusterMemberCancellationTokens _clusterMemberCancellationTokens;
private readonly DirectoryInstruments _directoryInstruments;

internal CancellationToken OnStoppedToken => _stoppedCts.Token;
Expand Down Expand Up @@ -96,6 +97,7 @@ public DistributedGrainDirectory(
_membershipService = membershipService;
_logger = logger;
_directoryInstruments = directoryInstruments;
_clusterMemberCancellationTokens = new(_stoppedCts.Token);
var partitionsPerSilo = membershipService.PartitionsPerSilo;
var partitions = ImmutableArray.CreateBuilder<GrainDirectoryPartition>(partitionsPerSilo);
for (var i = 0; i < partitionsPerSilo; i++)
Expand Down Expand Up @@ -224,18 +226,32 @@ private async Task<TResult> InvokeAsync<TState, TResult>(
internal static bool CanInvokeClusterMember(ClusterMembershipSnapshot snapshot, SiloAddress siloAddress)
=> snapshot.GetSiloStatus(siloAddress) is SiloStatus.Active or SiloStatus.Joining or SiloStatus.ShuttingDown;

public async ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(MembershipVersion membershipVersion, RingRange range, SiloAddress siloAddress, int partitionIndex)
internal CancellationToken GetClusterMemberCancellationToken(SiloAddress siloAddress) =>
_clusterMemberCancellationTokens.GetToken(siloAddress);

public async ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
SiloAddress siloAddress,
int partitionIndex,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
foreach (var partition in _partitions)
{
partition.OnRecoveringPartition(membershipVersion, range, siloAddress, partitionIndex).Ignore();
}

return await GetRegisteredActivations(membershipVersion, range, false);
return await GetRegisteredActivations(membershipVersion, range, false, cancellationToken);
}

public ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(MembershipVersion membershipVersion, RingRange range, bool isValidation)
public ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
bool isValidation,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (!isValidation)
{
LogDebugCollectingRegisteredActivations(_logger, range, membershipVersion);
Expand All @@ -254,6 +270,7 @@ public ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(Members

foreach (var (grainId, activation) in _localActivations)
{
cancellationToken.ThrowIfCancellationRequested();
var directory = GetGrainDirectory(activation, _grainDirectoryResolver!);
if (directory == this)
{
Expand Down Expand Up @@ -345,7 +362,6 @@ async Task OnShuttingDown(CancellationToken token)
{
tasks.Add(partition.OnShuttingDown(token));
}

await Task.WhenAll(tasks).SuppressThrowing();
}
}
Expand All @@ -364,6 +380,7 @@ private async Task ProcessMembershipUpdates()
await foreach (var update in _membershipService.ViewUpdates.WithCancellation(_stoppedCts.Token))
{
tasks.RemoveAll(t => t.IsCompleted);
_clusterMemberCancellationTokens.Update(update.ClusterMembershipSnapshot);
var changes = update.ClusterMembershipSnapshot.CreateUpdate(previousUpdate);

foreach (var change in changes.Changes)
Expand Down Expand Up @@ -405,6 +422,7 @@ private async Task ProcessMembershipUpdates()
}

await Task.WhenAll(tasks).SuppressThrowing();
_clusterMemberCancellationTokens.Dispose();
}

private async Task ObserveMembershipUpdateTask(Task task)
Expand Down Expand Up @@ -488,3 +506,75 @@ internal interface ITestHooks
)]
private static partial void LogTraceInvokingOperation(ILogger logger, string operation, SiloAddress owner, GrainId grainId);
}

internal sealed class ClusterMemberCancellationTokens(CancellationToken shutdownToken) : IDisposable
{
private static readonly CancellationToken CanceledToken = new(canceled: true);
private Dictionary<SiloAddress, Entry> _entries = [];

internal int Count => Volatile.Read(ref _entries).Count;

public CancellationToken GetToken(SiloAddress siloAddress)
{
var entries = Volatile.Read(ref _entries);
return entries.TryGetValue(siloAddress, out var entry) ? entry.Token : CanceledToken;
}

public void Update(ClusterMembershipSnapshot snapshot)
{
var previous = Volatile.Read(ref _entries);
var updated = new Dictionary<SiloAddress, Entry>(snapshot.Members.Count);
foreach (var (siloAddress, _) in snapshot.Members)
{
if (!DistributedGrainDirectory.CanInvokeClusterMember(snapshot, siloAddress))
{
continue;
}

updated.Add(
siloAddress,
previous.TryGetValue(siloAddress, out var entry) ? entry : new Entry(shutdownToken));
}

Volatile.Write(ref _entries, updated);
DisposeRemoved(previous, updated);
}

public void Dispose()
{
var previous = Interlocked.Exchange(ref _entries, []);
DisposeRemoved(previous, []);
}

private static void DisposeRemoved(
Dictionary<SiloAddress, Entry> previous,
Dictionary<SiloAddress, Entry> updated)
{
foreach (var (siloAddress, entry) in previous)
{
if (!updated.ContainsKey(siloAddress))
{
entry.Dispose();
}
}
}

private sealed class Entry : IDisposable
{
private readonly CancellationTokenSource _source;

public Entry(CancellationToken shutdownToken)
{
_source = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken);
Token = _source.Token;
}

public CancellationToken Token { get; }

public void Dispose()
{
_source.Cancel();
_source.Dispose();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ private async Task EnsureDirectoryInitializedAsync(CancellationToken cancellatio
TaskScheduler = TaskScheduler.Current,
};

private CancellationTokenSource CreateTimeoutCts() => new(OperationTimeout);
private CancellationTokenSource CreateTimeoutCts() => CreateTimeoutCts(CancellationToken.None);

private CancellationTokenSource CreateTimeoutCts(CancellationToken cancellationToken)
{
Expand Down
40 changes: 27 additions & 13 deletions src/Orleans.Runtime/GrainDirectory/GrainDirectoryPartition.cs
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,13 @@ public async ValueTask<DirectoryMembershipSnapshot> RefreshViewAsync(MembershipV
return null;
}

ValueTask<bool> IGrainDirectoryPartition.AcknowledgeSnapshotTransferAsync(SiloAddress silo, int partitionIndex, MembershipVersion rangeVersion)
ValueTask<bool> IGrainDirectoryPartition.AcknowledgeSnapshotTransferAsync(
SiloAddress silo,
int partitionIndex,
MembershipVersion rangeVersion,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
RemoveSnapshotTransferPartner(
(silo, partitionIndex, rangeVersion),
snapshotFilter: (state, snapshot) => snapshot.DirectoryMembershipVersion == state.rangeVersion,
Expand Down Expand Up @@ -583,7 +588,11 @@ private async Task<bool> TransferSnapshotAsync(
// The acknowledgement step lets the previous owner know that the snapshot has been received so that it can proceed.
InvokeOnClusterMember(
previousOwner,
async () => await partition.AcknowledgeSnapshotTransferAsync(_id, _partitionIndex, previousVersion),
async cancellationToken => await partition.AcknowledgeSnapshotTransferAsync(
_id,
_partitionIndex,
previousVersion,
cancellationToken),
false,
nameof(IGrainDirectoryPartition.AcknowledgeSnapshotTransferAsync)).Ignore();

Expand Down Expand Up @@ -666,17 +675,17 @@ async Task<List<GrainAddress>> GetRegisteredActivationsFromClusterMember(Members
var client = _grainFactory.GetSystemTarget<IGrainDirectoryClient>(Constants.GrainDirectoryType, siloAddress);
var result = await InvokeOnClusterMember(
siloAddress,
async () =>
async cancellationToken =>
{
var innerSw = ValueStopwatch.StartNew();
Immutable<List<GrainAddress>> result = default;
if (isValidation)
{
result = await client.GetRegisteredActivations(version, range, isValidation: true);
result = await client.GetRegisteredActivations(version, range, isValidation: true, cancellationToken);
}
else
{
result = await client.RecoverRegisteredActivations(version, range, _id, _partitionIndex);
result = await client.RecoverRegisteredActivations(version, range, _id, _partitionIndex, cancellationToken);
}

return result;
Expand Down Expand Up @@ -756,20 +765,24 @@ static ulong GetRangeCoverageSize(RingRange range)
}
}

private async Task<T> InvokeOnClusterMember<T>(SiloAddress siloAddress, Func<Task<T>> func, T defaultValue, string operationName)
private async Task<T> InvokeOnClusterMember<T>(
SiloAddress siloAddress,
Func<CancellationToken, Task<T>> func,
T defaultValue,
string operationName)
{
GrainRuntime.CheckRuntimeContext(this);
var memberCancellationToken = _owner.GetClusterMemberCancellationToken(siloAddress);
var attempt = 0;
while (!ShutdownToken.IsCancellationRequested)
while (!memberCancellationToken.IsCancellationRequested)
{
if (!DistributedGrainDirectory.CanInvokeClusterMember(_owner.LatestClusterMembershipSnapshot, siloAddress))
try
{
break;
return await func(memberCancellationToken);
}

try
catch (OperationCanceledException) when (memberCancellationToken.IsCancellationRequested)
{
return await func();
break;
}
catch (Exception ex)
{
Expand All @@ -779,7 +792,8 @@ private async Task<T> InvokeOnClusterMember<T>(SiloAddress siloAddress, Func<Tas
}

await _owner.RefreshViewAsync(default, ShutdownToken);
if (!DistributedGrainDirectory.CanInvokeClusterMember(_owner.LatestClusterMembershipSnapshot, siloAddress))
if (memberCancellationToken.IsCancellationRequested
|| !DistributedGrainDirectory.CanInvokeClusterMember(_owner.LatestClusterMembershipSnapshot, siloAddress))
{
break;
}
Comment thread
ReubenBond marked this conversation as resolved.
Resolved
Expand Down
20 changes: 17 additions & 3 deletions src/Orleans.Runtime/GrainDirectory/IGrainDirectoryPartition.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Orleans.Concurrency;

Expand All @@ -20,17 +21,30 @@ internal interface IGrainDirectoryPartition : ISystemTarget
ValueTask<GrainDirectoryPartitionSnapshot?> GetSnapshotAsync(MembershipVersion version, MembershipVersion rangeVersion, RingRange range);

[Alias("AcknowledgeSnapshotTransferAsync")]
ValueTask<bool> AcknowledgeSnapshotTransferAsync(SiloAddress silo, int partitionIndex, MembershipVersion version);
ValueTask<bool> AcknowledgeSnapshotTransferAsync(
SiloAddress silo,
int partitionIndex,
MembershipVersion version,
CancellationToken cancellationToken = default);
}

[Alias("IGrainDirectoryClient")]
internal interface IGrainDirectoryClient : ISystemTarget
{
[Alias("GetRegisteredActivations")]
ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(MembershipVersion membershipVersion, RingRange range, bool isValidation);
ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
bool isValidation,
CancellationToken cancellationToken = default);

[Alias("RecoverRegisteredActivations")]
ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(MembershipVersion membershipVersion, RingRange range, SiloAddress siloAddress, int partitionId);
ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
SiloAddress siloAddress,
int partitionId,
CancellationToken cancellationToken = default);
}

[Alias("IGrainDirectoryTestHooks")]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Orleans.Concurrency;
Expand All @@ -22,12 +23,18 @@ internal LocalGrainDirectoryClientCompatibility(SystemTargetShared shared)
shared.ActivationDirectory.RecordNewTarget(this);
}

public ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(MembershipVersion membershipVersion, RingRange range, bool isValidation)
public ValueTask<Immutable<List<GrainAddress>>> GetRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
bool isValidation,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var grainDirectoryResolver = _grainDirectoryResolver ??= ActivationServices.GetRequiredService<GrainDirectoryResolver>();
List<GrainAddress> result = [];
foreach (var (_, activation) in _localActivations)
{
cancellationToken.ThrowIfCancellationRequested();
if (!UsesLocalGrainDirectory(activation, grainDirectoryResolver))
{
continue;
Expand Down Expand Up @@ -66,8 +73,13 @@ private static bool UsesLocalGrainDirectory(IGrainContext activation, GrainDirec
return false;
}

public ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(MembershipVersion membershipVersion, RingRange range, SiloAddress siloAddress, int partitionId)
=> GetRegisteredActivations(membershipVersion, range, isValidation: false);
public ValueTask<Immutable<List<GrainAddress>>> RecoverRegisteredActivations(
MembershipVersion membershipVersion,
RingRange range,
SiloAddress siloAddress,
int partitionId,
CancellationToken cancellationToken)
=> GetRegisteredActivations(membershipVersion, range, isValidation: false, cancellationToken);
}

internal sealed class LocalGrainDirectoryPartitionCompatibility : SystemTarget, IGrainDirectoryPartition
Expand Down Expand Up @@ -106,5 +118,13 @@ public ValueTask<DirectoryResult<bool>> DeregisterAsync(MembershipVersion versio
return new((GrainDirectoryPartitionSnapshot?)null);
}

public ValueTask<bool> AcknowledgeSnapshotTransferAsync(SiloAddress silo, int partitionIndex, MembershipVersion version) => new(true);
public ValueTask<bool> AcknowledgeSnapshotTransferAsync(
SiloAddress silo,
int partitionIndex,
MembershipVersion version,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
return new(true);
}
}
Loading