Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 57 additions & 33 deletions src/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#nullable enable

using System.Diagnostics.CodeAnalysis;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Reactive.Threading.Tasks;
Expand Down Expand Up @@ -28,11 +29,7 @@ public sealed class ReminderDiagnosticObserver : IDisposable
private readonly IDisposable _storageSubscription;
private readonly Dictionary<GrainId, int> _tickCountsByGrain = [];
private readonly Dictionary<ReminderTickKey, int> _tickCountsByReminder = [];
private readonly Dictionary<ReminderTickKey, int> _tickAttemptsByReminder = [];
private readonly Dictionary<ReminderTickKey, HashSet<LocalReminderInstanceKey>> _activeLocalReminders = [];
private readonly Dictionary<ReminderTickKey, int> _localReminderTickWaitArmedCounts = [];
private readonly Dictionary<ReminderTickKey, long> _localReminderScheduleVersions = [];
private readonly Dictionary<ReminderTickKey, long> _localReminderTickWaitArmedVersions = [];
private readonly Dictionary<ReminderTickKey, Dictionary<LocalReminderInstanceKey, LocalReminderInstanceState>> _activeLocalReminders = [];
private readonly List<TickCountWaiter> _tickCountWaiters = [];
private readonly List<ActiveReminderCountWaiter> _activeReminderCountWaiters = [];
private readonly List<LocalReminderScheduleWaiter> _localReminderScheduleWaiters = [];
Expand Down Expand Up @@ -66,7 +63,17 @@ private void StoreEvent(ReminderEvents.ReminderEvent value)
{
case ReminderEvents.TickFiring tickFiring:
var tickFiringKey = new ReminderTickKey(tickFiring.GrainId, tickFiring.ReminderName);
_tickAttemptsByReminder[tickFiringKey] = _tickAttemptsByReminder.GetValueOrDefault(tickFiringKey) + 1;
if (_activeLocalReminders.TryGetValue(tickFiringKey, out var firingInstances))
{
foreach (var instance in firingInstances.Values)
{
if (Equals(instance.SiloAddress, tickFiring.SiloAddress))
{
instance.TickAttemptCount++;
}
}
}

break;
case ReminderEvents.TickCompleted tickCompleted:
_tickCountsByGrain[tickCompleted.GrainId] = _tickCountsByGrain.GetValueOrDefault(tickCompleted.GrainId) + 1;
Expand All @@ -78,13 +85,13 @@ private void StoreEvent(ReminderEvents.ReminderEvent value)
var startedKey = new ReminderTickKey(localReminderStarted.GrainId, localReminderStarted.ReminderName);
if (!_activeLocalReminders.TryGetValue(startedKey, out var startedInstances))
{
ResetLocalReminderScheduleState(startedKey);
startedInstances = [];
startedInstances = new Dictionary<LocalReminderInstanceKey, LocalReminderInstanceState>();
_activeLocalReminders[startedKey] = startedInstances;
}

startedInstances.Add(new LocalReminderInstanceKey(
localReminderStarted.Identity));
startedInstances.TryAdd(
new LocalReminderInstanceKey(localReminderStarted.Identity),
new LocalReminderInstanceState(localReminderStarted.SiloAddress));
ReleaseReadyActiveReminderWaiters(ready);
break;
case ReminderEvents.LocalReminderStopped localReminderStopped:
Expand All @@ -97,31 +104,31 @@ private void StoreEvent(ReminderEvents.ReminderEvent value)
if (stoppedInstances.Count == 0)
{
_activeLocalReminders.Remove(stoppedKey);
ResetLocalReminderScheduleState(stoppedKey);
}
}

ReleaseReadyActiveReminderWaiters(ready);
ReleaseReadyLocalReminderScheduleWaiters(ready);
break;
case ReminderEvents.LocalReminderScheduleChanged localReminderScheduleChanged:
var changedKey = new ReminderTickKey(localReminderScheduleChanged.GrainId, localReminderScheduleChanged.ReminderName);
if (IsActiveLocalReminder(changedKey, localReminderScheduleChanged.Identity))
if (TryGetLocalReminderInstance(changedKey, localReminderScheduleChanged.Identity, out var changedInstance))
{
_localReminderScheduleVersions[changedKey] = Math.Max(
_localReminderScheduleVersions.GetValueOrDefault(changedKey),
changedInstance.ScheduleVersion = Math.Max(
changedInstance.ScheduleVersion,
localReminderScheduleChanged.ScheduleVersion);
}

break;
case ReminderEvents.LocalReminderTickWaitArmed localReminderTickWaitArmed:
var tickWaitArmedKey = new ReminderTickKey(localReminderTickWaitArmed.GrainId, localReminderTickWaitArmed.ReminderName);
if (IsActiveLocalReminder(tickWaitArmedKey, localReminderTickWaitArmed.Identity)
&& localReminderTickWaitArmed.ScheduleVersion >= _localReminderScheduleVersions.GetValueOrDefault(tickWaitArmedKey))
if (TryGetLocalReminderInstance(tickWaitArmedKey, localReminderTickWaitArmed.Identity, out var armedInstance)
&& localReminderTickWaitArmed.ScheduleVersion >= armedInstance.ScheduleVersion)
{
_localReminderTickWaitArmedVersions[tickWaitArmedKey] = Math.Max(
_localReminderTickWaitArmedVersions.GetValueOrDefault(tickWaitArmedKey, -1),
armedInstance.TickWaitArmedVersion = Math.Max(
armedInstance.TickWaitArmedVersion,
localReminderTickWaitArmed.ScheduleVersion);
_localReminderTickWaitArmedCounts[tickWaitArmedKey] = _localReminderTickWaitArmedCounts.GetValueOrDefault(tickWaitArmedKey) + 1;
armedInstance.TickWaitArmedCount++;
ReleaseReadyLocalReminderScheduleWaiters(ready);
}

Expand Down Expand Up @@ -260,7 +267,7 @@ public Task WaitForActiveReminderCountAsync(GrainId grainId, int expectedCount,
}

/// <summary>
/// Waits until an active local owner has armed the next tick wait for a reminder.
/// Waits until all active local owners have armed the next tick wait for a reminder.
/// </summary>
public Task WaitForLocalReminderScheduleAsync(GrainId grainId, string reminderName, CancellationToken cancellationToken)
{
Expand Down Expand Up @@ -437,22 +444,30 @@ private bool IsLocalReminderScheduleReadyCore(GrainId grainId, string reminderNa
return false;
}

return _localReminderTickWaitArmedCounts.GetValueOrDefault(key) > _tickAttemptsByReminder.GetValueOrDefault(key)
&& _localReminderTickWaitArmedVersions.GetValueOrDefault(key, -1) >= _localReminderScheduleVersions.GetValueOrDefault(key);
}
foreach (var instance in _activeLocalReminders[key].Values)
{
if (instance.TickWaitArmedCount <= instance.TickAttemptCount
|| instance.TickWaitArmedVersion < instance.ScheduleVersion)
{
return false;
}
}

private bool IsActiveLocalReminder(ReminderTickKey key, object identity)
{
return _activeLocalReminders.TryGetValue(key, out var instances)
&& instances.Contains(new LocalReminderInstanceKey(identity));
return true;
}

private void ResetLocalReminderScheduleState(ReminderTickKey key)
private bool TryGetLocalReminderInstance(
ReminderTickKey key,
object identity,
[NotNullWhen(true)] out LocalReminderInstanceState? instance)
{
_tickAttemptsByReminder.Remove(key);
_localReminderTickWaitArmedCounts.Remove(key);
_localReminderScheduleVersions.Remove(key);
_localReminderTickWaitArmedVersions.Remove(key);
if (_activeLocalReminders.TryGetValue(key, out var instances))
{
return instances.TryGetValue(new LocalReminderInstanceKey(identity), out instance);
}

instance = null;
return false;
}

private void ReleaseReadyTickWaiters(List<TaskCompletionSource<bool>> ready)
Expand Down Expand Up @@ -508,6 +523,15 @@ private readonly record struct LocalReminderInstanceKey(object Identity)
public override int GetHashCode() => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(Identity);
}

private sealed class LocalReminderInstanceState(SiloAddress? siloAddress)
{
public SiloAddress? SiloAddress { get; } = siloAddress;
public int TickAttemptCount { get; set; }
public int TickWaitArmedCount { get; set; }
public long ScheduleVersion { get; set; }
public long TickWaitArmedVersion { get; set; } = -1;
}

private sealed class TickCountWaiter(GrainId grainId, string? reminderName, int targetCount)
{
public GrainId GrainId { get; } = grainId;
Expand Down Expand Up @@ -578,7 +602,7 @@ public static Task WaitForActiveReminderCountAsync(this ReminderDiagnosticObserv
}

/// <summary>
/// Waits until an active local owner has armed the next tick wait for a grain reminder.
/// Waits until all active local owners have armed the next tick wait for a grain reminder.
/// </summary>
public static Task WaitForLocalReminderScheduleAsync(this ReminderDiagnosticObserver observer, IAddressable grain, string reminderName, CancellationToken cancellationToken)
{
Expand Down
37 changes: 37 additions & 0 deletions test/Orleans.Reminders.Tests/Diagnostics/ReminderEventsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,43 @@ public async Task ReminderDiagnosticObserver_WaitsForReminderQuiescence_FromLife
Assert.Equal(0, observer.GetActiveReminderCount(grainId, reminderName));
}

[Fact, TestCategory("BVT")]
public async Task ReminderDiagnosticObserver_WaitsForCurrentOwnerSchedule_AfterOwnershipChange()
{
using var observer = ReminderDiagnosticObserver.Create();
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var grainId = GrainId.Create("test", "grain");
const string reminderName = "reminder";
var previousOwner = new object();
var currentOwner = new object();
var previousSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 14007), 8);
var currentSilo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 14008), 9);
var now = DateTime.UtcNow;

ReminderEvents.EmitLocalReminderStarted(grainId, reminderName, previousOwner, previousSilo);
ReminderEvents.EmitLocalReminderScheduleChanged(grainId, reminderName, previousOwner, 2, previousSilo);
ReminderEvents.EmitLocalReminderTickWaitArmed(grainId, reminderName, previousOwner, 2, previousSilo);
ReminderEvents.EmitLocalReminderStarted(grainId, reminderName, currentOwner, currentSilo);
ReminderEvents.EmitLocalReminderTickWaitArmed(grainId, reminderName, currentOwner, 0, currentSilo);
ReminderEvents.EmitTickFiring(
grainId,
reminderName,
new TickStatus(now, TimeSpan.FromSeconds(5), now),
previousSilo);

var waitTask = observer.WaitForLocalReminderScheduleAsync(grainId, reminderName, cts.Token);
Assert.False(waitTask.IsCompleted);

ReminderEvents.EmitLocalReminderStopped(
grainId,
reminderName,
previousOwner,
ReminderEvents.LocalReminderStopReason.RemovedFromRange,
previousSilo);

await waitTask.WaitAsync(TimeSpan.FromSeconds(1));
}

private sealed class Observer : IObserver<ReminderEvents.ReminderEvent>, IDisposable
{
private readonly IDisposable _subscription;
Expand Down
Loading