From 9fab788d204caf035b2ae9684331bee77bd42522 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 8 Jun 2026 15:06:15 -0700 Subject: [PATCH 1/2] test: stabilize timing-sensitive functional tests Wait for reminder service readiness in the minimal reminder test and drive activation collection through a convergence-based test hook instead of fixed timing delays. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Catalog/ActivationCollector.cs | 2 + .../MinimalReminderTests.cs | 83 ++++++++++++++++++- .../ActivationCollectorTests.cs | 45 +++++++--- 3 files changed, 115 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Runtime/Catalog/ActivationCollector.cs b/src/Orleans.Runtime/Catalog/ActivationCollector.cs index e6dd647b344..9919f28bd3e 100644 --- a/src/Orleans.Runtime/Catalog/ActivationCollector.cs +++ b/src/Orleans.Runtime/Catalog/ActivationCollector.cs @@ -95,6 +95,8 @@ public int GetNumRecentlyUsed(TimeSpan recencyPeriod) /// A representing the work performed. public Task CollectActivations(TimeSpan ageLimit, CancellationToken cancellationToken) => CollectActivationsImpl(false, ageLimit, cancellationToken); + internal Task CollectStaleActivations(CancellationToken cancellationToken) => CollectActivationsImpl(scanStale: true, ageLimit: default, cancellationToken: cancellationToken); + /// /// Schedules the provided grain context for collection if it becomes idle for the specified duration. /// diff --git a/test/Orleans.Reminders.Tests/MinimalReminderTests.cs b/test/Orleans.Reminders.Tests/MinimalReminderTests.cs index 091bdb3d39f..95f9240389a 100644 --- a/test/Orleans.Reminders.Tests/MinimalReminderTests.cs +++ b/test/Orleans.Reminders.Tests/MinimalReminderTests.cs @@ -1,3 +1,4 @@ +using Orleans.Runtime; using Orleans.TestingHost; using TestExtensions; using UnitTests.GrainInterfaces; @@ -40,13 +41,89 @@ public async Task MinimalReminderInterval() { var grainGuid = Guid.NewGuid(); const string reminderName = "minimal_reminder"; + var period = TimeSpan.FromMilliseconds(100); var reminderGrain = this.fixture.GrainFactory.GetGrain(grainGuid); - _ = await reminderGrain.StartReminder(reminderName, TimeSpan.FromMilliseconds(100), true); + _ = await WaitForReminderServiceReadiness(() => reminderGrain.StartReminder(reminderName, period, true)); - var r = await reminderGrain.GetReminderObject(reminderName); - await reminderGrain.StopReminder(r); + var r = await WaitForReminder(reminderGrain, reminderName); + await WaitForReminderServiceReadiness(() => reminderGrain.StopReminder(r)); } + + private static async Task WaitForReminder(IReminderTestGrain2 reminderGrain, string reminderName) + { + var deadline = DateTime.UtcNow + TestConstants.InitTimeout; + Exception lastException = null; + + while (true) + { + var remaining = deadline - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) + { + throw new TimeoutException($"Timed out waiting for reminder {reminderName} to be readable.", lastException); + } + + try + { + var reminder = await reminderGrain.GetReminderObject(reminderName).WaitAsync(remaining); + if (reminder is not null) + { + return reminder; + } + } + catch (OrleansException exception) when (IsReminderServiceInitializing(exception) && DateTime.UtcNow < deadline) + { + lastException = exception; + } + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + } + + private static async Task WaitForReminderServiceReadiness(Func> operation) + { + return await WaitUntilSuccess(operation); + } + + private static Task WaitForReminderServiceReadiness(Func operation) + { + return WaitUntilSuccess(async () => + { + await operation(); + return true; + }); + } + + private static async Task WaitUntilSuccess(Func> operation) + { + var deadline = DateTime.UtcNow + TestConstants.InitTimeout; + Exception lastException = null; + + while (true) + { + var remaining = deadline - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) + { + throw new TimeoutException("Timed out waiting for the reminder operation to complete.", lastException); + } + + try + { + return await operation().WaitAsync(remaining); + } + catch (OrleansException exception) when (IsReminderServiceInitializing(exception) && DateTime.UtcNow < deadline) + { + lastException = exception; + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + } + } + + private static bool IsReminderServiceInitializing(Exception exception) + { + return exception is OrleansException { Message: { } message } + && message.Contains("Reminder Service is still initializing", StringComparison.Ordinal); + } } } diff --git a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs index cc514c31a72..f24d7b0c24d 100644 --- a/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/ActivationsLifeCycleTests/ActivationCollectorTests.cs @@ -22,6 +22,7 @@ public class ActivationCollectorTests : OrleansTestingBase, IAsyncLifetime private static readonly TimeSpan DEFAULT_COLLECTION_QUANTUM = TimeSpan.FromSeconds(10); private static readonly TimeSpan DEFAULT_IDLE_TIMEOUT = DEFAULT_COLLECTION_QUANTUM + TimeSpan.FromSeconds(1); private static readonly TimeSpan WAIT_TIME = DEFAULT_IDLE_TIMEOUT.Multiply(3.0); + private static readonly TimeSpan COLLECTION_SPECIFIC_AGE_LIMIT = TimeSpan.FromSeconds(12); private TestCluster testCluster; @@ -57,7 +58,7 @@ public void Configure(IHostBuilder hostBuilder) { [typeof(IdleActivationGcTestGrain2).FullName] = DEFAULT_IDLE_TIMEOUT, [typeof(BusyActivationGcTestGrain2).FullName] = DEFAULT_IDLE_TIMEOUT, - [typeof(CollectionSpecificAgeLimitForTenSecondsActivationGcTestGrain).FullName] = TimeSpan.FromSeconds(12), + [typeof(CollectionSpecificAgeLimitForTenSecondsActivationGcTestGrain).FullName] = COLLECTION_SPECIFIC_AGE_LIMIT, }; }); }); @@ -495,9 +496,8 @@ async Task workerFunc() [Fact, TestCategory("ActivationCollector"), TestCategory("Functional")] public async Task ActivationCollectorShouldCollectByCollectionSpecificAgeLimitForTwelveSeconds() { - var waitTime = TimeSpan.FromSeconds(30); - var defaultCollectionAge = waitTime.Multiply(2); - //make sure defaultCollectionAge value won't cause activation collection in wait time + var defaultCollectionAge = COLLECTION_SPECIFIC_AGE_LIMIT.Multiply(4); + //make sure defaultCollectionAge value won't cause activation collection before the per-type limit await Initialize(defaultCollectionAge); const int grainCount = 1000; @@ -518,15 +518,36 @@ public async Task ActivationCollectorShouldCollectByCollectionSpecificAgeLimitFo Assert.Equal(grainCount, activationsCreated); logger.LogInformation( - "ActivationCollectorShouldCollectByCollectionSpecificAgeLimit: grains activated; waiting {WaitSeconds} sec (activation GC idle timeout is {DefaultIdleTimeout} sec).", - WAIT_TIME.TotalSeconds, - DEFAULT_IDLE_TIMEOUT.TotalSeconds); - - // Some time is required for GC to collect all of the Grains) - await Task.Delay(waitTime); + "ActivationCollectorShouldCollectByCollectionSpecificAgeLimit: grains activated; waiting for activation count to converge to zero using class-specific age limit {CollectionSpecificAgeLimit} sec.", + COLLECTION_SPECIFIC_AGE_LIMIT.TotalSeconds); - int activationsNotCollected = await TestUtils.GetActivationCount(this.testCluster.GrainFactory, fullGrainTypeName); - Assert.Equal(0, activationsNotCollected); + var activationCollector = this.testCluster.GetSiloServiceProvider().GetRequiredService(); + await activationCollector.CollectStaleActivations(CancellationToken.None); + int activationsAfterDefaultCollection = await TestUtils.GetActivationCount(this.testCluster.GrainFactory, fullGrainTypeName); + Assert.Equal(grainCount, activationsAfterDefaultCollection); + + await WaitForActivationCountToConverge(fullGrainTypeName, activationCollector, expectedCount: 0); + } + + private async Task WaitForActivationCountToConverge(string grainTypeName, ActivationCollector activationCollector, int expectedCount) + { + var deadline = DateTime.UtcNow + TestConstants.InitTimeout; + var activationCount = await TestUtils.GetActivationCount(this.testCluster.GrainFactory, grainTypeName); + + while (activationCount != expectedCount && DateTime.UtcNow < deadline) + { + await activationCollector.CollectStaleActivations(CancellationToken.None); + activationCount = await TestUtils.GetActivationCount(this.testCluster.GrainFactory, grainTypeName); + + if (activationCount == expectedCount) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(250)); + } + + Assert.Equal(expectedCount, activationCount); } [Fact, TestCategory("ActivationCollector"), TestCategory("Functional")] From 893c38d517de4c253ad040da1c51d036aecd2169 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 10 Jun 2026 16:57:18 -0700 Subject: [PATCH 2/2] test: use reminder diagnostics for readiness Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Diagnostics/ReminderEvents.cs | 60 ++++++++++ .../ReminderService/LocalReminderService.cs | 1 + .../ReminderDiagnosticObserver.cs | 16 +++ .../Orleans.Reminders/Orleans.Reminders.cs | 12 ++ .../MinimalReminderTests.cs | 105 +++++------------- 5 files changed, 118 insertions(+), 76 deletions(-) diff --git a/src/Orleans.Reminders/Diagnostics/ReminderEvents.cs b/src/Orleans.Reminders/Diagnostics/ReminderEvents.cs index 1b6194930fa..571467e4eb5 100644 --- a/src/Orleans.Reminders/Diagnostics/ReminderEvents.cs +++ b/src/Orleans.Reminders/Diagnostics/ReminderEvents.cs @@ -26,6 +26,31 @@ public static class ReminderEvents /// public static IObservable AllEvents { get; } = new Observable(); + /// + /// Gets an observable sequence of reminder service events. + /// + public static IObservable ServiceEvents { get; } = new ServiceObservable(); + + /// + /// The base class used for reminder service diagnostic events. + /// + /// The address of the silo associated with the event, if any. + public abstract class ReminderServiceEvent(SiloAddress? siloAddress) + { + /// + /// The address of the silo associated with the event, if any. + /// + public readonly SiloAddress? SiloAddress = siloAddress; + } + + /// + /// Event payload for when a reminder service completes startup and is ready for reminder operations. + /// + /// The address of the silo whose reminder service started. + public sealed class ReminderServiceStarted(SiloAddress? siloAddress) : ReminderServiceEvent(siloAddress) + { + } + /// /// The base class used for reminder diagnostic events. /// @@ -269,6 +294,22 @@ static void Emit(GrainId grainId, string reminderName, SiloAddress? siloAddress) } } + internal static void EmitReminderServiceStarted(SiloAddress? siloAddress) + { + if (!Listener.IsEnabled(nameof(ReminderServiceStarted))) + { + return; + } + + Emit(siloAddress); + + [MethodImpl(MethodImplOptions.NoInlining)] + static void Emit(SiloAddress? siloAddress) + { + Listener.Write(nameof(ReminderServiceStarted), new ReminderServiceStarted(siloAddress)); + } + } + internal static void EmitUnregistered(GrainId grainId, string reminderName, SiloAddress? siloAddress) { if (!Listener.IsEnabled(nameof(Unregistered))) @@ -458,4 +499,23 @@ public void OnNext(KeyValuePair value) } } } + + private sealed class ServiceObservable : IObservable + { + public IDisposable Subscribe(IObserver observer) => Listener.Subscribe(new Observer(observer)); + + private sealed class Observer(IObserver observer) : IObserver> + { + public void OnCompleted() => observer.OnCompleted(); + public void OnError(Exception error) => observer.OnError(error); + + public void OnNext(KeyValuePair value) + { + if (value.Value is ReminderServiceEvent evt) + { + observer.OnNext(evt); + } + } + } + } } diff --git a/src/Orleans.Reminders/ReminderService/LocalReminderService.cs b/src/Orleans.Reminders/ReminderService/LocalReminderService.cs index 0dd04dddbf0..b28e91ab7e2 100644 --- a/src/Orleans.Reminders/ReminderService/LocalReminderService.cs +++ b/src/Orleans.Reminders/ReminderService/LocalReminderService.cs @@ -439,6 +439,7 @@ private async Task DoInitialReadAndUpdateReminders() Status = GrainServiceStatus.Started; startedTask.TrySetResult(true); + ReminderEvents.EmitReminderServiceStarted(Silo); } catch (Exception ex) { diff --git a/src/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs b/src/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs index aae848f0826..c5427b6011d 100644 --- a/src/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs +++ b/src/Orleans.Testing.Reminders/ReminderDiagnosticObserver.cs @@ -22,7 +22,9 @@ public sealed class ReminderDiagnosticObserver : IDisposable { private readonly object _lock = new(); private readonly IConnectableObservable _events; + private readonly IConnectableObservable _serviceEvents; private readonly IDisposable _connection; + private readonly IDisposable _serviceConnection; private readonly IDisposable _storageSubscription; private readonly Dictionary _tickCountsByGrain = []; private readonly Dictionary _tickCountsByReminder = []; @@ -49,8 +51,10 @@ public static ReminderDiagnosticObserver Create() public ReminderDiagnosticObserver() { _events = ReminderEvents.AllEvents.Replay(); + _serviceEvents = ReminderEvents.ServiceEvents.Replay(); _storageSubscription = _events.Subscribe(StoreEvent); _connection = _events.Connect(); + _serviceConnection = _serviceEvents.Connect(); } private void StoreEvent(ReminderEvents.ReminderEvent value) @@ -199,6 +203,17 @@ public async Task WaitForTickConditionAsync(GrainId grainId, Func + /// Waits for a reminder service to complete startup. + /// + public Task WaitForReminderServiceStartedAsync(CancellationToken cancellationToken, SiloAddress? siloAddress = null) + { + return _serviceEvents + .OfType() + .FirstAsync(e => siloAddress is null || Equals(e.SiloAddress, siloAddress)) + .ToTask(cancellationToken); + } + /// /// Waits for a reminder to be unregistered. /// @@ -521,6 +536,7 @@ public void Dispose() { _storageSubscription.Dispose(); _connection.Dispose(); + _serviceConnection.Dispose(); } } diff --git a/src/api/Orleans.Reminders/Orleans.Reminders.cs b/src/api/Orleans.Reminders/Orleans.Reminders.cs index 4f45bb79c8d..c617bedb2e9 100644 --- a/src/api/Orleans.Reminders/Orleans.Reminders.cs +++ b/src/api/Orleans.Reminders/Orleans.Reminders.cs @@ -165,6 +165,7 @@ public static partial class ReminderEvents { public const string ListenerName = "Orleans.Reminders"; public static System.IObservable AllEvents { get { throw null; } } + public static System.IObservable ServiceEvents { get { throw null; } } public sealed partial class LocalReminderScheduleChanged : ReminderEvent { @@ -216,6 +217,17 @@ public abstract partial class ReminderEvent protected ReminderEvent(Runtime.GrainId grainId, string reminderName, Runtime.SiloAddress? siloAddress) { } } + public abstract partial class ReminderServiceEvent + { + public readonly Runtime.SiloAddress? SiloAddress; + protected ReminderServiceEvent(Runtime.SiloAddress? siloAddress) { } + } + + public sealed partial class ReminderServiceStarted : ReminderServiceEvent + { + public ReminderServiceStarted(Runtime.SiloAddress? siloAddress) : base(default) { } + } + public sealed partial class TickCompleted : ReminderEvent { public readonly Runtime.TickStatus Status; diff --git a/test/Orleans.Reminders.Tests/MinimalReminderTests.cs b/test/Orleans.Reminders.Tests/MinimalReminderTests.cs index 95f9240389a..74f828304cb 100644 --- a/test/Orleans.Reminders.Tests/MinimalReminderTests.cs +++ b/test/Orleans.Reminders.Tests/MinimalReminderTests.cs @@ -1,4 +1,5 @@ -using Orleans.Runtime; +using Orleans.Internal; +using Orleans.Testing.Reminders; using Orleans.TestingHost; using TestExtensions; using UnitTests.GrainInterfaces; @@ -15,10 +16,24 @@ public class MinimalReminderTests : IClassFixture public class Fixture : BaseTestClusterFixture { + public ReminderDiagnosticObserver ReminderObserver { get; } = ReminderDiagnosticObserver.Create(); + protected override void ConfigureTestCluster(TestClusterBuilder builder) { builder.AddSiloBuilderConfigurator(); } + + public override async Task DisposeAsync() + { + try + { + await base.DisposeAsync(); + } + finally + { + ReminderObserver.Dispose(); + } + } } public class SiloConfiguration : ISiloConfigurator @@ -44,86 +59,24 @@ public async Task MinimalReminderInterval() var period = TimeSpan.FromMilliseconds(100); var reminderGrain = this.fixture.GrainFactory.GetGrain(grainGuid); - _ = await WaitForReminderServiceReadiness(() => reminderGrain.StartReminder(reminderName, period, true)); - - var r = await WaitForReminder(reminderGrain, reminderName); - await WaitForReminderServiceReadiness(() => reminderGrain.StopReminder(r)); - - } + var grainId = reminderGrain.GetGrainId(); + var observer = this.fixture.ReminderObserver; - private static async Task WaitForReminder(IReminderTestGrain2 reminderGrain, string reminderName) - { - var deadline = DateTime.UtcNow + TestConstants.InitTimeout; - Exception lastException = null; - - while (true) + using var cts = new CancellationTokenSource(TestConstants.InitTimeout); + foreach (var silo in this.fixture.HostedCluster.Silos) { - var remaining = deadline - DateTime.UtcNow; - if (remaining <= TimeSpan.Zero) - { - throw new TimeoutException($"Timed out waiting for reminder {reminderName} to be readable.", lastException); - } - - try - { - var reminder = await reminderGrain.GetReminderObject(reminderName).WaitAsync(remaining); - if (reminder is not null) - { - return reminder; - } - } - catch (OrleansException exception) when (IsReminderServiceInitializing(exception) && DateTime.UtcNow < deadline) - { - lastException = exception; - } - - await Task.Delay(TimeSpan.FromMilliseconds(100)); + await observer.WaitForReminderServiceStartedAsync(cts.Token, silo.SiloAddress); } - } - - private static async Task WaitForReminderServiceReadiness(Func> operation) - { - return await WaitUntilSuccess(operation); - } - - private static Task WaitForReminderServiceReadiness(Func operation) - { - return WaitUntilSuccess(async () => - { - await operation(); - return true; - }); - } - - private static async Task WaitUntilSuccess(Func> operation) - { - var deadline = DateTime.UtcNow + TestConstants.InitTimeout; - Exception lastException = null; - - while (true) - { - var remaining = deadline - DateTime.UtcNow; - if (remaining <= TimeSpan.Zero) - { - throw new TimeoutException("Timed out waiting for the reminder operation to complete.", lastException); - } - try - { - return await operation().WaitAsync(remaining); - } - catch (OrleansException exception) when (IsReminderServiceInitializing(exception) && DateTime.UtcNow < deadline) - { - lastException = exception; - await Task.Delay(TimeSpan.FromMilliseconds(100)); - } - } - } + var registeredTask = observer.WaitForReminderRegisteredAsync(grainId, reminderName, cts.Token); + var reminder = await reminderGrain.StartReminder(reminderName, period, true).WaitAsync(cts.Token); + await registeredTask; + await observer.WaitForLocalReminderScheduleAsync(grainId, reminderName, cts.Token); - private static bool IsReminderServiceInitializing(Exception exception) - { - return exception is OrleansException { Message: { } message } - && message.Contains("Reminder Service is still initializing", StringComparison.Ordinal); + var unregisteredTask = observer.WaitForReminderUnregisteredAsync(grainId, reminderName, cts.Token); + await reminderGrain.StopReminder(reminder).WaitAsync(cts.Token); + await unregisteredTask; + await observer.WaitForReminderQuiescenceAsync(grainId, reminderName, cts.Token); } } }