From 496a1763690f92f82a6c9901cd85e8263b716f4e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 12:22:06 -0700 Subject: [PATCH 01/26] feat(journaling): support composable grain participants Initialize grain-scoped journaling participants before recovery, make state rollback explicit and fenced, preserve pending mutations across atomic snapshot retries, and reset binary replay sessions at independent snapshot value boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 209c9811-96cc-46b3-8705-e2ad263b1b98 --- src/Orleans.Journaling/DurableGrain.cs | 13 +- .../OrleansBinaryCommandCodecHelpers.cs | 8 + .../OrleansBinaryDurableListCommandCodec.cs | 2 +- .../OrleansBinaryDurableQueueCommandCodec.cs | 2 +- .../OrleansBinaryDurableSetCommandCodec.cs | 2 +- .../IJournaledGrainParticipant.cs | 12 + .../IJournaledStateManager.cs | 7 + .../JournaledStateManager.cs | 120 +++++-- .../Orleans.Journaling/Orleans.Journaling.cs | 6 + .../DurableListJournalBenchmarks.cs | 2 + .../DurableCollectionDirectWriteTests.cs | 2 + .../DurableListDirectWriteTests.cs | 2 + .../JournaledGrainParticipantTests.cs | 311 ++++++++++++++++++ .../StateManagerTests.cs | 139 +++++++- 14 files changed, 596 insertions(+), 32 deletions(-) create mode 100644 src/Orleans.Journaling/IJournaledGrainParticipant.cs create mode 100644 test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs diff --git a/src/Orleans.Journaling/DurableGrain.cs b/src/Orleans.Journaling/DurableGrain.cs index 7ffec5d7c2a..c37c43cae3f 100644 --- a/src/Orleans.Journaling/DurableGrain.cs +++ b/src/Orleans.Journaling/DurableGrain.cs @@ -11,6 +11,11 @@ protected DurableGrain() { participant.Participate(((IGrainBase)this).GrainContext.ObservableLifecycle); } + + foreach (var feature in ServiceProvider.GetServices()) + { + feature.Initialize(); + } } protected IJournaledStateManager StateManager { get; } @@ -31,5 +36,11 @@ protected TState GetOrCreateState(string name, Func return result; } - protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => StateManager.WriteStateAsync(cancellationToken); + /// + /// Writes pending journaled state. + /// + /// Cancellation token. + /// A task which completes when the state is written. + protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => + StateManager.WriteStateAsync(cancellationToken); } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs index 5b1e4757fe5..3a847a5ff7e 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs @@ -35,4 +35,12 @@ public static T ReadValue(IFieldCodec codec, ref Reader re reader.Session.Reset(); } } + + public static T ReadIndependentValue(IFieldCodec codec, ref Reader reader) + { + var result = ReadValue(codec, ref reader); + // Snapshot values are each encoded with a fresh serializer session. + reader.Session.Reset(); + return result; + } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs index ba38edf008a..a68d48d0c44 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs @@ -156,7 +156,7 @@ private void ApplySnapshot(ref Reader reader, IDurableListComman consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); + consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs index 2ab6b34fcfb..e9882426068 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs @@ -113,7 +113,7 @@ private void ApplySnapshot(ref Reader reader, IDurableQueueComma consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyEnqueue(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); + consumer.ApplyEnqueue(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs index 8b8585b13a3..a938b18fb58 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs @@ -115,7 +115,7 @@ private void ApplySnapshot(ref Reader reader, IDurableSetCommand consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); + consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); } } } diff --git a/src/Orleans.Journaling/IJournaledGrainParticipant.cs b/src/Orleans.Journaling/IJournaledGrainParticipant.cs new file mode 100644 index 00000000000..cbb6a5dc370 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledGrainParticipant.cs @@ -0,0 +1,12 @@ +namespace Orleans.Journaling; + +/// +/// Initializes a feature which contributes journaled state to a grain activation. +/// +public interface IJournaledGrainParticipant +{ + /// + /// Materializes the feature's grain-scoped services before journal recovery begins. + /// + void Initialize(); +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index cc8fd7af3eb..2861d6e3deb 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -38,6 +38,13 @@ public interface IJournaledStateManager : IAsyncDisposable /// A which represents the operation. ValueTask WriteStateAsync(CancellationToken cancellationToken); + /// + /// Discards uncommitted mutations and reloads the last durable state. + /// + /// The cancellation token. + /// A which represents the operation. + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); + /// /// Resets this instance, removing any persistent state. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 591a988271d..de6088f35b0 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -177,9 +177,42 @@ private async Task WorkLoop() { if (needsRecovery) { - await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); - needsRecovery = false; - CompleteRecoveryTrigger(); + bool fenceOnFailure; + lock (_lock) + { + fenceOnFailure = _state is not ManagerState.Unknown; + if (fenceOnFailure) + { + _state = ManagerState.Recovering; + } + } + + try + { + await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); + lock (_lock) + { + if (fenceOnFailure) + { + _state = ManagerState.Ready; + } + } + + needsRecovery = false; + CompleteRecoveryTrigger(); + } + catch + { + lock (_lock) + { + if (fenceOnFailure) + { + _state = ManagerState.Fenced; + } + } + throw; + } + } } WorkItem workItem; @@ -367,22 +400,6 @@ private async Task WorkLoop() var writeCompleted = false; try { - if (isSnapshot && hasBufferToConsume) - { - await AppendStorageAsync(bufferToConsume.AsReadOnlySequence(), _shutdownCancellation.Token).ConfigureAwait(true); - lock (_lock) - { - _journalWriter.Consume(bufferToConsume); - foreach (var state in _states.Values) - { - state.OnWriteCompleted(); - } - } - - bufferToConsume.Dispose(); - hasBufferToConsume = false; - } - if (isSnapshot) { await ReplaceStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true); @@ -461,6 +478,17 @@ private async Task WorkLoop() break; } + case RevertPendingChangesWorkItem: + { + await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); + lock (_lock) + { + _state = ManagerState.Ready; + } + + break; + } + case InitializeWorkItem: { lock (_lock) @@ -528,11 +556,22 @@ private async Task WorkLoop() Debug.Assert(recoveryTrigger is null); recoveryTrigger = workItem; recoveryTriggerException = exception; + lock (_lock) + { + _state = ManagerState.Fenced; + } needsRecovery = true; } else { workItem.SetException(exception); + if (workItem is RevertPendingChangesWorkItem) + { + lock (_lock) + { + _state = ManagerState.Fenced; + } + } } } finally @@ -664,10 +703,12 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; lock (_lock) { + ThrowIfWritesFenced(); task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -842,6 +883,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) string operation; lock (_lock) { + ThrowIfWritesFenced(); var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -867,6 +909,40 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) } } + public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + Task pendingRecovery; + bool didEnqueue; + lock (_lock) + { + if (_state is ManagerState.Unknown) + { + throw new InvalidOperationException("The journaled state manager has not been initialized."); + } + + pendingRecovery = EnqueueOrGetPendingWorkItem(out didEnqueue); + _state = ManagerState.Recovering; + } + + if (didEnqueue) + { + _workSignal.Signal(); + } + + await pendingRecovery.WaitAsync(cancellationToken); + } + + private void ThrowIfWritesFenced() + { + if (_state is ManagerState.Recovering or ManagerState.Fenced) + { + throw new InvalidOperationException( + "Journaled state writes are fenced until recovery completes successfully. Call RevertPendingChangesAsync to retry recovery."); + } + } + private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -1103,6 +1179,8 @@ private sealed class WriteSnapshotWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; + private sealed class RevertPendingChangesWorkItem : WorkItem; + private sealed class RegisterStateWorkItem(string name) : WorkItem(name) { public string Name => (string)Task.AsyncState!; @@ -1111,7 +1189,9 @@ private sealed class RegisterStateWorkItem(string name) : WorkItem(name) private enum ManagerState : byte { Unknown, - Ready + Ready, + Recovering, + Fenced } private sealed class StateDirectory( diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 83096227060..951cd580f52 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,6 +219,11 @@ public partial interface IJournaledState void Reset(JournalStreamWriter writer); } + public partial interface IJournaledGrainParticipant + { + void Initialize(); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { long PendingWriteByteCount { get; } @@ -226,6 +231,7 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken); void RegisterState(string name, IJournaledState state); + System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetState(string name, out IJournaledState? state); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index bb5c47cc049..5a7a602c620 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,6 +138,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 174b160d496..bdd53841f81 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,6 +205,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 2b2e90641d3..5dff6e54cd0 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,6 +104,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs b/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs new file mode 100644 index 00000000000..7958c14a83a --- /dev/null +++ b/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs @@ -0,0 +1,311 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Core.Internal; +using Orleans.TestingHost; +using Xunit; + +namespace Orleans.Journaling.Tests; + +[TestCategory("BVT")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Journaling")] +public sealed class JournaledGrainParticipantTests(JournaledGrainParticipantFixture fixture) + : IClassFixture +{ + [Fact] + public async Task DurableGrain_InitializesComposedParticipantsExactlyOnceBeforeRecovery() + { + var grain = fixture.Client.GetGrain(Guid.NewGuid()); + + Assert.Equal(new[] { 1, 1, 1, 1 }, await grain.GetCounts()); + await grain.SetValues("one", "two"); + Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); + + var activationId = await grain.GetActivationId(); + await grain.Cast().DeactivateOnIdle(); + + Assert.NotEqual(activationId, await grain.GetActivationId()); + Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); + Assert.Equal(new[] { 1, 1, 1, 1 }, await grain.GetCounts()); + var counts = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); + Assert.Equal( + new ParticipantCounts( + FirstConstructions: 2, + SecondConstructions: 2, + FailureConstructions: 2, + FirstInitializations: 2, + SecondInitializations: 2), + counts); + } + + [Fact] + public async Task DurableGrain_PropagatesParticipantConstructorFailure() + { + var grain = fixture.Client.GetGrain(Guid.NewGuid()); + + var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); + + Assert.Contains(ParticipantFailure.ConstructorMessage, exception.ToString(), StringComparison.Ordinal); + var counts = fixture.GetCounts(ConstructorFailureParticipantGrain.GrainTypeName); + Assert.True(counts.FailureConstructions > 0); + Assert.Equal(0, counts.FirstInitializations + counts.SecondInitializations); + } + + [Fact] + public async Task DurableGrain_PropagatesParticipantInitializeFailure() + { + var grain = fixture.Client.GetGrain(Guid.NewGuid()); + + var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); + + Assert.Contains(ParticipantFailure.InitializeMessage, exception.ToString(), StringComparison.Ordinal); + var counts = fixture.GetCounts(InitializeFailureParticipantGrain.GrainTypeName); + Assert.True(counts.FirstInitializations > 0); + Assert.Equal(counts.FirstConstructions, counts.FirstInitializations); + Assert.Equal(0, counts.SecondInitializations); + } + + [Fact] + public async Task DurableGrain_InitializesParticipantsBeforePropagatingActivationFailure() + { + var grain = fixture.Client.GetGrain(Guid.NewGuid()); + + var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); + + Assert.Contains(ParticipantFailure.ActivationMessage, exception.ToString(), StringComparison.Ordinal); + var counts = fixture.GetCounts(ActivationFailureParticipantGrain.GrainTypeName); + Assert.True(counts.FirstInitializations > 0); + Assert.Equal(counts.FirstConstructions, counts.FirstInitializations); + Assert.Equal(counts.SecondConstructions, counts.SecondInitializations); + } +} + +public sealed class JournaledGrainParticipantFixture : IntegrationTestFixture +{ + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + builder.ConfigureSilo((_, siloBuilder) => + { + siloBuilder.Services.AddSingleton(); + siloBuilder.Services.AddScoped(); + siloBuilder.Services.AddScoped(); + siloBuilder.Services.AddScoped(); + }); + } + + public ParticipantCounts GetCounts(string grainType) + { + var result = new ParticipantCounts(); + foreach (var silo in Cluster.Silos) + { + result += silo.ServiceProvider.GetRequiredService().GetCounts(grainType); + } + + return result; + } +} + +internal sealed class FirstJournaledParticipant : IJournaledGrainParticipant +{ + private readonly string _grainType; + private readonly IServiceProvider _serviceProvider; + private readonly ParticipantRecorder _recorder; + + public FirstJournaledParticipant(IGrainContext grainContext, IServiceProvider serviceProvider, ParticipantRecorder recorder) + { + _grainType = grainContext.GrainId.Type.ToString(); + _serviceProvider = serviceProvider; + _recorder = recorder; + recorder.RecordFirstConstruction(_grainType); + } + + public void Initialize() + { + _recorder.RecordFirstInitialization(_grainType); + if (_grainType == InitializeFailureParticipantGrain.GrainTypeName) + { + throw new InvalidOperationException(ParticipantFailure.InitializeMessage); + } + + _ = _serviceProvider.GetRequiredKeyedService>("participant-one"); + } +} + +internal sealed class SecondJournaledParticipant : IJournaledGrainParticipant +{ + private readonly string _grainType; + private readonly IServiceProvider _serviceProvider; + private readonly ParticipantRecorder _recorder; + + public SecondJournaledParticipant(IGrainContext grainContext, IServiceProvider serviceProvider, ParticipantRecorder recorder) + { + _grainType = grainContext.GrainId.Type.ToString(); + _serviceProvider = serviceProvider; + _recorder = recorder; + recorder.RecordSecondConstruction(_grainType); + } + + public void Initialize() + { + _recorder.RecordSecondInitialization(_grainType); + _ = _serviceProvider.GetRequiredKeyedService>("participant-two"); + } +} + +internal sealed class ParticipantFailure : IJournaledGrainParticipant +{ + public const string ConstructorMessage = "Expected participant constructor failure."; + public const string InitializeMessage = "Expected participant initialization failure."; + public const string ActivationMessage = "Expected grain activation failure."; + + public ParticipantFailure(IGrainContext grainContext, ParticipantRecorder recorder) + { + var grainType = grainContext.GrainId.Type.ToString(); + recorder.RecordFailureConstruction(grainType); + if (grainType == ConstructorFailureParticipantGrain.GrainTypeName) + { + throw new InvalidOperationException(ConstructorMessage); + } + } + + public void Initialize() + { + } +} + +public sealed class ParticipantRecorder +{ + private readonly ConcurrentDictionary _counts = new(StringComparer.Ordinal); + + public void RecordFirstConstruction(string grainType) => + _counts.AddOrUpdate(grainType, static _ => new() { FirstConstructions = 1 }, static (_, value) => value with { FirstConstructions = value.FirstConstructions + 1 }); + + public void RecordSecondConstruction(string grainType) => + _counts.AddOrUpdate(grainType, static _ => new() { SecondConstructions = 1 }, static (_, value) => value with { SecondConstructions = value.SecondConstructions + 1 }); + + public void RecordFailureConstruction(string grainType) => + _counts.AddOrUpdate(grainType, static _ => new() { FailureConstructions = 1 }, static (_, value) => value with { FailureConstructions = value.FailureConstructions + 1 }); + + public void RecordFirstInitialization(string grainType) => + _counts.AddOrUpdate(grainType, static _ => new() { FirstInitializations = 1 }, static (_, value) => value with { FirstInitializations = value.FirstInitializations + 1 }); + + public void RecordSecondInitialization(string grainType) => + _counts.AddOrUpdate(grainType, static _ => new() { SecondInitializations = 1 }, static (_, value) => value with { SecondInitializations = value.SecondInitializations + 1 }); + + public ParticipantCounts GetCounts(string grainType) => _counts.GetValueOrDefault(grainType); +} + +public readonly record struct ParticipantCounts( + int FirstConstructions = 0, + int SecondConstructions = 0, + int FailureConstructions = 0, + int FirstInitializations = 0, + int SecondInitializations = 0) +{ + public static ParticipantCounts operator +(ParticipantCounts left, ParticipantCounts right) => + new( + left.FirstConstructions + right.FirstConstructions, + left.SecondConstructions + right.SecondConstructions, + left.FailureConstructions + right.FailureConstructions, + left.FirstInitializations + right.FirstInitializations, + left.SecondInitializations + right.SecondInitializations); +} + +[GrainType(GrainTypeName)] +public sealed class ComposedParticipantGrain( + IGrainContext grainContext, + ParticipantRecorder recorder) : DurableGrain, IComposedParticipantGrain +{ + public const string GrainTypeName = "journaling-composed-participant"; + private readonly Guid _activationId = Guid.NewGuid(); + + public async Task SetValues(string first, string second) + { + GetValue("participant-one").Value = first; + GetValue("participant-two").Value = second; + await WriteStateAsync(); + } + + public Task GetValues() => + Task.FromResult(new[] { GetValue("participant-one").Value!, GetValue("participant-two").Value! }); + + public Task GetActivationId() => Task.FromResult(_activationId); + + public Task GetCounts() + { + var counts = recorder.GetCounts(grainContext.GrainId.Type.ToString()); + return Task.FromResult(new[] + { + counts.FirstConstructions, + counts.SecondConstructions, + counts.FirstInitializations, + counts.SecondInitializations + }); + } + + private IDurableValue GetValue(string name) + { + AssertStateExists(name, StateManager.TryGetState(name, out var state)); + return state as IDurableValue + ?? throw new InvalidOperationException($"The participant state '{name}' has an unexpected type."); + + static void AssertStateExists(string name, bool exists) + { + if (!exists) + { + throw new InvalidOperationException($"The participant state '{name}' was not initialized."); + } + } + } +} + +[GrainType(GrainTypeName)] +public sealed class ConstructorFailureParticipantGrain : DurableGrain, IConstructorFailureParticipantGrain +{ + public const string GrainTypeName = "journaling-constructor-failure-participant"; + + public Task Ping() => Task.CompletedTask; +} + +[GrainType(GrainTypeName)] +public sealed class InitializeFailureParticipantGrain : DurableGrain, IInitializeFailureParticipantGrain +{ + public const string GrainTypeName = "journaling-initialize-failure-participant"; + + public Task Ping() => Task.CompletedTask; +} + +[GrainType(GrainTypeName)] +public sealed class ActivationFailureParticipantGrain : DurableGrain, IActivationFailureParticipantGrain +{ + public const string GrainTypeName = "journaling-activation-failure-participant"; + + public override Task OnActivateAsync(CancellationToken cancellationToken) => + Task.FromException(new InvalidOperationException(ParticipantFailure.ActivationMessage)); + + public Task Ping() => Task.CompletedTask; +} + +public interface IComposedParticipantGrain : IGrainWithGuidKey +{ + Task SetValues(string first, string second); + Task GetValues(); + Task GetCounts(); + Task GetActivationId(); +} + +public interface IConstructorFailureParticipantGrain : IGrainWithGuidKey +{ + Task Ping(); +} + +public interface IInitializeFailureParticipantGrain : IGrainWithGuidKey +{ + Task Ping(); +} + +public interface IActivationFailureParticipantGrain : IGrainWithGuidKey +{ + Task Ping(); +} diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 0a157bf7149..47cf4e2f3ed 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -259,7 +259,7 @@ public async Task StateManager_BinarySnapshot_StoresBinaryVarUIntEntries() await sut.Manager.WriteStateAsync(CancellationToken.None); var replacement = Assert.Single(storage.Replaces); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); AssertContainsRuntimeAndApplicationEntries(ReadBinaryEntries(replacement)); } @@ -276,9 +276,8 @@ public async Task StateManager_StorageOperationBytesMetric_RecordsWritePayloadSi value.Value = 42; await sut.Manager.WriteStateAsync(CancellationToken.None); - var append = Assert.Single(storage.Appends); var replacement = Assert.Single(storage.Replaces); - Assert.Contains(observedBytes, measurement => measurement.Operation == "append" && measurement.Status == "ok" && measurement.Value == append.Length); + Assert.Empty(storage.Appends); Assert.Contains(observedBytes, measurement => measurement.Operation == "replace" && measurement.Status == "ok" && measurement.Value == replacement.Length); } @@ -382,8 +381,9 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData() dictionary.Add("key", 1); await sut.Manager.WriteStateAsync(CancellationToken.None); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); Assert.Single(storage.Replaces); + Assert.Equal(0, sut.Manager.PendingWriteByteCount); var recovered = CreateTestSystem(storage: storage, journalFormat: new TrackingJournalFormat(SessionPool)); var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec()); @@ -392,6 +392,58 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData() Assert.Equal(1, recoveredDictionary["key"]); } + [Fact] + public async Task StateManager_SnapshotFailure_PreservesMixedPendingMutationsForOneCommitRetry() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var notifications = new AlwaysWritingState(); + sut.Manager.RegisterState("notifications", notifications); + + await sut.Lifecycle.OnStart(); + dictionary.Add("persisted", 1); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(1, notifications.WriteCompletedCount); + + storage.IsCompactionRequested = true; + dictionary.Add("pending", 2); + value.Value = 2; + var pendingBytes = sut.Manager.PendingWriteByteCount; + var expected = new IOException("Expected replace failure."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + + Assert.Same(expected, exception); + Assert.Single(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(1, notifications.WriteCompletedCount); + Assert.Equal(pendingBytes, sut.Manager.PendingWriteByteCount); + Assert.Equal(2, dictionary["pending"]); + Assert.Equal(2, value.Value); + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Single(storage.Appends); + Assert.Single(storage.Replaces); + Assert.Equal(2, storage.ReplaceAttemptCount); + Assert.Equal(2, notifications.WriteCompletedCount); + Assert.Equal(0, sut.Manager.PendingWriteByteCount); + + var recovered = CreateTestSystem(storage: storage); + var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec()); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(); + + Assert.Equal(1, recoveredDictionary["persisted"]); + Assert.Equal(2, recoveredDictionary["pending"]); + Assert.Equal(2, recoveredValue.Value); + } + [Fact] public async Task StateManager_DirectWriteFailure_AbortsEntryBeforeMutation() { @@ -541,6 +593,63 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu Assert.False(dictionary.ContainsKey("second")); } + [Fact] + public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(); + dictionary.Add("persisted", 1); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + + dictionary.Add("pending", 2); + value.Value = 2; + Assert.True(sut.Manager.PendingWriteByteCount > 0); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(0, sut.Manager.PendingWriteByteCount); + Assert.Equal(1, dictionary["persisted"]); + Assert.False(dictionary.ContainsKey("pending")); + Assert.Equal(1, value.Value); + } + + [Fact] + public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfully() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + value.Value = 2; + + var expected = new IOException("Expected recovery failure."); + storage.NextReadException = expected; + var exception = await Assert.ThrowsAsync( + () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, exception); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase); + await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + Assert.Equal(1, value.Value); + + value.Value = 3; + await sut.Manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(3, value.Value); + } + [Fact] public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() { @@ -634,13 +743,13 @@ public async Task StateManager_SnapshotWrite_DoesNotDiscardActiveEntry() await writeTask.WaitAsync(TimeSpan.FromSeconds(10)); var replacement = Assert.Single(storage.Replaces); Assert.DoesNotContain(ReadBinaryEntries(replacement), entry => entry.StreamId.Value >= 8); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); storage.IsCompactionRequested = false; await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(2, storage.Appends.Count); - var append = storage.Appends[^1]; + Assert.Single(storage.Appends); + var append = storage.Appends[0]; Assert.Contains(ReadBinaryEntries(append), entry => entry.StreamId.Value >= 8); Task StartSnapshotAndCommitActiveEntry() @@ -1562,6 +1671,10 @@ private sealed class CapturingStorage : IJournalStorage public TaskCompletionSource AllowBlockedRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Exception? NextReplaceException { get; set; } + + public int ReplaceAttemptCount { get; private set; } + public int ReadConsumeCount { get; private set; } public void ResetReadConsumeCount() => ReadConsumeCount = 0; @@ -1577,7 +1690,6 @@ private sealed class CapturingStorage : IJournalStorage public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(consumer); - if (NextReadException is { } exception) { NextReadException = null; @@ -1637,6 +1749,13 @@ IEnumerable> GetSegments() public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + ReplaceAttemptCount++; + if (NextReplaceException is { } exception) + { + NextReplaceException = null; + throw exception; + } + if (DelayReplace) { ReplaceStarted.SetResult(); @@ -1908,6 +2027,8 @@ private sealed class AlwaysWritingState : IJournaledState { public int AppendEntriesCount { get; private set; } + public int WriteCompletedCount { get; private set; } + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) { } public void Reset(JournalStreamWriter writer) { } @@ -1923,6 +2044,8 @@ public void AppendEntries(JournalStreamWriter writer) public void AppendSnapshot(JournalStreamWriter writer) => AppendEntries(writer); + public void OnWriteCompleted() => WriteCompletedCount++; + public IJournaledState DeepCopy() => throw new NotSupportedException(); } From 072e213778c49ae2eeafd1d9949261ff39d84ee1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 12:49:03 -0700 Subject: [PATCH 02/26] test(journaling): assert participant counts per activation Compare singleton recorder snapshots around each activation so cumulative silo-local counts do not make the exactly-once lifecycle assertion placement-dependent. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 209c9811-96cc-46b3-8705-e2ad263b1b98 --- .../JournaledGrainParticipantTests.cs | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs b/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs index 7958c14a83a..20b92cae34b 100644 --- a/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs +++ b/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs @@ -17,26 +17,33 @@ public sealed class JournaledGrainParticipantTests(JournaledGrainParticipantFixt public async Task DurableGrain_InitializesComposedParticipantsExactlyOnceBeforeRecovery() { var grain = fixture.Client.GetGrain(Guid.NewGuid()); + var beforeFirstActivation = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); - Assert.Equal(new[] { 1, 1, 1, 1 }, await grain.GetCounts()); + var activationId = await grain.GetActivationId(); + var afterFirstActivation = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); + AssertActivationDelta(beforeFirstActivation, afterFirstActivation); await grain.SetValues("one", "two"); Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); - var activationId = await grain.GetActivationId(); await grain.Cast().DeactivateOnIdle(); Assert.NotEqual(activationId, await grain.GetActivationId()); Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); - Assert.Equal(new[] { 1, 1, 1, 1 }, await grain.GetCounts()); - var counts = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); - Assert.Equal( - new ParticipantCounts( - FirstConstructions: 2, - SecondConstructions: 2, - FailureConstructions: 2, - FirstInitializations: 2, - SecondInitializations: 2), - counts); + AssertActivationDelta( + afterFirstActivation, + fixture.GetCounts(ComposedParticipantGrain.GrainTypeName)); + + static void AssertActivationDelta(ParticipantCounts beforeActivation, ParticipantCounts afterActivation) + { + Assert.Equal( + new ParticipantCounts( + FirstConstructions: 1, + SecondConstructions: 1, + FailureConstructions: 1, + FirstInitializations: 1, + SecondInitializations: 1), + afterActivation - beforeActivation); + } } [Fact] @@ -210,12 +217,18 @@ public readonly record struct ParticipantCounts( left.FailureConstructions + right.FailureConstructions, left.FirstInitializations + right.FirstInitializations, left.SecondInitializations + right.SecondInitializations); + + public static ParticipantCounts operator -(ParticipantCounts left, ParticipantCounts right) => + new( + left.FirstConstructions - right.FirstConstructions, + left.SecondConstructions - right.SecondConstructions, + left.FailureConstructions - right.FailureConstructions, + left.FirstInitializations - right.FirstInitializations, + left.SecondInitializations - right.SecondInitializations); } [GrainType(GrainTypeName)] -public sealed class ComposedParticipantGrain( - IGrainContext grainContext, - ParticipantRecorder recorder) : DurableGrain, IComposedParticipantGrain +public sealed class ComposedParticipantGrain : DurableGrain, IComposedParticipantGrain { public const string GrainTypeName = "journaling-composed-participant"; private readonly Guid _activationId = Guid.NewGuid(); @@ -232,18 +245,6 @@ public Task GetValues() => public Task GetActivationId() => Task.FromResult(_activationId); - public Task GetCounts() - { - var counts = recorder.GetCounts(grainContext.GrainId.Type.ToString()); - return Task.FromResult(new[] - { - counts.FirstConstructions, - counts.SecondConstructions, - counts.FirstInitializations, - counts.SecondInitializations - }); - } - private IDurableValue GetValue(string name) { AssertStateExists(name, StateManager.TryGetState(name, out var state)); @@ -291,7 +292,6 @@ public interface IComposedParticipantGrain : IGrainWithGuidKey { Task SetValues(string first, string second); Task GetValues(); - Task GetCounts(); Task GetActivationId(); } From 924aeab521360a179a0a56606c260f7e53e12b96 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 19:35:53 -0700 Subject: [PATCH 03/26] Fix journal manager write lifecycle guards Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../JournaledStateManager.cs | 45 ++++++++-- .../StateManagerTests.cs | 83 +++++++++++++++++-- 2 files changed, 113 insertions(+), 15 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index de6088f35b0..bb735b214b9 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -708,7 +708,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) bool didEnqueue; lock (_lock) { - ThrowIfWritesFenced(); + ThrowIfWriteOperationsUnavailable(); task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -720,7 +720,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) var startTimestamp = _shared.TimeProvider.GetTimestamp(); try { - await task; + await task.WaitAsync(cancellationToken); _shared.Instruments.OnStateDeleteRequest(_shared.TimeProvider.GetElapsedTime(startTimestamp), succeeded: true); } catch @@ -883,7 +883,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) string operation; lock (_lock) { - ThrowIfWritesFenced(); + ThrowIfWriteOperationsUnavailable(); var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -934,6 +934,17 @@ public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationT await pendingRecovery.WaitAsync(cancellationToken); } + private void ThrowIfWriteOperationsUnavailable() + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_workLoop is null) + { + throw new InvalidOperationException("The journaled state manager has not been initialized."); + } + + ThrowIfWritesFenced(); + } + private void ThrowIfWritesFenced() { if (_state is ManagerState.Recovering or ManagerState.Fenced) @@ -1059,11 +1070,33 @@ private void BindState(string name, uint id) private async Task StopAsync(CancellationToken cancellationToken) { - _shutdownCancellation.Cancel(); + lock (_lock) + { + _shutdownCancellation.Cancel(); + } + _workSignal.Signal(); - if (_workLoop is { } task) + try { - await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + if (_workLoop is { } task) + { + await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + } + finally + { + CancelQueuedWorkItems(_shutdownCancellation.Token); + } + } + + private void CancelQueuedWorkItems(CancellationToken cancellationToken) + { + lock (_lock) + { + while (_workQueue.TryDequeue(out var workItem)) + { + workItem.TrySetCanceled(cancellationToken); + } } } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 47cf4e2f3ed..b56696b4227 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -86,6 +86,33 @@ public async Task StateManager_Initialize_ThrowsWhenCompletedReadLeavesData() Assert.Contains("did not read the completed journal data", exception.InnerException.Message, StringComparison.Ordinal); } + [Fact] + public async Task StateManager_WriteOperations_RequireInitialization() + { + var sut = CreateTestSystem(); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StateManager_WriteOperations_ObserveShutdown() + { + var sut = CreateTestSystem(); + await sut.Lifecycle.OnStart(); + await sut.Lifecycle.OnStop(CancellationToken.None); + + await Assert.ThrowsAnyAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + await Assert.ThrowsAnyAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + } + [Fact] public async Task StateManager_Recovery_PreservesUnknownStateEntry() { @@ -653,34 +680,40 @@ await Assert.ThrowsAsync( [Fact] public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() { - var storage = new CapturingStorage(); + var storage = new BlockingAppendStorage(); var sut = CreateTestSystem(storage: storage); var state = new AlwaysWritingState(); sut.Manager.RegisterState("state", state); + await sut.Lifecycle.OnStart(); var first = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + await storage.FirstAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); var second = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + var third = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); - await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)); - await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + storage.AllowFirstAppend.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(1, state.AppendEntriesCount); - Assert.Single(storage.Appends); + Assert.Equal(2, state.AppendEntriesCount); + Assert.Equal(2, storage.Appends.Count); } [Fact] public async Task StateManager_DeleteStateAsync_CoalescesQueuedDeletes() { - var storage = new CapturingStorage(); + var storage = new BlockingDeleteStorage(); var sut = CreateTestSystem(storage: storage); + await sut.Lifecycle.OnStart(); var first = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); + await storage.FirstDeleteStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); var second = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); + var third = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); - await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)); - await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + storage.AllowFirstDelete.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(1, storage.DeleteCount); + Assert.Equal(2, storage.DeleteCount); } [Fact] @@ -1649,6 +1682,38 @@ public async ValueTask AppendAsync(ReadOnlySequence value, CancellationTok public ValueTask DeleteAsync(CancellationToken cancellationToken) => default; } + private sealed class BlockingDeleteStorage : IJournalStorage + { + public TaskCompletionSource FirstDeleteStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowFirstDelete { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int DeleteCount { get; private set; } + + public bool IsCompactionRequested => false; + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + consumer.Complete(metadata: null); + return default; + } + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + DeleteCount++; + if (DeleteCount == 1) + { + FirstDeleteStarted.SetResult(); + await AllowFirstDelete.Task.WaitAsync(cancellationToken); + } + } + } + private sealed class CapturingStorage : IJournalStorage { private readonly List _segments = []; From a7bfdf4aeb277b123eb5c4d1314f694681ea31b6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 19:49:14 -0700 Subject: [PATCH 04/26] Clarify journal recovery write errors Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../JournaledStateManager.cs | 10 +++- .../StateManagerTests.cs | 56 ++++++++++++++++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index bb735b214b9..da1dc6b73b5 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -947,10 +947,14 @@ private void ThrowIfWriteOperationsUnavailable() private void ThrowIfWritesFenced() { - if (_state is ManagerState.Recovering or ManagerState.Fenced) + if (_state is ManagerState.Recovering) { - throw new InvalidOperationException( - "Journaled state writes are fenced until recovery completes successfully. Call RevertPendingChangesAsync to retry recovery."); + throw new InvalidOperationException("Journaled state writes are unavailable while recovery is in progress."); + } + + if (_state is ManagerState.Fenced) + { + throw new InvalidOperationException("Journaled state writes are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); } } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index b56696b4227..f8700339c36 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -666,8 +666,10 @@ public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfull var writeException = await Assert.ThrowsAsync( () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase); - await Assert.ThrowsAsync( + Assert.Contains("Call RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); + var deleteException = await Assert.ThrowsAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("Call RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal); await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); Assert.Equal(1, value.Value); @@ -677,6 +679,30 @@ await Assert.ThrowsAsync( Assert.Equal(3, value.Value); } + [Fact] + public async Task StateManager_WriteOperations_ReportRecoveryInProgress() + { + var storage = new BlockingRecoveryStorage(); + var sut = CreateTestSystem(storage: storage); + await sut.Lifecycle.OnStart(); + + var recovery = sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask(); + await storage.RecoveryReadStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("recovery is in progress", writeException.Message, StringComparison.Ordinal); + Assert.Contains("recovery is in progress", deleteException.Message, StringComparison.Ordinal); + Assert.DoesNotContain("RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); + Assert.DoesNotContain("RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal); + + storage.AllowRecoveryRead.SetResult(); + await recovery.WaitAsync(TimeSpan.FromSeconds(10)); + } + [Fact] public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() { @@ -1714,6 +1740,34 @@ public async ValueTask DeleteAsync(CancellationToken cancellationToken) } } + private sealed class BlockingRecoveryStorage : IJournalStorage + { + private int _readCount; + + public TaskCompletionSource RecoveryReadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowRecoveryRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool IsCompactionRequested => false; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _readCount) == 2) + { + RecoveryReadStarted.SetResult(); + await AllowRecoveryRead.Task.WaitAsync(cancellationToken); + } + + consumer.Complete(metadata: null); + } + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => default; + } + private sealed class CapturingStorage : IJournalStorage { private readonly List _segments = []; From 0f9ed9484e097fef89e9f408e7111616c6349bd1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Wed, 19 Aug 2026 19:59:42 -0700 Subject: [PATCH 05/26] Require successful journal initialization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../JournaledStateManager.cs | 2 +- .../StateManagerTests.cs | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index da1dc6b73b5..6b39b0dfe67 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -937,7 +937,7 @@ public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationT private void ThrowIfWriteOperationsUnavailable() { _shutdownCancellation.Token.ThrowIfCancellationRequested(); - if (_workLoop is null) + if (_state is ManagerState.Unknown) { throw new InvalidOperationException("The journaled state manager has not been initialized."); } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index f8700339c36..23ea203035f 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -100,6 +100,26 @@ public async Task StateManager_WriteOperations_RequireInitialization() Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); } + [Fact] + public async Task StateManager_WriteOperations_RequireSuccessfulInitialization() + { + var storage = new CapturingStorage + { + NextReadException = new IOException("Expected recovery failure.") + }; + var sut = CreateTestSystem(storage: storage); + + await Assert.ThrowsAsync(() => sut.Lifecycle.OnStart()); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); + } + [Fact] public async Task StateManager_WriteOperations_ObserveShutdown() { @@ -1017,6 +1037,7 @@ public async Task StateManager_RecoveryRetry_ReplaysFixedStorage() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); Assert.Equal(42, value.Value); @@ -1033,6 +1054,7 @@ public async Task StateManager_RecoveryRetry_PreservesUnknownStreamOnce() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); var replacement = Assert.Single(storage.Replaces); @@ -1050,6 +1072,7 @@ public async Task StateManager_RecoveryRetry_RemovesStaleRetiredPlaceholder() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); Assert.False(sut.Manager.TryGetState("stale", out _)); From 412467113ab2802e1fc957e51403f6cd0bf00988 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 07:33:03 -0700 Subject: [PATCH 06/26] Add stable journal observer snapshots Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../IJournaledStateManager.cs | 11 + .../IJournaledStateObserver.cs | 56 +++ .../JournaledStateManager.cs | 84 ++++- .../Orleans.Journaling/Orleans.Journaling.cs | 9 + .../StateManagerTests.cs | 337 +++++++++++++++++- 5 files changed, 495 insertions(+), 2 deletions(-) create mode 100644 src/Orleans.Journaling/IJournaledStateObserver.cs diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 2861d6e3deb..2a00291b05a 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -24,6 +24,17 @@ public interface IJournaledStateManager : IAsyncDisposable /// The state instance to register. void RegisterState(string name, IJournaledState state); + /// + /// Registers an observer for durable write and recovery notifications. + /// + /// The observer. + /// + /// Each operation uses a stable snapshot of the registered observers. An observer registered + /// from a callback participates in subsequent operations, not later phases of the current one. + /// + void RegisterObserver(IJournaledStateObserver observer) => + throw new NotSupportedException("This journaled state manager does not support observers."); + /// /// Attempts to get a state registered with the manager. /// diff --git a/src/Orleans.Journaling/IJournaledStateObserver.cs b/src/Orleans.Journaling/IJournaledStateObserver.cs new file mode 100644 index 00000000000..c649a51f5ea --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateObserver.cs @@ -0,0 +1,56 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Orleans.Journaling; + +/// +/// Observes durable state manager commit and recovery boundaries. +/// +/// +/// +/// Callbacks run synchronously on the owning grain scheduler and must not block. Implementations +/// should queue asynchronous follow-up work instead of performing it inline. +/// +/// +/// Before each write, the manager awaits and then invokes +/// immediately before capturing registered states. Mutations remain +/// provisional until marks a successful commit. Restoring the last +/// durable state, including through , +/// invokes after all registered states have been restored. +/// +/// Exceptions from abort the write before state is captured. +/// Exceptions from the synchronous notification methods are logged and don't change the outcome +/// of a completed commit or recovery. +/// Each operation uses one stable snapshot of registered observers for all of its callbacks. +/// Observers registered from a callback begin participating in subsequent operations. +/// +public interface IJournaledStateObserver +{ + /// + /// Prepares external prerequisites for a durable write before registered states are captured. + /// + /// The cancellation token. + /// A task representing the preparation operation. + /// + /// State mutated by this method is included in the pending write. Throwing aborts the write. + /// Implementations must be idempotent because a failed storage write can be retried. + /// + ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) => default; + + /// + /// Called immediately before registered states are captured for a durable write. + /// Mutations made after this callback aren't included in the corresponding + /// boundary. + /// + void OnWriteStarted(); + + /// + /// Called after all pending journal entries have been committed to stable storage. + /// + void OnWriteCompleted(); + + /// + /// Called after all registered states have been restored to the last durable version. + /// + void OnRecoveryCompleted(); +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6b39b0dfe67..1b0df8b4fda 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -19,6 +19,7 @@ internal sealed partial class JournaledStateManager : IJournaledStateManager, IJ #endif private readonly Dictionary _states = new(StringComparer.Ordinal); private readonly Dictionary _statesMap = []; + private readonly HashSet _observers = []; private readonly JournaledStateManagerShared _shared; private readonly IJournalStorage _storage; private readonly JournalBufferWriter _journalWriter; @@ -130,6 +131,18 @@ public void RegisterState(string name, IJournaledState state) _workSignal.Signal(); } + public void RegisterObserver(IJournaledStateObserver observer) + { + ArgumentNullException.ThrowIfNull(observer); + lock (_lock) + { + if (!_observers.Add(observer)) + { + throw new InvalidOperationException("The journaled state observer is already registered."); + } + } + } + public async ValueTask InitializeAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -281,6 +294,29 @@ private async Task WorkLoop() var hasBufferToConsume = false; var bufferToConsumeIsCommittedBuffer = false; + IJournaledStateObserver[] observers; + lock (_lock) + { + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + await observer.OnWritePreparingAsync(_shutdownCancellation.Token).ConfigureAwait(true); + } + + foreach (var observer in observers) + { + try + { + observer.OnWriteStarted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteStarted)); + } + } + lock (_lock) { if (isSnapshot) @@ -445,7 +481,14 @@ private async Task WorkLoop() { foreach (var state in _states.Values) { - state.OnWriteCompleted(); + try + { + state.OnWriteCompleted(); + } + catch (Exception exception) + { + LogStateCallbackError(_shared.Logger, exception, state.GetType().FullName ?? state.GetType().Name); + } } if (isSnapshot) @@ -454,6 +497,19 @@ private async Task WorkLoop() } } } + + foreach (var observer in observers) + { + try + { + observer.OnWriteCompleted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteCompleted)); + } + } + break; } @@ -669,6 +725,7 @@ private void RetireOrResurectStates() _retirementTracker.ApplyRemove(name); } } + } } @@ -749,6 +806,7 @@ private async Task RecoverAsync(CancellationToken cancellationToken) throw; } + IJournaledStateObserver[] observers; lock (_lock) { foreach ((var name, var state) in _states) @@ -764,6 +822,20 @@ private async Task RecoverAsync(CancellationToken cancellationToken) } } } + + observers = [.. _observers]; + } + + foreach (var observer in observers) + { + try + { + observer.OnRecoveryCompleted(); + } + catch (Exception exception) + { + LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnRecoveryCompleted)); + } } } @@ -1363,6 +1435,16 @@ void IJournaledState.AppendEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journaled state observer callback {Callback} failed.")] + private static partial void LogObserverError(ILogger logger, Exception exception, string callback); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journaled state {StateType} failed its write-completion callback.")] + private static partial void LogStateCallbackError(ILogger logger, Exception exception, string stateType); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 951cd580f52..652df1b3ad8 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -230,6 +230,7 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken); + void RegisterObserver(IJournaledStateObserver observer); void RegisterState(string name, IJournaledState state); System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); @@ -242,6 +243,14 @@ public partial interface IJournaledStateManagerFactory IJournaledStateManager Create(JournalId journalId); } + public partial interface IJournaledStateObserver + { + void OnRecoveryCompleted(); + void OnWriteCompleted(); + System.Threading.Tasks.ValueTask OnWritePreparingAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + void OnWriteStarted(); + } + public partial interface IJournalFormat { string FormatKey { get; } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 23ea203035f..0458d452d1f 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -665,6 +665,219 @@ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() Assert.Equal(1, value.Value); } + [Fact] + public async Task StateManager_Observer_NotifiesOnceAfterWholeRecoveryAndCommit() + { + var sut = CreateTestSystem(); + _ = new DurableDictionary("first", sut.Manager, CreateDictionaryCodec()); + var second = new DurableDictionary("second", sut.Manager, CreateDictionaryCodec()); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + + await sut.Lifecycle.OnStart(); + second["value"] = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal(1, observer.RecoveryCompletedCount); + Assert.Equal(1, observer.WritePreparingCount); + Assert.Equal(1, observer.WriteStartedCount); + Assert.Equal(1, observer.WriteCompletedCount); + Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); + } + + [Fact] + public async Task StateManager_Observer_FailedWriteDoesNotCommitAndRevertNotifiesRecovery() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + + await sut.Lifecycle.OnStart(); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + storage.NextAppendException = new IOException("Expected write failure."); + value.Value = 2; + + await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Equal(2, observer.WriteStartedCount); + Assert.Equal(1, observer.WriteCompletedCount); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(1, value.Value); + Assert.Equal(2, observer.RecoveryCompletedCount); + Assert.Equal(2, observer.WriteStartedCount); + Assert.Equal(1, observer.WriteCompletedCount); + } + + [Fact] + public async Task StateManager_ObserverException_DoesNotChangeCompletedOperationOutcome() + { + var sut = CreateTestSystem(); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + sut.Manager.RegisterObserver(new ThrowingStateObserver()); + + await sut.Lifecycle.OnStart(); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + value.Value = 2; + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(1, value.Value); + } + + [Fact] + public async Task StateManager_Observer_NoOpWriteStillPairsStartAndCompletion() + { + var sut = CreateTestSystem(); + await sut.Lifecycle.OnStart(); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal(1, observer.WritePreparingCount); + Assert.Equal(1, observer.WriteStartedCount); + Assert.Equal(1, observer.WriteCompletedCount); + Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); + } + + [Theory] + [InlineData(false, ObserverRegistrationPhase.Preparing)] + [InlineData(false, ObserverRegistrationPhase.Started)] + [InlineData(false, ObserverRegistrationPhase.Completed)] + [InlineData(true, ObserverRegistrationPhase.Preparing)] + [InlineData(true, ObserverRegistrationPhase.Started)] + [InlineData(true, ObserverRegistrationPhase.Completed)] + public async Task StateManager_ObserverRegisteredDuringWritePhase_JoinsNextWrite( + bool hasPendingState, + ObserverRegistrationPhase registrationPhase) + { + var sut = CreateTestSystem(); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var lateObserver = new RecordingStateObserver(); + var registeringObserver = new RegisteringStateObserver( + onWritePreparing: RegisterDuring(ObserverRegistrationPhase.Preparing), + onWriteStarted: RegisterDuring(ObserverRegistrationPhase.Started), + onWriteCompleted: RegisterDuring(ObserverRegistrationPhase.Completed)); + sut.Manager.RegisterObserver(registeringObserver); + await sut.Lifecycle.OnStart(); + if (hasPendingState) + { + value.Value = 1; + } + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal(["Preparing", "Started", "Completed"], registeringObserver.WriteCalls); + Assert.Empty(lateObserver.WriteCalls); + + if (hasPendingState) + { + value.Value = 2; + } + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal( + ["Preparing", "Started", "Completed", "Preparing", "Started", "Completed"], + registeringObserver.WriteCalls); + Assert.Equal(["Preparing", "Started", "Completed"], lateObserver.WriteCalls); + + Action? RegisterDuring(ObserverRegistrationPhase phase) => + registrationPhase == phase ? () => sut.Manager.RegisterObserver(lateObserver) : null; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StateManager_Observer_WritePhasesAreConsistentForAppendAndSnapshot(bool isSnapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = isSnapshot }; + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(); + value.Value = 1; + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); + if (isSnapshot) + { + Assert.Single(storage.Replaces); + Assert.Empty(storage.Appends); + } + else + { + Assert.Single(storage.Appends); + Assert.Empty(storage.Replaces); + } + } + + [Fact] + public async Task StateManager_ObserverRegisteredDuringRecovery_JoinsNextRecovery() + { + var sut = CreateTestSystem(); + var lateObserver = new RecordingStateObserver(); + var registeringObserver = new RegisteringStateObserver( + onRecoveryCompleted: () => sut.Manager.RegisterObserver(lateObserver)); + sut.Manager.RegisterObserver(registeringObserver); + + await sut.Lifecycle.OnStart(); + + Assert.Equal(1, registeringObserver.RecoveryCompletedCount); + Assert.Equal(0, lateObserver.RecoveryCompletedCount); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(2, registeringObserver.RecoveryCompletedCount); + Assert.Equal(1, lateObserver.RecoveryCompletedCount); + } + + [Fact] + public async Task StateManager_PreparationFailureAbortsCaptureAndRetryPersistsPreparedState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new PreparingStateObserver(() => value.Value = "owned"); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(); + + observer.FailNext = true; + await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + + Assert.Empty(storage.Appends); + Assert.Equal(0, observer.WriteStartedCount); + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Single(storage.Appends); + Assert.Equal("owned", value.Value); + Assert.Equal(2, observer.PreparingCount); + Assert.Equal(1, observer.WriteStartedCount); + } + + [Fact] + public async Task StateManager_StateCompletionFailure_DoesNotSkipCommitObserver() + { + var sut = CreateTestSystem(); + sut.Manager.RegisterState("throwing", new AlwaysWritingState { ThrowOnWriteCompleted = true }); + var observer = new RecordingStateObserver(); + sut.Manager.RegisterObserver(observer); + + await sut.Lifecycle.OnStart(); + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Equal(1, observer.WriteCompletedCount); + } + [Fact] public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfully() { @@ -1398,6 +1611,119 @@ static async Task TriggerCompaction(IJournaledStateManager manager, DurableDicti } } + private sealed class RecordingStateObserver : IJournaledStateObserver + { + public List WriteCalls { get; } = []; + public int WritePreparingCount { get; private set; } + public int WriteStartedCount { get; private set; } + public int WriteCompletedCount { get; private set; } + public int RecoveryCompletedCount { get; private set; } + + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + WriteCalls.Add("Preparing"); + WritePreparingCount++; + return default; + } + + public void OnWriteStarted() + { + WriteCalls.Add("Started"); + WriteStartedCount++; + } + + public void OnWriteCompleted() + { + WriteCalls.Add("Completed"); + WriteCompletedCount++; + } + + public void OnRecoveryCompleted() => RecoveryCompletedCount++; + } + + private sealed class ThrowingStateObserver : IJournaledStateObserver + { + public void OnWriteStarted() => throw new InvalidOperationException("Expected observer failure."); + public void OnWriteCompleted() => throw new InvalidOperationException("Expected observer failure."); + public void OnRecoveryCompleted() => throw new InvalidOperationException("Expected observer failure."); + } + + private sealed class RegisteringStateObserver( + Action? onWritePreparing = null, + Action? onWriteStarted = null, + Action? onWriteCompleted = null, + Action? onRecoveryCompleted = null) : IJournaledStateObserver + { + private Action? _onWritePreparing = onWritePreparing; + private Action? _onWriteStarted = onWriteStarted; + private Action? _onWriteCompleted = onWriteCompleted; + private Action? _onRecoveryCompleted = onRecoveryCompleted; + + public List WriteCalls { get; } = []; + public int RecoveryCompletedCount { get; private set; } + + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + WriteCalls.Add("Preparing"); + Interlocked.Exchange(ref _onWritePreparing, null)?.Invoke(); + return default; + } + + public void OnWriteStarted() + { + WriteCalls.Add("Started"); + Interlocked.Exchange(ref _onWriteStarted, null)?.Invoke(); + } + + public void OnWriteCompleted() + { + WriteCalls.Add("Completed"); + Interlocked.Exchange(ref _onWriteCompleted, null)?.Invoke(); + } + + public void OnRecoveryCompleted() + { + RecoveryCompletedCount++; + Interlocked.Exchange(ref _onRecoveryCompleted, null)?.Invoke(); + } + } + + public enum ObserverRegistrationPhase + { + Preparing, + Started, + Completed + } + + private sealed class PreparingStateObserver(Action prepare) : IJournaledStateObserver + { + public bool FailNext { get; set; } + public int PreparingCount { get; private set; } + public int WriteStartedCount { get; private set; } + + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + PreparingCount++; + prepare(); + if (FailNext) + { + FailNext = false; + throw new IOException("Expected preparation failure."); + } + + return default; + } + + public void OnWriteStarted() => WriteStartedCount++; + public void OnWriteCompleted() + { + } + + public void OnRecoveryCompleted() + { + } + } + private sealed class StreamingOnlyStorage : IJournalStorage { public bool StreamingReadCalled { get; private set; } @@ -2171,6 +2497,8 @@ private sealed class AlwaysWritingState : IJournaledState public int WriteCompletedCount { get; private set; } + public bool ThrowOnWriteCompleted { get; init; } + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) { } public void Reset(JournalStreamWriter writer) { } @@ -2186,7 +2514,14 @@ public void AppendEntries(JournalStreamWriter writer) public void AppendSnapshot(JournalStreamWriter writer) => AppendEntries(writer); - public void OnWriteCompleted() => WriteCompletedCount++; + public void OnWriteCompleted() + { + WriteCompletedCount++; + if (ThrowOnWriteCompleted) + { + throw new InvalidOperationException("Expected state completion failure."); + } + } public IJournaledState DeepCopy() => throw new NotSupportedException(); } From 809d3bed2bb59e847dbdcac8c6eed21ae26e6bd2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 07:43:41 -0700 Subject: [PATCH 07/26] Guard journal observer registration after shutdown Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/JournaledStateManager.cs | 1 + test/Orleans.Journaling.Tests/StateManagerTests.cs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 1b0df8b4fda..b12da1084ae 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -134,6 +134,7 @@ public void RegisterState(string name, IJournaledState state) public void RegisterObserver(IJournaledStateObserver observer) { ArgumentNullException.ThrowIfNull(observer); + _shutdownCancellation.Token.ThrowIfCancellationRequested(); lock (_lock) { if (!_observers.Add(observer)) diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 0458d452d1f..2d69fff6f12 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Collections.Concurrent; using System.Diagnostics.Metrics; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; using Orleans.Serialization.Buffers; @@ -131,6 +132,8 @@ await Assert.ThrowsAnyAsync( () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); await Assert.ThrowsAnyAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + Assert.ThrowsAny( + () => sut.Manager.RegisterObserver(new RecordingStateObserver())); } [Fact] @@ -2221,7 +2224,7 @@ public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationTo if (NextReplaceException is { } exception) { NextReplaceException = null; - throw exception; + ExceptionDispatchInfo.Throw(exception); } if (DelayReplace) From acf19c358567db033781d1715046404384d6767d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 07:58:06 -0700 Subject: [PATCH 08/26] Preserve journal manager interface compatibility Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/IJournaledStateManager.cs | 6 +++++- src/Orleans.Journaling/JournaledStateManager.cs | 2 +- test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs | 2 -- .../DurableCollectionDirectWriteTests.cs | 2 -- .../Orleans.Journaling.Tests/DurableListDirectWriteTests.cs | 2 -- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 2a00291b05a..f4efed1f330 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -54,7 +54,11 @@ void RegisterObserver(IJournaledStateObserver observer) => /// /// The cancellation token. /// A which represents the operation. - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); + /// + /// Implementations override this member to provide rollback to the last durable state. + /// + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => + throw new NotSupportedException("This journaled state manager does not support reverting pending changes."); /// /// Resets this instance, removing any persistent state. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index b12da1084ae..cc6c623ccb9 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -134,9 +134,9 @@ public void RegisterState(string name, IJournaledState state) public void RegisterObserver(IJournaledStateObserver observer) { ArgumentNullException.ThrowIfNull(observer); - _shutdownCancellation.Token.ThrowIfCancellationRequested(); lock (_lock) { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (!_observers.Add(observer)) { throw new InvalidOperationException("The journaled state observer is already registered."); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index 5a7a602c620..bb5c47cc049 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,8 +138,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index bdd53841f81..174b160d496 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,8 +205,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 5dff6e54cd0..2b2e90641d3 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,8 +104,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } From 1791acdd994752a3117bef739a5bcdc8756a29d1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 08:07:57 -0700 Subject: [PATCH 09/26] Guard journal recovery enqueue after shutdown Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/JournaledStateManager.cs | 2 +- test/Orleans.Journaling.Tests/StateManagerTests.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index cc6c623ccb9..d70162147e1 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -985,11 +985,11 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - _shutdownCancellation.Token.ThrowIfCancellationRequested(); Task pendingRecovery; bool didEnqueue; lock (_lock) { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_state is ManagerState.Unknown) { throw new InvalidOperationException("The journaled state manager has not been initialized."); diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 2d69fff6f12..40509f18af7 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -132,6 +132,8 @@ await Assert.ThrowsAnyAsync( () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); await Assert.ThrowsAnyAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + await Assert.ThrowsAnyAsync( + () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); Assert.ThrowsAny( () => sut.Manager.RegisterObserver(new RecordingStateObserver())); } From 50a1bda3e83ddb4c79564d83eb0e36934ce1c40a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 08:37:53 -0700 Subject: [PATCH 10/26] fix(journaling): snapshot observers before recovery Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../JournaledStateManager.cs | 5 ++-- .../StateManagerTests.cs | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index d70162147e1..bc877ebd85b 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -726,7 +726,6 @@ private void RetireOrResurectStates() _retirementTracker.ApplyRemove(name); } } - } } @@ -791,8 +790,10 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) private async Task RecoverAsync(CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); + IJournaledStateObserver[] observers; lock (_lock) { + observers = [.. _observers]; ResetForRecovery(); } @@ -807,7 +808,6 @@ private async Task RecoverAsync(CancellationToken cancellationToken) throw; } - IJournaledStateObserver[] observers; lock (_lock) { foreach ((var name, var state) in _states) @@ -824,7 +824,6 @@ private async Task RecoverAsync(CancellationToken cancellationToken) } } - observers = [.. _observers]; } foreach (var observer in observers) diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 40509f18af7..ff702e722c2 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -844,6 +844,24 @@ public async Task StateManager_ObserverRegisteredDuringRecovery_JoinsNextRecover Assert.Equal(1, lateObserver.RecoveryCompletedCount); } + [Fact] + public async Task StateManager_ObserverRegisteredByStateDuringRecovery_JoinsNextRecovery() + { + var sut = CreateTestSystem(); + var lateObserver = new RecordingStateObserver(); + sut.Manager.RegisterState( + "registering", + new RecoveryRegisteringState(() => sut.Manager.RegisterObserver(lateObserver))); + + await sut.Lifecycle.OnStart(); + + Assert.Equal(0, lateObserver.RecoveryCompletedCount); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(1, lateObserver.RecoveryCompletedCount); + } + [Fact] public async Task StateManager_PreparationFailureAbortsCaptureAndRetryPersistsPreparedState() { @@ -2531,6 +2549,18 @@ public void OnWriteCompleted() public IJournaledState DeepCopy() => throw new NotSupportedException(); } + private sealed class RecoveryRegisteringState(Action onRecoveryCompleted) : IJournaledState + { + private Action? _onRecoveryCompleted = onRecoveryCompleted; + + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } + public void Reset(JournalStreamWriter writer) { } + public void AppendEntries(JournalStreamWriter writer) { } + public void AppendSnapshot(JournalStreamWriter writer) { } + public void OnRecoveryCompleted() => Interlocked.Exchange(ref _onRecoveryCompleted, null)?.Invoke(); + public IJournaledState DeepCopy() => throw new NotSupportedException(); + } + private sealed class ThrowingDictionarySetCodec : IDurableDictionaryCommandCodec where K : notnull { public void WriteSet(K key, V value, JournalStreamWriter writer) From aaa3b30293d8af842f663308913a32e4b74327a3 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 08:59:12 -0700 Subject: [PATCH 11/26] docs(journaling): clarify observer snapshots Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 209c9811-96cc-46b3-8705-e2ad263b1b98 --- src/Orleans.Journaling/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index c02bb4328ba..e44540c031e 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -65,6 +65,10 @@ siloBuilder To migrate to JSON, configure `JournaledStateManagerOptions.JournalFormatKey` to `JsonJournalExtensions.JournalFormatKey` and call `UseJsonJournalFormat(...)`. When a grain recovers data written with a different format than the configured write format, the next write is forced to a full snapshot so the journal is rewritten using JSON and the storage format metadata is updated. +Journaled state observers are snapshotted once at the start of each write or recovery +operation. An observer registered from inside a callback begins participating in the +next operation, so every phase of the current operation sees the same observer set. + ## Example - Using durable states ```csharp using Microsoft.Extensions.DependencyInjection; From 9b15f705939459b323e16462b15e2d43ed3b6a99 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 09:19:30 -0700 Subject: [PATCH 12/26] Materialize journaling participants before initialization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/DurableGrain.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/DurableGrain.cs b/src/Orleans.Journaling/DurableGrain.cs index c37c43cae3f..ff51517b780 100644 --- a/src/Orleans.Journaling/DurableGrain.cs +++ b/src/Orleans.Journaling/DurableGrain.cs @@ -12,7 +12,8 @@ protected DurableGrain() participant.Participate(((IGrainBase)this).GrainContext.ObservableLifecycle); } - foreach (var feature in ServiceProvider.GetServices()) + var features = ServiceProvider.GetServices().ToArray(); + foreach (var feature in features) { feature.Initialize(); } From 6cee8c4289db664d01ba71b9f05e95510fbdde27 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 09:30:41 -0700 Subject: [PATCH 13/26] Align journaling observer API baseline Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/api/Orleans.Journaling/Orleans.Journaling.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 652df1b3ad8..ff33275ab31 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -247,7 +247,7 @@ public partial interface IJournaledStateObserver { void OnRecoveryCompleted(); void OnWriteCompleted(); - System.Threading.Tasks.ValueTask OnWritePreparingAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + System.Threading.Tasks.ValueTask OnWritePreparingAsync(System.Threading.CancellationToken cancellationToken); void OnWriteStarted(); } From 5517304b3f0b3ca8e41a70b2b40bdbb9cffce923 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Thu, 20 Aug 2026 09:51:55 -0700 Subject: [PATCH 14/26] fix(journaling): preserve rollback interface compatibility Copilot-Session: 209c9811-96cc-46b3-8705-e2ad263b1b98 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../IJournaledStateManager.cs | 2 +- .../StateManagerTests.cs | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index f4efed1f330..9eba4a173a3 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -58,7 +58,7 @@ void RegisterObserver(IJournaledStateObserver observer) => /// Implementations override this member to provide rollback to the last durable state. /// ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => - throw new NotSupportedException("This journaled state manager does not support reverting pending changes."); + ValueTask.FromException(new NotSupportedException("This journaled state manager does not support rollback.")); /// /// Resets this instance, removing any persistent state. diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index ff702e722c2..9ef31ee8062 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -690,6 +690,17 @@ public async Task StateManager_Observer_NotifiesOnceAfterWholeRecoveryAndCommit( Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); } + [Fact] + public async Task StateManager_DefaultRollbackImplementation_PreservesLegacyImplementations() + { + IJournaledStateManager manager = new LegacyStateManager(); + + var operation = manager.RevertPendingChangesAsync(CancellationToken.None); + var exception = await Assert.ThrowsAsync(() => operation.AsTask()); + + Assert.Contains("does not support rollback", exception.Message, StringComparison.Ordinal); + } + [Fact] public async Task StateManager_Observer_FailedWriteDoesNotCommitAndRevertNotifiesRecovery() { @@ -1664,6 +1675,23 @@ public void OnWriteCompleted() public void OnRecoveryCompleted() => RecoveryCompletedCount++; } + private sealed class LegacyStateManager : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + public void RegisterState(string name, IJournaledState state) { } + + public bool TryGetState( + string name, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IJournaledState? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } + private sealed class ThrowingStateObserver : IJournaledStateObserver { public void OnWriteStarted() => throw new InvalidOperationException("Expected observer failure."); From cb4c42db8101b806c71a53942b01bf260ef1cabb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 02:31:22 -0700 Subject: [PATCH 15/26] fix(journaling): preserve retryable conflict recovery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/JournaledStateManager.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index bc877ebd85b..8b86fc34b1e 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -221,12 +221,12 @@ private async Task WorkLoop() { if (fenceOnFailure) { - _state = ManagerState.Fenced; + _state = ManagerState.Ready; } } + throw; } - } } WorkItem workItem; @@ -615,8 +615,9 @@ private async Task WorkLoop() recoveryTriggerException = exception; lock (_lock) { - _state = ManagerState.Fenced; + _state = ManagerState.Recovering; } + needsRecovery = true; } else From c56baf4c726ff768b16e35adab4f386388e31e1c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 03:21:46 -0700 Subject: [PATCH 16/26] fix(journaling): model pending recovery retries Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/JournaledStateManager.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 8b86fc34b1e..592ed0a8e7b 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -208,7 +208,7 @@ private async Task WorkLoop() { if (fenceOnFailure) { - _state = ManagerState.Ready; + _state = ManagerState.RecoveryPending; } } @@ -1015,6 +1015,7 @@ private void ThrowIfWriteOperationsUnavailable() throw new InvalidOperationException("The journaled state manager has not been initialized."); } + // Recovery-pending operations are queued as retry triggers; recovery completes before they are processed. ThrowIfWritesFenced(); } @@ -1300,6 +1301,7 @@ private enum ManagerState : byte { Unknown, Ready, + RecoveryPending, Recovering, Fenced } From 96bee116f46e629911b1a3f0449f9ca334cd399f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 03:37:51 -0700 Subject: [PATCH 17/26] refactor(journaling): make recovery states explicit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../JournaledStateManager.cs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 592ed0a8e7b..52d5241302e 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1010,25 +1010,19 @@ public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationT private void ThrowIfWriteOperationsUnavailable() { _shutdownCancellation.Token.ThrowIfCancellationRequested(); - if (_state is ManagerState.Unknown) + switch (_state) { - throw new InvalidOperationException("The journaled state manager has not been initialized."); - } - - // Recovery-pending operations are queued as retry triggers; recovery completes before they are processed. - ThrowIfWritesFenced(); - } - - private void ThrowIfWritesFenced() - { - if (_state is ManagerState.Recovering) - { - throw new InvalidOperationException("Journaled state writes are unavailable while recovery is in progress."); - } - - if (_state is ManagerState.Fenced) - { - throw new InvalidOperationException("Journaled state writes are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); + case ManagerState.Unknown: + throw new InvalidOperationException("The journaled state manager has not been initialized."); + case ManagerState.Ready: + case ManagerState.RecoveryPending: + return; + case ManagerState.Recovering: + throw new InvalidOperationException("Journaled state writes are unavailable while recovery is in progress."); + case ManagerState.Fenced: + throw new InvalidOperationException("Journaled state writes are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); + default: + throw new UnreachableException(); } } From 798df201bfd13bd3b9f9097f3e37b1195427649d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 15:29:49 -0700 Subject: [PATCH 18/26] refactor(journaling): require new interface members Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- .../IJournaledStateManager.cs | 9 ++---- .../IJournaledStateObserver.cs | 2 +- .../DurableListJournalBenchmarks.cs | 4 +++ .../DurableCollectionDirectWriteTests.cs | 4 +++ .../DurableListDirectWriteTests.cs | 4 +++ .../StateManagerTests.cs | 29 +------------------ 6 files changed, 16 insertions(+), 36 deletions(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 9eba4a173a3..b8fcabe3977 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -32,8 +32,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// Each operation uses a stable snapshot of the registered observers. An observer registered /// from a callback participates in subsequent operations, not later phases of the current one. /// - void RegisterObserver(IJournaledStateObserver observer) => - throw new NotSupportedException("This journaled state manager does not support observers."); + void RegisterObserver(IJournaledStateObserver observer); /// /// Attempts to get a state registered with the manager. @@ -54,11 +53,7 @@ void RegisterObserver(IJournaledStateObserver observer) => /// /// The cancellation token. /// A which represents the operation. - /// - /// Implementations override this member to provide rollback to the last durable state. - /// - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => - ValueTask.FromException(new NotSupportedException("This journaled state manager does not support rollback.")); + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); /// /// Resets this instance, removing any persistent state. diff --git a/src/Orleans.Journaling/IJournaledStateObserver.cs b/src/Orleans.Journaling/IJournaledStateObserver.cs index c649a51f5ea..b069d57d4e9 100644 --- a/src/Orleans.Journaling/IJournaledStateObserver.cs +++ b/src/Orleans.Journaling/IJournaledStateObserver.cs @@ -35,7 +35,7 @@ public interface IJournaledStateObserver /// State mutated by this method is included in the pending write. Throwing aborts the write. /// Implementations must be idempotent because a failed storage write can be retried. /// - ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) => default; + ValueTask OnWritePreparingAsync(CancellationToken cancellationToken); /// /// Called immediately before registered states are captured for a durable write. diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index bb5c47cc049..02e03061f10 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -130,6 +130,8 @@ private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter bu public void RegisterState(string name, IJournaledState state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); + public void RegisterObserver(IJournaledStateObserver observer) { } + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState state) { state = null!; @@ -138,6 +140,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 174b160d496..0194fa182fb 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -197,6 +197,8 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn public void RegisterState(string name, IJournaledState state) => state.Reset(writer.CreateWriter()); + public void RegisterObserver(IJournaledStateObserver observer) { } + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) { state = null; @@ -205,6 +207,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 2b2e90641d3..37260a15490 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -96,6 +96,8 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn public void RegisterState(string name, IJournaledState state) => state.Reset(writer.CreateWriter()); + public void RegisterObserver(IJournaledStateObserver observer) { } + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) { state = null; @@ -104,6 +106,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 9ef31ee8062..8c8cda769f1 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -690,17 +690,6 @@ public async Task StateManager_Observer_NotifiesOnceAfterWholeRecoveryAndCommit( Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); } - [Fact] - public async Task StateManager_DefaultRollbackImplementation_PreservesLegacyImplementations() - { - IJournaledStateManager manager = new LegacyStateManager(); - - var operation = manager.RevertPendingChangesAsync(CancellationToken.None); - var exception = await Assert.ThrowsAsync(() => operation.AsTask()); - - Assert.Contains("does not support rollback", exception.Message, StringComparison.Ordinal); - } - [Fact] public async Task StateManager_Observer_FailedWriteDoesNotCommitAndRevertNotifiesRecovery() { @@ -1675,25 +1664,9 @@ public void OnWriteCompleted() public void OnRecoveryCompleted() => RecoveryCompletedCount++; } - private sealed class LegacyStateManager : IJournaledStateManager - { - public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; - public void RegisterState(string name, IJournaledState state) { } - - public bool TryGetState( - string name, - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IJournaledState? state) - { - state = null; - return false; - } - - public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; - } - private sealed class ThrowingStateObserver : IJournaledStateObserver { + public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) => default; public void OnWriteStarted() => throw new InvalidOperationException("Expected observer failure."); public void OnWriteCompleted() => throw new InvalidOperationException("Expected observer failure."); public void OnRecoveryCompleted() => throw new InvalidOperationException("Expected observer failure."); From 686ba1ceb117703955adc13dd1aaad447960fa30 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 19:30:29 -0700 Subject: [PATCH 19/26] fix(journaling): preserve observer interface compatibility --- .../IJournaledStateManager.cs | 5 +- .../IJournaledStateObserver.cs | 16 ++-- .../StateManagerTests.cs | 86 +++++++++++++++++++ 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b8fcabe3977..b79c09483ab 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -31,8 +31,11 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Each operation uses a stable snapshot of the registered observers. An observer registered /// from a callback participates in subsequent operations, not later phases of the current one. + /// Implementations enable observer support by overriding this method. The default implementation + /// throws . /// - void RegisterObserver(IJournaledStateObserver observer); + void RegisterObserver(IJournaledStateObserver observer) => + throw new NotSupportedException("Observer registration requires a state manager implementation which supports observers."); /// /// Attempts to get a state registered with the manager. diff --git a/src/Orleans.Journaling/IJournaledStateObserver.cs b/src/Orleans.Journaling/IJournaledStateObserver.cs index b069d57d4e9..5081fb8feee 100644 --- a/src/Orleans.Journaling/IJournaledStateObserver.cs +++ b/src/Orleans.Journaling/IJournaledStateObserver.cs @@ -8,15 +8,17 @@ namespace Orleans.Journaling; /// /// /// -/// Callbacks run synchronously on the owning grain scheduler and must not block. Implementations -/// should queue asynchronous follow-up work instead of performing it inline. +/// The manager awaits , which can perform short, +/// cancellation-aware asynchronous preparation before state capture. The remaining callbacks run +/// synchronously on the owning grain scheduler and must return promptly. Implementations should +/// queue asynchronous follow-up work from those synchronous callbacks. /// /// -/// Before each write, the manager awaits and then invokes -/// immediately before capturing registered states. Mutations remain -/// provisional until marks a successful commit. Restoring the last -/// durable state, including through , -/// invokes after all registered states have been restored. +/// Before each write, the manager invokes immediately before capturing +/// registered states. Mutations remain provisional until marks a +/// successful commit. Restoring the last durable state, including through +/// , invokes +/// after all registered states have been restored. /// /// Exceptions from abort the write before state is captured. /// Exceptions from the synchronous notification methods are logged and don't change the outcome diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 8c8cda769f1..edecf4137be 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -690,6 +690,43 @@ public async Task StateManager_Observer_NotifiesOnceAfterWholeRecoveryAndCommit( Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); } + [Fact] + public void StateManager_DefaultRegisterObserver_PreservesLegacyImplementations() + { + IJournaledStateManager manager = new LegacyStateManager(); + + var exception = Assert.Throws( + () => manager.RegisterObserver(new RecordingStateObserver())); + + Assert.Contains("Observer registration requires", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StateManager_AwaitsObserverPreparationBeforeCapturingState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var observer = new BlockingPreparingStateObserver(() => value.Value = 42); + sut.Manager.RegisterObserver(observer); + await sut.Lifecycle.OnStart(); + + var write = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + await observer.PreparingStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.False(write.IsCompleted); + Assert.Empty(storage.Appends); + + observer.AllowPreparation.SetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Single(storage.Appends); + var recovered = CreateTestSystem(storage: storage); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(); + Assert.Equal(42, recoveredValue.Value); + } + [Fact] public async Task StateManager_Observer_FailedWriteDoesNotCommitAndRevertNotifiesRecovery() { @@ -1634,6 +1671,29 @@ static async Task TriggerCompaction(IJournaledStateManager manager, DurableDicti } } + private sealed class LegacyStateManager : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; + + public void RegisterState(string name, IJournaledState state) + { + } + + public bool TryGetState( + string name, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IJournaledState? state) + { + state = null; + return false; + } + + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; + } + private sealed class RecordingStateObserver : IJournaledStateObserver { public List WriteCalls { get; } = []; @@ -1748,6 +1808,32 @@ public void OnRecoveryCompleted() } } + private sealed class BlockingPreparingStateObserver(Action prepare) : IJournaledStateObserver + { + public TaskCompletionSource PreparingStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowPreparation { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) + { + PreparingStarted.SetResult(); + await AllowPreparation.Task.WaitAsync(cancellationToken); + prepare(); + } + + public void OnWriteStarted() + { + } + + public void OnWriteCompleted() + { + } + + public void OnRecoveryCompleted() + { + } + } + private sealed class StreamingOnlyStorage : IJournalStorage { public bool StreamingReadCalled { get; private set; } From 76213dbcc5fa5a2abfb6dad430be234b0fd5faac Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 21:10:21 -0700 Subject: [PATCH 20/26] fix(journaling): preserve rollback interface compatibility --- src/Orleans.Journaling/IJournaledStateManager.cs | 7 ++++++- test/Orleans.Journaling.Tests/StateManagerTests.cs | 13 +++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b79c09483ab..bb12a59cf72 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -56,7 +56,12 @@ void RegisterObserver(IJournaledStateObserver observer) => /// /// The cancellation token. /// A which represents the operation. - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); + /// + /// Implementations enable rollback support by overriding this method. The default implementation + /// throws . + /// + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => + ValueTask.FromException(new NotSupportedException("Rollback requires a state manager implementation which supports rollback.")); /// /// Resets this instance, removing any persistent state. diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index edecf4137be..d58cd20bd4f 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -701,6 +701,17 @@ public void StateManager_DefaultRegisterObserver_PreservesLegacyImplementations( Assert.Contains("Observer registration requires", exception.Message, StringComparison.Ordinal); } + [Fact] + public async Task StateManager_DefaultRevertPendingChanges_PreservesLegacyImplementations() + { + IJournaledStateManager manager = new LegacyStateManager(); + + var exception = await Assert.ThrowsAsync( + () => manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); + + Assert.Contains("Rollback requires", exception.Message, StringComparison.Ordinal); + } + [Fact] public async Task StateManager_AwaitsObserverPreparationBeforeCapturingState() { @@ -1689,8 +1700,6 @@ public bool TryGetState( public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } From 1f1d9b960ca8479c45b153920abc56cdc40ae581 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 15:48:42 -0700 Subject: [PATCH 21/26] refactor(journaling): reuse grain lifecycle participation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/DurableGrain.cs | 18 +- .../IJournaledGrainParticipant.cs | 12 - .../Orleans.Journaling/Orleans.Journaling.cs | 7 +- .../JournaledGrainParticipantTests.cs | 311 ------------------ 4 files changed, 9 insertions(+), 339 deletions(-) delete mode 100644 src/Orleans.Journaling/IJournaledGrainParticipant.cs delete mode 100644 test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs diff --git a/src/Orleans.Journaling/DurableGrain.cs b/src/Orleans.Journaling/DurableGrain.cs index ff51517b780..a0c236181e2 100644 --- a/src/Orleans.Journaling/DurableGrain.cs +++ b/src/Orleans.Journaling/DurableGrain.cs @@ -2,25 +2,23 @@ namespace Orleans.Journaling; -public abstract class DurableGrain : Grain, IGrainBase +public abstract class DurableGrain : Grain, IGrainBase, ILifecycleParticipant { protected DurableGrain() { StateManager = ServiceProvider.GetRequiredService(); - if (StateManager is ILifecycleParticipant participant) - { - participant.Participate(((IGrainBase)this).GrainContext.ObservableLifecycle); - } + } + + protected IJournaledStateManager StateManager { get; } - var features = ServiceProvider.GetServices().ToArray(); - foreach (var feature in features) + void ILifecycleParticipant.Participate(IGrainLifecycle lifecycle) + { + if (StateManager is ILifecycleParticipant participant) { - feature.Initialize(); + participant.Participate(lifecycle); } } - protected IJournaledStateManager StateManager { get; } - protected TState GetOrCreateState(string name) where TState : class, IJournaledState => GetOrCreateState(name, static sp => sp.GetRequiredService(), ServiceProvider); diff --git a/src/Orleans.Journaling/IJournaledGrainParticipant.cs b/src/Orleans.Journaling/IJournaledGrainParticipant.cs deleted file mode 100644 index cbb6a5dc370..00000000000 --- a/src/Orleans.Journaling/IJournaledGrainParticipant.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Initializes a feature which contributes journaled state to a grain activation. -/// -public interface IJournaledGrainParticipant -{ - /// - /// Materializes the feature's grain-scoped services before journal recovery begins. - /// - void Initialize(); -} diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index ff33275ab31..b936c5222e5 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -8,7 +8,7 @@ //------------------------------------------------------------------------------ namespace Orleans.Journaling { - public abstract partial class DurableGrain : Grain, IGrainBase + public abstract partial class DurableGrain : Grain, IGrainBase, ILifecycleParticipant { protected IJournaledStateManager StateManager { get { throw null; } } @@ -219,11 +219,6 @@ public partial interface IJournaledState void Reset(JournalStreamWriter writer); } - public partial interface IJournaledGrainParticipant - { - void Initialize(); - } - public partial interface IJournaledStateManager : System.IAsyncDisposable { long PendingWriteByteCount { get; } diff --git a/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs b/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs deleted file mode 100644 index 20b92cae34b..00000000000 --- a/test/Orleans.Journaling.Tests/JournaledGrainParticipantTests.cs +++ /dev/null @@ -1,311 +0,0 @@ -using System.Collections.Concurrent; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Core.Internal; -using Orleans.TestingHost; -using Xunit; - -namespace Orleans.Journaling.Tests; - -[TestCategory("BVT")] -[TestSuite("BVT")] -[TestProvider("None")] -[TestArea("Journaling")] -public sealed class JournaledGrainParticipantTests(JournaledGrainParticipantFixture fixture) - : IClassFixture -{ - [Fact] - public async Task DurableGrain_InitializesComposedParticipantsExactlyOnceBeforeRecovery() - { - var grain = fixture.Client.GetGrain(Guid.NewGuid()); - var beforeFirstActivation = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); - - var activationId = await grain.GetActivationId(); - var afterFirstActivation = fixture.GetCounts(ComposedParticipantGrain.GrainTypeName); - AssertActivationDelta(beforeFirstActivation, afterFirstActivation); - await grain.SetValues("one", "two"); - Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); - - await grain.Cast().DeactivateOnIdle(); - - Assert.NotEqual(activationId, await grain.GetActivationId()); - Assert.Equal(new[] { "one", "two" }, await grain.GetValues()); - AssertActivationDelta( - afterFirstActivation, - fixture.GetCounts(ComposedParticipantGrain.GrainTypeName)); - - static void AssertActivationDelta(ParticipantCounts beforeActivation, ParticipantCounts afterActivation) - { - Assert.Equal( - new ParticipantCounts( - FirstConstructions: 1, - SecondConstructions: 1, - FailureConstructions: 1, - FirstInitializations: 1, - SecondInitializations: 1), - afterActivation - beforeActivation); - } - } - - [Fact] - public async Task DurableGrain_PropagatesParticipantConstructorFailure() - { - var grain = fixture.Client.GetGrain(Guid.NewGuid()); - - var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); - - Assert.Contains(ParticipantFailure.ConstructorMessage, exception.ToString(), StringComparison.Ordinal); - var counts = fixture.GetCounts(ConstructorFailureParticipantGrain.GrainTypeName); - Assert.True(counts.FailureConstructions > 0); - Assert.Equal(0, counts.FirstInitializations + counts.SecondInitializations); - } - - [Fact] - public async Task DurableGrain_PropagatesParticipantInitializeFailure() - { - var grain = fixture.Client.GetGrain(Guid.NewGuid()); - - var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); - - Assert.Contains(ParticipantFailure.InitializeMessage, exception.ToString(), StringComparison.Ordinal); - var counts = fixture.GetCounts(InitializeFailureParticipantGrain.GrainTypeName); - Assert.True(counts.FirstInitializations > 0); - Assert.Equal(counts.FirstConstructions, counts.FirstInitializations); - Assert.Equal(0, counts.SecondInitializations); - } - - [Fact] - public async Task DurableGrain_InitializesParticipantsBeforePropagatingActivationFailure() - { - var grain = fixture.Client.GetGrain(Guid.NewGuid()); - - var exception = await Assert.ThrowsAnyAsync(() => grain.Ping()); - - Assert.Contains(ParticipantFailure.ActivationMessage, exception.ToString(), StringComparison.Ordinal); - var counts = fixture.GetCounts(ActivationFailureParticipantGrain.GrainTypeName); - Assert.True(counts.FirstInitializations > 0); - Assert.Equal(counts.FirstConstructions, counts.FirstInitializations); - Assert.Equal(counts.SecondConstructions, counts.SecondInitializations); - } -} - -public sealed class JournaledGrainParticipantFixture : IntegrationTestFixture -{ - protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) - { - builder.ConfigureSilo((_, siloBuilder) => - { - siloBuilder.Services.AddSingleton(); - siloBuilder.Services.AddScoped(); - siloBuilder.Services.AddScoped(); - siloBuilder.Services.AddScoped(); - }); - } - - public ParticipantCounts GetCounts(string grainType) - { - var result = new ParticipantCounts(); - foreach (var silo in Cluster.Silos) - { - result += silo.ServiceProvider.GetRequiredService().GetCounts(grainType); - } - - return result; - } -} - -internal sealed class FirstJournaledParticipant : IJournaledGrainParticipant -{ - private readonly string _grainType; - private readonly IServiceProvider _serviceProvider; - private readonly ParticipantRecorder _recorder; - - public FirstJournaledParticipant(IGrainContext grainContext, IServiceProvider serviceProvider, ParticipantRecorder recorder) - { - _grainType = grainContext.GrainId.Type.ToString(); - _serviceProvider = serviceProvider; - _recorder = recorder; - recorder.RecordFirstConstruction(_grainType); - } - - public void Initialize() - { - _recorder.RecordFirstInitialization(_grainType); - if (_grainType == InitializeFailureParticipantGrain.GrainTypeName) - { - throw new InvalidOperationException(ParticipantFailure.InitializeMessage); - } - - _ = _serviceProvider.GetRequiredKeyedService>("participant-one"); - } -} - -internal sealed class SecondJournaledParticipant : IJournaledGrainParticipant -{ - private readonly string _grainType; - private readonly IServiceProvider _serviceProvider; - private readonly ParticipantRecorder _recorder; - - public SecondJournaledParticipant(IGrainContext grainContext, IServiceProvider serviceProvider, ParticipantRecorder recorder) - { - _grainType = grainContext.GrainId.Type.ToString(); - _serviceProvider = serviceProvider; - _recorder = recorder; - recorder.RecordSecondConstruction(_grainType); - } - - public void Initialize() - { - _recorder.RecordSecondInitialization(_grainType); - _ = _serviceProvider.GetRequiredKeyedService>("participant-two"); - } -} - -internal sealed class ParticipantFailure : IJournaledGrainParticipant -{ - public const string ConstructorMessage = "Expected participant constructor failure."; - public const string InitializeMessage = "Expected participant initialization failure."; - public const string ActivationMessage = "Expected grain activation failure."; - - public ParticipantFailure(IGrainContext grainContext, ParticipantRecorder recorder) - { - var grainType = grainContext.GrainId.Type.ToString(); - recorder.RecordFailureConstruction(grainType); - if (grainType == ConstructorFailureParticipantGrain.GrainTypeName) - { - throw new InvalidOperationException(ConstructorMessage); - } - } - - public void Initialize() - { - } -} - -public sealed class ParticipantRecorder -{ - private readonly ConcurrentDictionary _counts = new(StringComparer.Ordinal); - - public void RecordFirstConstruction(string grainType) => - _counts.AddOrUpdate(grainType, static _ => new() { FirstConstructions = 1 }, static (_, value) => value with { FirstConstructions = value.FirstConstructions + 1 }); - - public void RecordSecondConstruction(string grainType) => - _counts.AddOrUpdate(grainType, static _ => new() { SecondConstructions = 1 }, static (_, value) => value with { SecondConstructions = value.SecondConstructions + 1 }); - - public void RecordFailureConstruction(string grainType) => - _counts.AddOrUpdate(grainType, static _ => new() { FailureConstructions = 1 }, static (_, value) => value with { FailureConstructions = value.FailureConstructions + 1 }); - - public void RecordFirstInitialization(string grainType) => - _counts.AddOrUpdate(grainType, static _ => new() { FirstInitializations = 1 }, static (_, value) => value with { FirstInitializations = value.FirstInitializations + 1 }); - - public void RecordSecondInitialization(string grainType) => - _counts.AddOrUpdate(grainType, static _ => new() { SecondInitializations = 1 }, static (_, value) => value with { SecondInitializations = value.SecondInitializations + 1 }); - - public ParticipantCounts GetCounts(string grainType) => _counts.GetValueOrDefault(grainType); -} - -public readonly record struct ParticipantCounts( - int FirstConstructions = 0, - int SecondConstructions = 0, - int FailureConstructions = 0, - int FirstInitializations = 0, - int SecondInitializations = 0) -{ - public static ParticipantCounts operator +(ParticipantCounts left, ParticipantCounts right) => - new( - left.FirstConstructions + right.FirstConstructions, - left.SecondConstructions + right.SecondConstructions, - left.FailureConstructions + right.FailureConstructions, - left.FirstInitializations + right.FirstInitializations, - left.SecondInitializations + right.SecondInitializations); - - public static ParticipantCounts operator -(ParticipantCounts left, ParticipantCounts right) => - new( - left.FirstConstructions - right.FirstConstructions, - left.SecondConstructions - right.SecondConstructions, - left.FailureConstructions - right.FailureConstructions, - left.FirstInitializations - right.FirstInitializations, - left.SecondInitializations - right.SecondInitializations); -} - -[GrainType(GrainTypeName)] -public sealed class ComposedParticipantGrain : DurableGrain, IComposedParticipantGrain -{ - public const string GrainTypeName = "journaling-composed-participant"; - private readonly Guid _activationId = Guid.NewGuid(); - - public async Task SetValues(string first, string second) - { - GetValue("participant-one").Value = first; - GetValue("participant-two").Value = second; - await WriteStateAsync(); - } - - public Task GetValues() => - Task.FromResult(new[] { GetValue("participant-one").Value!, GetValue("participant-two").Value! }); - - public Task GetActivationId() => Task.FromResult(_activationId); - - private IDurableValue GetValue(string name) - { - AssertStateExists(name, StateManager.TryGetState(name, out var state)); - return state as IDurableValue - ?? throw new InvalidOperationException($"The participant state '{name}' has an unexpected type."); - - static void AssertStateExists(string name, bool exists) - { - if (!exists) - { - throw new InvalidOperationException($"The participant state '{name}' was not initialized."); - } - } - } -} - -[GrainType(GrainTypeName)] -public sealed class ConstructorFailureParticipantGrain : DurableGrain, IConstructorFailureParticipantGrain -{ - public const string GrainTypeName = "journaling-constructor-failure-participant"; - - public Task Ping() => Task.CompletedTask; -} - -[GrainType(GrainTypeName)] -public sealed class InitializeFailureParticipantGrain : DurableGrain, IInitializeFailureParticipantGrain -{ - public const string GrainTypeName = "journaling-initialize-failure-participant"; - - public Task Ping() => Task.CompletedTask; -} - -[GrainType(GrainTypeName)] -public sealed class ActivationFailureParticipantGrain : DurableGrain, IActivationFailureParticipantGrain -{ - public const string GrainTypeName = "journaling-activation-failure-participant"; - - public override Task OnActivateAsync(CancellationToken cancellationToken) => - Task.FromException(new InvalidOperationException(ParticipantFailure.ActivationMessage)); - - public Task Ping() => Task.CompletedTask; -} - -public interface IComposedParticipantGrain : IGrainWithGuidKey -{ - Task SetValues(string first, string second); - Task GetValues(); - Task GetActivationId(); -} - -public interface IConstructorFailureParticipantGrain : IGrainWithGuidKey -{ - Task Ping(); -} - -public interface IInitializeFailureParticipantGrain : IGrainWithGuidKey -{ - Task Ping(); -} - -public interface IActivationFailureParticipantGrain : IGrainWithGuidKey -{ - Task Ping(); -} From 172615d4e2bdb03cd8c43aed0089968961ebf698 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 23 Aug 2026 18:13:15 -0700 Subject: [PATCH 22/26] refactor(journaling): simplify rollback lifecycle Remove the unused observer surface, restore constructor-time lifecycle registration, collapse recovery states, and move the OrleansBinary wire fix to a focused follow-up. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: f32c3454-1184-4a8b-81c3-7a9f9b094617 --- src/Orleans.Journaling/DurableGrain.cs | 20 +- .../OrleansBinaryCommandCodecHelpers.cs | 8 - .../OrleansBinaryDurableListCommandCodec.cs | 2 +- .../OrleansBinaryDurableQueueCommandCodec.cs | 2 +- .../OrleansBinaryDurableSetCommandCodec.cs | 2 +- .../IJournaledStateManager.cs | 20 +- .../IJournaledStateObserver.cs | 58 --- .../JournaledStateManager.cs | 122 +---- src/Orleans.Journaling/README.md | 4 - .../Orleans.Journaling/Orleans.Journaling.cs | 11 +- .../DurableListJournalBenchmarks.cs | 2 - .../DurableCollectionDirectWriteTests.cs | 2 - .../DurableListDirectWriteTests.cs | 2 - .../OrleansBinaryCommandCodecTests.cs | 63 --- .../StateManagerTests.cs | 465 +----------------- 15 files changed, 25 insertions(+), 758 deletions(-) delete mode 100644 src/Orleans.Journaling/IJournaledStateObserver.cs diff --git a/src/Orleans.Journaling/DurableGrain.cs b/src/Orleans.Journaling/DurableGrain.cs index a0c236181e2..7ffec5d7c2a 100644 --- a/src/Orleans.Journaling/DurableGrain.cs +++ b/src/Orleans.Journaling/DurableGrain.cs @@ -2,23 +2,19 @@ namespace Orleans.Journaling; -public abstract class DurableGrain : Grain, IGrainBase, ILifecycleParticipant +public abstract class DurableGrain : Grain, IGrainBase { protected DurableGrain() { StateManager = ServiceProvider.GetRequiredService(); - } - - protected IJournaledStateManager StateManager { get; } - - void ILifecycleParticipant.Participate(IGrainLifecycle lifecycle) - { if (StateManager is ILifecycleParticipant participant) { - participant.Participate(lifecycle); + participant.Participate(((IGrainBase)this).GrainContext.ObservableLifecycle); } } + protected IJournaledStateManager StateManager { get; } + protected TState GetOrCreateState(string name) where TState : class, IJournaledState => GetOrCreateState(name, static sp => sp.GetRequiredService(), ServiceProvider); @@ -35,11 +31,5 @@ protected TState GetOrCreateState(string name, Func return result; } - /// - /// Writes pending journaled state. - /// - /// Cancellation token. - /// A task which completes when the state is written. - protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => - StateManager.WriteStateAsync(cancellationToken); + protected ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => StateManager.WriteStateAsync(cancellationToken); } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs index 3a847a5ff7e..5b1e4757fe5 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryCommandCodecHelpers.cs @@ -35,12 +35,4 @@ public static T ReadValue(IFieldCodec codec, ref Reader re reader.Session.Reset(); } } - - public static T ReadIndependentValue(IFieldCodec codec, ref Reader reader) - { - var result = ReadValue(codec, ref reader); - // Snapshot values are each encoded with a fresh serializer session. - reader.Session.Reset(); - return result; - } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs index a68d48d0c44..ba38edf008a 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableListCommandCodec.cs @@ -156,7 +156,7 @@ private void ApplySnapshot(ref Reader reader, IDurableListComman consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); + consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs index e9882426068..2ab6b34fcfb 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableQueueCommandCodec.cs @@ -113,7 +113,7 @@ private void ApplySnapshot(ref Reader reader, IDurableQueueComma consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyEnqueue(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); + consumer.ApplyEnqueue(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); } } diff --git a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs index a938b18fb58..8b8585b13a3 100644 --- a/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs +++ b/src/Orleans.Journaling/Formats/OrleansBinary/OrleansBinaryDurableSetCommandCodec.cs @@ -115,7 +115,7 @@ private void ApplySnapshot(ref Reader reader, IDurableSetCommand consumer.Reset(count); for (var i = 0; i < count; i++) { - consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadIndependentValue(codec, ref reader)); + consumer.ApplyAdd(OrleansBinaryCommandCodecHelpers.ReadValue(codec, ref reader)); } } } diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index bb12a59cf72..2861d6e3deb 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -24,19 +24,6 @@ public interface IJournaledStateManager : IAsyncDisposable /// The state instance to register. void RegisterState(string name, IJournaledState state); - /// - /// Registers an observer for durable write and recovery notifications. - /// - /// The observer. - /// - /// Each operation uses a stable snapshot of the registered observers. An observer registered - /// from a callback participates in subsequent operations, not later phases of the current one. - /// Implementations enable observer support by overriding this method. The default implementation - /// throws . - /// - void RegisterObserver(IJournaledStateObserver observer) => - throw new NotSupportedException("Observer registration requires a state manager implementation which supports observers."); - /// /// Attempts to get a state registered with the manager. /// @@ -56,12 +43,7 @@ void RegisterObserver(IJournaledStateObserver observer) => /// /// The cancellation token. /// A which represents the operation. - /// - /// Implementations enable rollback support by overriding this method. The default implementation - /// throws . - /// - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => - ValueTask.FromException(new NotSupportedException("Rollback requires a state manager implementation which supports rollback.")); + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); /// /// Resets this instance, removing any persistent state. diff --git a/src/Orleans.Journaling/IJournaledStateObserver.cs b/src/Orleans.Journaling/IJournaledStateObserver.cs deleted file mode 100644 index 5081fb8feee..00000000000 --- a/src/Orleans.Journaling/IJournaledStateObserver.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; - -namespace Orleans.Journaling; - -/// -/// Observes durable state manager commit and recovery boundaries. -/// -/// -/// -/// The manager awaits , which can perform short, -/// cancellation-aware asynchronous preparation before state capture. The remaining callbacks run -/// synchronously on the owning grain scheduler and must return promptly. Implementations should -/// queue asynchronous follow-up work from those synchronous callbacks. -/// -/// -/// Before each write, the manager invokes immediately before capturing -/// registered states. Mutations remain provisional until marks a -/// successful commit. Restoring the last durable state, including through -/// , invokes -/// after all registered states have been restored. -/// -/// Exceptions from abort the write before state is captured. -/// Exceptions from the synchronous notification methods are logged and don't change the outcome -/// of a completed commit or recovery. -/// Each operation uses one stable snapshot of registered observers for all of its callbacks. -/// Observers registered from a callback begin participating in subsequent operations. -/// -public interface IJournaledStateObserver -{ - /// - /// Prepares external prerequisites for a durable write before registered states are captured. - /// - /// The cancellation token. - /// A task representing the preparation operation. - /// - /// State mutated by this method is included in the pending write. Throwing aborts the write. - /// Implementations must be idempotent because a failed storage write can be retried. - /// - ValueTask OnWritePreparingAsync(CancellationToken cancellationToken); - - /// - /// Called immediately before registered states are captured for a durable write. - /// Mutations made after this callback aren't included in the corresponding - /// boundary. - /// - void OnWriteStarted(); - - /// - /// Called after all pending journal entries have been committed to stable storage. - /// - void OnWriteCompleted(); - - /// - /// Called after all registered states have been restored to the last durable version. - /// - void OnRecoveryCompleted(); -} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 52d5241302e..1adacb68df9 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -19,7 +19,6 @@ internal sealed partial class JournaledStateManager : IJournaledStateManager, IJ #endif private readonly Dictionary _states = new(StringComparer.Ordinal); private readonly Dictionary _statesMap = []; - private readonly HashSet _observers = []; private readonly JournaledStateManagerShared _shared; private readonly IJournalStorage _storage; private readonly JournalBufferWriter _journalWriter; @@ -131,19 +130,6 @@ public void RegisterState(string name, IJournaledState state) _workSignal.Signal(); } - public void RegisterObserver(IJournaledStateObserver observer) - { - ArgumentNullException.ThrowIfNull(observer); - lock (_lock) - { - _shutdownCancellation.Token.ThrowIfCancellationRequested(); - if (!_observers.Add(observer)) - { - throw new InvalidOperationException("The journaled state observer is already registered."); - } - } - } - public async ValueTask InitializeAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -208,7 +194,7 @@ private async Task WorkLoop() { if (fenceOnFailure) { - _state = ManagerState.RecoveryPending; + _state = ManagerState.Ready; } } @@ -295,29 +281,6 @@ private async Task WorkLoop() var hasBufferToConsume = false; var bufferToConsumeIsCommittedBuffer = false; - IJournaledStateObserver[] observers; - lock (_lock) - { - observers = [.. _observers]; - } - - foreach (var observer in observers) - { - await observer.OnWritePreparingAsync(_shutdownCancellation.Token).ConfigureAwait(true); - } - - foreach (var observer in observers) - { - try - { - observer.OnWriteStarted(); - } - catch (Exception exception) - { - LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteStarted)); - } - } - lock (_lock) { if (isSnapshot) @@ -434,7 +397,6 @@ private async Task WorkLoop() writeSequence = new ReadOnlySequence(debugPoisonBuffer, 0, debugPoisonLength); #endif - var writeCompleted = false; try { if (isSnapshot) @@ -446,35 +408,25 @@ private async Task WorkLoop() await AppendStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true); } - writeCompleted = true; - } - finally - { - try + lock (_lock) { - if (writeCompleted) + if (hasBufferToConsume) { - lock (_lock) - { - if (hasBufferToConsume) - { - _journalWriter.Consume(bufferToConsume); - } - } + _journalWriter.Consume(bufferToConsume); } } - finally + } + finally + { + committedBuffer.Dispose(); + if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer) { - committedBuffer.Dispose(); - if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer) - { - bufferToConsume.Dispose(); - } + bufferToConsume.Dispose(); + } #if DEBUG - debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67); - ArrayPool.Shared.Return(debugPoisonBuffer); + debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67); + ArrayPool.Shared.Return(debugPoisonBuffer); #endif - } } // Notify all states that the operation completed. @@ -482,14 +434,7 @@ private async Task WorkLoop() { foreach (var state in _states.Values) { - try - { - state.OnWriteCompleted(); - } - catch (Exception exception) - { - LogStateCallbackError(_shared.Logger, exception, state.GetType().FullName ?? state.GetType().Name); - } + state.OnWriteCompleted(); } if (isSnapshot) @@ -499,18 +444,6 @@ private async Task WorkLoop() } } - foreach (var observer in observers) - { - try - { - observer.OnWriteCompleted(); - } - catch (Exception exception) - { - LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnWriteCompleted)); - } - } - break; } @@ -791,10 +724,8 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) private async Task RecoverAsync(CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); - IJournaledStateObserver[] observers; lock (_lock) { - observers = [.. _observers]; ResetForRecovery(); } @@ -824,19 +755,6 @@ private async Task RecoverAsync(CancellationToken cancellationToken) } } } - - } - - foreach (var observer in observers) - { - try - { - observer.OnRecoveryCompleted(); - } - catch (Exception exception) - { - LogObserverError(_shared.Logger, exception, nameof(IJournaledStateObserver.OnRecoveryCompleted)); - } } } @@ -1015,7 +933,6 @@ private void ThrowIfWriteOperationsUnavailable() case ManagerState.Unknown: throw new InvalidOperationException("The journaled state manager has not been initialized."); case ManagerState.Ready: - case ManagerState.RecoveryPending: return; case ManagerState.Recovering: throw new InvalidOperationException("Journaled state writes are unavailable while recovery is in progress."); @@ -1295,7 +1212,6 @@ private enum ManagerState : byte { Unknown, Ready, - RecoveryPending, Recovering, Fenced } @@ -1432,16 +1348,6 @@ void IJournaledState.AppendEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); - [LoggerMessage( - Level = LogLevel.Error, - Message = "Journaled state observer callback {Callback} failed.")] - private static partial void LogObserverError(ILogger logger, Exception exception, string callback); - - [LoggerMessage( - Level = LogLevel.Error, - Message = "Journaled state {StateType} failed its write-completion callback.")] - private static partial void LogStateCallbackError(ILogger logger, Exception exception, string stateType); - [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index e44540c031e..c02bb4328ba 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -65,10 +65,6 @@ siloBuilder To migrate to JSON, configure `JournaledStateManagerOptions.JournalFormatKey` to `JsonJournalExtensions.JournalFormatKey` and call `UseJsonJournalFormat(...)`. When a grain recovers data written with a different format than the configured write format, the next write is forced to a full snapshot so the journal is rewritten using JSON and the storage format metadata is updated. -Journaled state observers are snapshotted once at the start of each write or recovery -operation. An observer registered from inside a callback begins participating in the -next operation, so every phase of the current operation sees the same observer set. - ## Example - Using durable states ```csharp using Microsoft.Extensions.DependencyInjection; diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index b936c5222e5..e7fcd7aa32d 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -8,7 +8,7 @@ //------------------------------------------------------------------------------ namespace Orleans.Journaling { - public abstract partial class DurableGrain : Grain, IGrainBase, ILifecycleParticipant + public abstract partial class DurableGrain : Grain, IGrainBase { protected IJournaledStateManager StateManager { get { throw null; } } @@ -225,7 +225,6 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken); - void RegisterObserver(IJournaledStateObserver observer); void RegisterState(string name, IJournaledState state); System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); @@ -238,14 +237,6 @@ public partial interface IJournaledStateManagerFactory IJournaledStateManager Create(JournalId journalId); } - public partial interface IJournaledStateObserver - { - void OnRecoveryCompleted(); - void OnWriteCompleted(); - System.Threading.Tasks.ValueTask OnWritePreparingAsync(System.Threading.CancellationToken cancellationToken); - void OnWriteStarted(); - } - public partial interface IJournalFormat { string FormatKey { get; } diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index 02e03061f10..3cdc999687e 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -130,8 +130,6 @@ private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter bu public void RegisterState(string name, IJournaledState state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); - public void RegisterObserver(IJournaledStateObserver observer) { } - public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState state) { state = null!; diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 0194fa182fb..2d36e8c9a2a 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -197,8 +197,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn public void RegisterState(string name, IJournaledState state) => state.Reset(writer.CreateWriter()); - public void RegisterObserver(IJournaledStateObserver observer) { } - public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) { state = null; diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 37260a15490..134dc3d4e42 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -96,8 +96,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn public void RegisterState(string name, IJournaledState state) => state.Reset(writer.CreateWriter()); - public void RegisterObserver(IJournaledStateObserver observer) { } - public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) { state = null; diff --git a/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs b/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs index 4623248fcfc..8e4df708967 100644 --- a/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs +++ b/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs @@ -618,67 +618,4 @@ private static void AssertTrailingDataRejected(Action action) var exception = Assert.Throws(action); Assert.Contains("trailing data", exception.Message); } - - private static SnapshotReferenceRecord[] CreateReferenceItems() - { - var first = new byte[] { 1, 2, 3 }; - var second = new byte[] { 4, 5, 6 }; - return - [ - new() { Payload = first, Alias = first }, - new() { Payload = second, Alias = second } - ]; - } - - private static void AssertAliases(IEnumerable items) - { - Assert.Collection( - items, - item => - { - Assert.Equal([1, 2, 3], item.Payload); - Assert.Same(item.Payload, item.Alias); - }, - item => - { - Assert.Equal([4, 5, 6], item.Payload); - Assert.Same(item.Payload, item.Alias); - }); - } -} - -[GenerateSerializer] -internal sealed class SnapshotReferenceRecord -{ - [Id(0)] - public required byte[] Payload { get; init; } - - [Id(1)] - public required byte[] Alias { get; init; } -} - -internal sealed class SnapshotCollectionConsumer : - IDurableListCommandHandler, - IDurableQueueCommandHandler, - IDurableSetCommandHandler -{ - public List Items { get; } = []; - - public void ApplyAdd(SnapshotReferenceRecord item) => Items.Add(item); - - public void ApplySet(int index, SnapshotReferenceRecord item) => throw new NotSupportedException(); - - public void ApplyInsert(int index, SnapshotReferenceRecord item) => throw new NotSupportedException(); - - public void ApplyRemoveAt(int index) => throw new NotSupportedException(); - - public void ApplyClear() => Items.Clear(); - - public void Reset(int capacityHint) => Items.Clear(); - - public void ApplyEnqueue(SnapshotReferenceRecord item) => Items.Add(item); - - public void ApplyDequeue() => throw new NotSupportedException(); - - public void ApplyRemove(SnapshotReferenceRecord item) => throw new NotSupportedException(); } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index d58cd20bd4f..5f48213173b 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -134,8 +134,6 @@ await Assert.ThrowsAnyAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); await Assert.ThrowsAnyAsync( () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); - Assert.ThrowsAny( - () => sut.Manager.RegisterObserver(new RecordingStateObserver())); } [Fact] @@ -670,285 +668,6 @@ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() Assert.Equal(1, value.Value); } - [Fact] - public async Task StateManager_Observer_NotifiesOnceAfterWholeRecoveryAndCommit() - { - var sut = CreateTestSystem(); - _ = new DurableDictionary("first", sut.Manager, CreateDictionaryCodec()); - var second = new DurableDictionary("second", sut.Manager, CreateDictionaryCodec()); - var observer = new RecordingStateObserver(); - sut.Manager.RegisterObserver(observer); - - await sut.Lifecycle.OnStart(); - second["value"] = 1; - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal(1, observer.RecoveryCompletedCount); - Assert.Equal(1, observer.WritePreparingCount); - Assert.Equal(1, observer.WriteStartedCount); - Assert.Equal(1, observer.WriteCompletedCount); - Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); - } - - [Fact] - public void StateManager_DefaultRegisterObserver_PreservesLegacyImplementations() - { - IJournaledStateManager manager = new LegacyStateManager(); - - var exception = Assert.Throws( - () => manager.RegisterObserver(new RecordingStateObserver())); - - Assert.Contains("Observer registration requires", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task StateManager_DefaultRevertPendingChanges_PreservesLegacyImplementations() - { - IJournaledStateManager manager = new LegacyStateManager(); - - var exception = await Assert.ThrowsAsync( - () => manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); - - Assert.Contains("Rollback requires", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task StateManager_AwaitsObserverPreparationBeforeCapturingState() - { - var storage = new CapturingStorage(); - var sut = CreateTestSystem(storage: storage); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - var observer = new BlockingPreparingStateObserver(() => value.Value = 42); - sut.Manager.RegisterObserver(observer); - await sut.Lifecycle.OnStart(); - - var write = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); - await observer.PreparingStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); - - Assert.False(write.IsCompleted); - Assert.Empty(storage.Appends); - - observer.AllowPreparation.SetResult(); - await write.WaitAsync(TimeSpan.FromSeconds(10)); - - Assert.Single(storage.Appends); - var recovered = CreateTestSystem(storage: storage); - var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); - await recovered.Lifecycle.OnStart(); - Assert.Equal(42, recoveredValue.Value); - } - - [Fact] - public async Task StateManager_Observer_FailedWriteDoesNotCommitAndRevertNotifiesRecovery() - { - var storage = new CapturingStorage(); - var sut = CreateTestSystem(storage: storage); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - var observer = new RecordingStateObserver(); - sut.Manager.RegisterObserver(observer); - - await sut.Lifecycle.OnStart(); - value.Value = 1; - await sut.Manager.WriteStateAsync(CancellationToken.None); - storage.NextAppendException = new IOException("Expected write failure."); - value.Value = 2; - - await Assert.ThrowsAsync( - () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Equal(2, observer.WriteStartedCount); - Assert.Equal(1, observer.WriteCompletedCount); - - await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); - - Assert.Equal(1, value.Value); - Assert.Equal(2, observer.RecoveryCompletedCount); - Assert.Equal(2, observer.WriteStartedCount); - Assert.Equal(1, observer.WriteCompletedCount); - } - - [Fact] - public async Task StateManager_ObserverException_DoesNotChangeCompletedOperationOutcome() - { - var sut = CreateTestSystem(); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - sut.Manager.RegisterObserver(new ThrowingStateObserver()); - - await sut.Lifecycle.OnStart(); - value.Value = 1; - await sut.Manager.WriteStateAsync(CancellationToken.None); - value.Value = 2; - await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); - - Assert.Equal(1, value.Value); - } - - [Fact] - public async Task StateManager_Observer_NoOpWriteStillPairsStartAndCompletion() - { - var sut = CreateTestSystem(); - await sut.Lifecycle.OnStart(); - var observer = new RecordingStateObserver(); - sut.Manager.RegisterObserver(observer); - - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal(1, observer.WritePreparingCount); - Assert.Equal(1, observer.WriteStartedCount); - Assert.Equal(1, observer.WriteCompletedCount); - Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); - } - - [Theory] - [InlineData(false, ObserverRegistrationPhase.Preparing)] - [InlineData(false, ObserverRegistrationPhase.Started)] - [InlineData(false, ObserverRegistrationPhase.Completed)] - [InlineData(true, ObserverRegistrationPhase.Preparing)] - [InlineData(true, ObserverRegistrationPhase.Started)] - [InlineData(true, ObserverRegistrationPhase.Completed)] - public async Task StateManager_ObserverRegisteredDuringWritePhase_JoinsNextWrite( - bool hasPendingState, - ObserverRegistrationPhase registrationPhase) - { - var sut = CreateTestSystem(); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - var lateObserver = new RecordingStateObserver(); - var registeringObserver = new RegisteringStateObserver( - onWritePreparing: RegisterDuring(ObserverRegistrationPhase.Preparing), - onWriteStarted: RegisterDuring(ObserverRegistrationPhase.Started), - onWriteCompleted: RegisterDuring(ObserverRegistrationPhase.Completed)); - sut.Manager.RegisterObserver(registeringObserver); - await sut.Lifecycle.OnStart(); - if (hasPendingState) - { - value.Value = 1; - } - - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal(["Preparing", "Started", "Completed"], registeringObserver.WriteCalls); - Assert.Empty(lateObserver.WriteCalls); - - if (hasPendingState) - { - value.Value = 2; - } - - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal( - ["Preparing", "Started", "Completed", "Preparing", "Started", "Completed"], - registeringObserver.WriteCalls); - Assert.Equal(["Preparing", "Started", "Completed"], lateObserver.WriteCalls); - - Action? RegisterDuring(ObserverRegistrationPhase phase) => - registrationPhase == phase ? () => sut.Manager.RegisterObserver(lateObserver) : null; - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task StateManager_Observer_WritePhasesAreConsistentForAppendAndSnapshot(bool isSnapshot) - { - var storage = new CapturingStorage { IsCompactionRequested = isSnapshot }; - var sut = CreateTestSystem(storage: storage); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - var observer = new RecordingStateObserver(); - sut.Manager.RegisterObserver(observer); - await sut.Lifecycle.OnStart(); - value.Value = 1; - - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal(["Preparing", "Started", "Completed"], observer.WriteCalls); - if (isSnapshot) - { - Assert.Single(storage.Replaces); - Assert.Empty(storage.Appends); - } - else - { - Assert.Single(storage.Appends); - Assert.Empty(storage.Replaces); - } - } - - [Fact] - public async Task StateManager_ObserverRegisteredDuringRecovery_JoinsNextRecovery() - { - var sut = CreateTestSystem(); - var lateObserver = new RecordingStateObserver(); - var registeringObserver = new RegisteringStateObserver( - onRecoveryCompleted: () => sut.Manager.RegisterObserver(lateObserver)); - sut.Manager.RegisterObserver(registeringObserver); - - await sut.Lifecycle.OnStart(); - - Assert.Equal(1, registeringObserver.RecoveryCompletedCount); - Assert.Equal(0, lateObserver.RecoveryCompletedCount); - - await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); - - Assert.Equal(2, registeringObserver.RecoveryCompletedCount); - Assert.Equal(1, lateObserver.RecoveryCompletedCount); - } - - [Fact] - public async Task StateManager_ObserverRegisteredByStateDuringRecovery_JoinsNextRecovery() - { - var sut = CreateTestSystem(); - var lateObserver = new RecordingStateObserver(); - sut.Manager.RegisterState( - "registering", - new RecoveryRegisteringState(() => sut.Manager.RegisterObserver(lateObserver))); - - await sut.Lifecycle.OnStart(); - - Assert.Equal(0, lateObserver.RecoveryCompletedCount); - - await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); - - Assert.Equal(1, lateObserver.RecoveryCompletedCount); - } - - [Fact] - public async Task StateManager_PreparationFailureAbortsCaptureAndRetryPersistsPreparedState() - { - var storage = new CapturingStorage(); - var sut = CreateTestSystem(storage: storage); - var value = new DurableValue("value", sut.Manager, CreateValueCodec()); - var observer = new PreparingStateObserver(() => value.Value = "owned"); - sut.Manager.RegisterObserver(observer); - await sut.Lifecycle.OnStart(); - - observer.FailNext = true; - await Assert.ThrowsAsync( - () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - - Assert.Empty(storage.Appends); - Assert.Equal(0, observer.WriteStartedCount); - - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Single(storage.Appends); - Assert.Equal("owned", value.Value); - Assert.Equal(2, observer.PreparingCount); - Assert.Equal(1, observer.WriteStartedCount); - } - - [Fact] - public async Task StateManager_StateCompletionFailure_DoesNotSkipCommitObserver() - { - var sut = CreateTestSystem(); - sut.Manager.RegisterState("throwing", new AlwaysWritingState { ThrowOnWriteCompleted = true }); - var observer = new RecordingStateObserver(); - sut.Manager.RegisterObserver(observer); - - await sut.Lifecycle.OnStart(); - await sut.Manager.WriteStateAsync(CancellationToken.None); - - Assert.Equal(1, observer.WriteCompletedCount); - } - [Fact] public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfully() { @@ -1682,167 +1401,6 @@ static async Task TriggerCompaction(IJournaledStateManager manager, DurableDicti } } - private sealed class LegacyStateManager : IJournaledStateManager - { - public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; - - public void RegisterState(string name, IJournaledState state) - { - } - - public bool TryGetState( - string name, - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IJournaledState? state) - { - state = null; - return false; - } - - public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; - } - - private sealed class RecordingStateObserver : IJournaledStateObserver - { - public List WriteCalls { get; } = []; - public int WritePreparingCount { get; private set; } - public int WriteStartedCount { get; private set; } - public int WriteCompletedCount { get; private set; } - public int RecoveryCompletedCount { get; private set; } - - public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) - { - WriteCalls.Add("Preparing"); - WritePreparingCount++; - return default; - } - - public void OnWriteStarted() - { - WriteCalls.Add("Started"); - WriteStartedCount++; - } - - public void OnWriteCompleted() - { - WriteCalls.Add("Completed"); - WriteCompletedCount++; - } - - public void OnRecoveryCompleted() => RecoveryCompletedCount++; - } - - private sealed class ThrowingStateObserver : IJournaledStateObserver - { - public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) => default; - public void OnWriteStarted() => throw new InvalidOperationException("Expected observer failure."); - public void OnWriteCompleted() => throw new InvalidOperationException("Expected observer failure."); - public void OnRecoveryCompleted() => throw new InvalidOperationException("Expected observer failure."); - } - - private sealed class RegisteringStateObserver( - Action? onWritePreparing = null, - Action? onWriteStarted = null, - Action? onWriteCompleted = null, - Action? onRecoveryCompleted = null) : IJournaledStateObserver - { - private Action? _onWritePreparing = onWritePreparing; - private Action? _onWriteStarted = onWriteStarted; - private Action? _onWriteCompleted = onWriteCompleted; - private Action? _onRecoveryCompleted = onRecoveryCompleted; - - public List WriteCalls { get; } = []; - public int RecoveryCompletedCount { get; private set; } - - public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) - { - WriteCalls.Add("Preparing"); - Interlocked.Exchange(ref _onWritePreparing, null)?.Invoke(); - return default; - } - - public void OnWriteStarted() - { - WriteCalls.Add("Started"); - Interlocked.Exchange(ref _onWriteStarted, null)?.Invoke(); - } - - public void OnWriteCompleted() - { - WriteCalls.Add("Completed"); - Interlocked.Exchange(ref _onWriteCompleted, null)?.Invoke(); - } - - public void OnRecoveryCompleted() - { - RecoveryCompletedCount++; - Interlocked.Exchange(ref _onRecoveryCompleted, null)?.Invoke(); - } - } - - public enum ObserverRegistrationPhase - { - Preparing, - Started, - Completed - } - - private sealed class PreparingStateObserver(Action prepare) : IJournaledStateObserver - { - public bool FailNext { get; set; } - public int PreparingCount { get; private set; } - public int WriteStartedCount { get; private set; } - - public ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) - { - PreparingCount++; - prepare(); - if (FailNext) - { - FailNext = false; - throw new IOException("Expected preparation failure."); - } - - return default; - } - - public void OnWriteStarted() => WriteStartedCount++; - public void OnWriteCompleted() - { - } - - public void OnRecoveryCompleted() - { - } - } - - private sealed class BlockingPreparingStateObserver(Action prepare) : IJournaledStateObserver - { - public TaskCompletionSource PreparingStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public TaskCompletionSource AllowPreparation { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - - public async ValueTask OnWritePreparingAsync(CancellationToken cancellationToken) - { - PreparingStarted.SetResult(); - await AllowPreparation.Task.WaitAsync(cancellationToken); - prepare(); - } - - public void OnWriteStarted() - { - } - - public void OnWriteCompleted() - { - } - - public void OnRecoveryCompleted() - { - } - } - private sealed class StreamingOnlyStorage : IJournalStorage { public bool StreamingReadCalled { get; private set; } @@ -2616,8 +2174,6 @@ private sealed class AlwaysWritingState : IJournaledState public int WriteCompletedCount { get; private set; } - public bool ThrowOnWriteCompleted { get; init; } - void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) { } public void Reset(JournalStreamWriter writer) { } @@ -2633,30 +2189,11 @@ public void AppendEntries(JournalStreamWriter writer) public void AppendSnapshot(JournalStreamWriter writer) => AppendEntries(writer); - public void OnWriteCompleted() - { - WriteCompletedCount++; - if (ThrowOnWriteCompleted) - { - throw new InvalidOperationException("Expected state completion failure."); - } - } + public void OnWriteCompleted() => WriteCompletedCount++; public IJournaledState DeepCopy() => throw new NotSupportedException(); } - private sealed class RecoveryRegisteringState(Action onRecoveryCompleted) : IJournaledState - { - private Action? _onRecoveryCompleted = onRecoveryCompleted; - - public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } - public void Reset(JournalStreamWriter writer) { } - public void AppendEntries(JournalStreamWriter writer) { } - public void AppendSnapshot(JournalStreamWriter writer) { } - public void OnRecoveryCompleted() => Interlocked.Exchange(ref _onRecoveryCompleted, null)?.Invoke(); - public IJournaledState DeepCopy() => throw new NotSupportedException(); - } - private sealed class ThrowingDictionarySetCodec : IDurableDictionaryCommandCodec where K : notnull { public void WriteSet(K key, V value, JournalStreamWriter writer) From 3cb26cc5a85aea8813b437d687de0e74f475890b Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 08:24:28 -0700 Subject: [PATCH 23/26] fix(journaling): preserve state manager compatibility --- src/Orleans.Journaling/IJournaledStateManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 2861d6e3deb..1c911246780 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -43,7 +43,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The cancellation token. /// A which represents the operation. - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) + => throw new NotSupportedException($"{nameof(IJournaledStateManager)} implementation does not support reverting pending changes."); /// /// Resets this instance, removing any persistent state. From 57e80c66b8cd030f7ae9a1019cc183361f8b6213 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 09:04:33 -0700 Subject: [PATCH 24/26] test(journaling): rely on revert default behavior --- test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs | 2 -- .../DurableCollectionDirectWriteTests.cs | 2 -- test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs | 2 -- 3 files changed, 6 deletions(-) diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index 3cdc999687e..bb5c47cc049 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,8 +138,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 2d36e8c9a2a..174b160d496 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,8 +205,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 134dc3d4e42..2b2e90641d3 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,8 +104,6 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } From 4e8eafe5cbddec4b485c6c8e16e36367d5914cd8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 15:00:41 -0700 Subject: [PATCH 25/26] refactor(journaling): require rollback support --- .../IJournaledStateManager.cs | 3 +- .../DurableListJournalBenchmarks.cs | 2 + .../DurableCollectionDirectWriteTests.cs | 2 + .../DurableListDirectWriteTests.cs | 2 + .../OrleansBinaryCommandCodecTests.cs | 63 +++++++++++++++++++ 5 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 1c911246780..2861d6e3deb 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -43,8 +43,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The cancellation token. /// A which represents the operation. - ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) - => throw new NotSupportedException($"{nameof(IJournaledStateManager)} implementation does not support reverting pending changes."); + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); /// /// Resets this instance, removing any persistent state. diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index bb5c47cc049..3cdc999687e 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,6 +138,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 174b160d496..2d36e8c9a2a 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,6 +205,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 2b2e90641d3..134dc3d4e42 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,6 +104,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs b/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs index 8e4df708967..4623248fcfc 100644 --- a/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs +++ b/test/Orleans.Journaling.Tests/OrleansBinaryCommandCodecTests.cs @@ -618,4 +618,67 @@ private static void AssertTrailingDataRejected(Action action) var exception = Assert.Throws(action); Assert.Contains("trailing data", exception.Message); } + + private static SnapshotReferenceRecord[] CreateReferenceItems() + { + var first = new byte[] { 1, 2, 3 }; + var second = new byte[] { 4, 5, 6 }; + return + [ + new() { Payload = first, Alias = first }, + new() { Payload = second, Alias = second } + ]; + } + + private static void AssertAliases(IEnumerable items) + { + Assert.Collection( + items, + item => + { + Assert.Equal([1, 2, 3], item.Payload); + Assert.Same(item.Payload, item.Alias); + }, + item => + { + Assert.Equal([4, 5, 6], item.Payload); + Assert.Same(item.Payload, item.Alias); + }); + } +} + +[GenerateSerializer] +internal sealed class SnapshotReferenceRecord +{ + [Id(0)] + public required byte[] Payload { get; init; } + + [Id(1)] + public required byte[] Alias { get; init; } +} + +internal sealed class SnapshotCollectionConsumer : + IDurableListCommandHandler, + IDurableQueueCommandHandler, + IDurableSetCommandHandler +{ + public List Items { get; } = []; + + public void ApplyAdd(SnapshotReferenceRecord item) => Items.Add(item); + + public void ApplySet(int index, SnapshotReferenceRecord item) => throw new NotSupportedException(); + + public void ApplyInsert(int index, SnapshotReferenceRecord item) => throw new NotSupportedException(); + + public void ApplyRemoveAt(int index) => throw new NotSupportedException(); + + public void ApplyClear() => Items.Clear(); + + public void Reset(int capacityHint) => Items.Clear(); + + public void ApplyEnqueue(SnapshotReferenceRecord item) => Items.Add(item); + + public void ApplyDequeue() => throw new NotSupportedException(); + + public void ApplyRemove(SnapshotReferenceRecord item) => throw new NotSupportedException(); } From d4b8af3f0a4c17be767a52a2c247f0cea9969bed Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 24 Aug 2026 15:28:11 -0700 Subject: [PATCH 26/26] fix(journaling): clarify unsupported state operations --- src/Orleans.Journaling/JournaledStateManager.cs | 10 +++++----- .../Journaling/DurableListJournalBenchmarks.cs | 2 +- .../DurableCollectionDirectWriteTests.cs | 2 +- .../DurableListDirectWriteTests.cs | 2 +- test/Orleans.Journaling.Tests/StateManagerTests.cs | 10 ++++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 1adacb68df9..658e3594695 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -699,7 +699,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) bool didEnqueue; lock (_lock) { - ThrowIfWriteOperationsUnavailable(); + ThrowIfStateOperationsUnavailable(); task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -874,7 +874,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) string operation; lock (_lock) { - ThrowIfWriteOperationsUnavailable(); + ThrowIfStateOperationsUnavailable(); var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -925,7 +925,7 @@ public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationT await pendingRecovery.WaitAsync(cancellationToken); } - private void ThrowIfWriteOperationsUnavailable() + private void ThrowIfStateOperationsUnavailable() { _shutdownCancellation.Token.ThrowIfCancellationRequested(); switch (_state) @@ -935,9 +935,9 @@ private void ThrowIfWriteOperationsUnavailable() case ManagerState.Ready: return; case ManagerState.Recovering: - throw new InvalidOperationException("Journaled state writes are unavailable while recovery is in progress."); + throw new InvalidOperationException("Journaled state operations are unavailable while recovery is in progress."); case ManagerState.Fenced: - throw new InvalidOperationException("Journaled state writes are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); + throw new InvalidOperationException("Journaled state operations are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); default: throw new UnreachableException(); } diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index 3cdc999687e..5a7a602c620 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,7 +138,7 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 2d36e8c9a2a..bdd53841f81 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,7 +205,7 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 134dc3d4e42..5dff6e54cd0 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,7 +104,7 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; - public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 5f48213173b..ea45acdbdeb 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -669,7 +669,7 @@ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() } [Fact] - public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfully() + public async Task StateManager_FencesStateOperationsUntilFailedRevertIsRetriedSuccessfully() { var storage = new CapturingStorage(); var sut = CreateTestSystem(storage: storage); @@ -688,10 +688,12 @@ public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfull var writeException = await Assert.ThrowsAsync( () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("state operations are fenced", writeException.Message, StringComparison.Ordinal); Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase); Assert.Contains("Call RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); var deleteException = await Assert.ThrowsAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("state operations are fenced", deleteException.Message, StringComparison.Ordinal); Assert.Contains("Call RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal); await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); @@ -703,7 +705,7 @@ public async Task StateManager_FencesWritesUntilFailedRevertIsRetriedSuccessfull } [Fact] - public async Task StateManager_WriteOperations_ReportRecoveryInProgress() + public async Task StateManager_StateOperations_ReportRecoveryInProgress() { var storage = new BlockingRecoveryStorage(); var sut = CreateTestSystem(storage: storage); @@ -717,8 +719,8 @@ public async Task StateManager_WriteOperations_ReportRecoveryInProgress() var deleteException = await Assert.ThrowsAsync( () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); - Assert.Contains("recovery is in progress", writeException.Message, StringComparison.Ordinal); - Assert.Contains("recovery is in progress", deleteException.Message, StringComparison.Ordinal); + Assert.Contains("state operations are unavailable while recovery is in progress", writeException.Message, StringComparison.Ordinal); + Assert.Contains("state operations are unavailable while recovery is in progress", deleteException.Message, StringComparison.Ordinal); Assert.DoesNotContain("RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); Assert.DoesNotContain("RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal);