diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs
index cc8fd7af3eb..2861d6e3deb 100644
--- a/src/Orleans.Journaling/IJournaledStateManager.cs
+++ b/src/Orleans.Journaling/IJournaledStateManager.cs
@@ -38,6 +38,13 @@ public interface IJournaledStateManager : IAsyncDisposable
/// A which represents the operation.
ValueTask WriteStateAsync(CancellationToken cancellationToken);
+ ///
+ /// Discards uncommitted mutations and reloads the last durable state.
+ ///
+ /// The cancellation token.
+ /// A which represents the operation.
+ ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken);
+
///
/// Resets this instance, removing any persistent state.
///
diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs
index 591a988271d..658e3594695 100644
--- a/src/Orleans.Journaling/JournaledStateManager.cs
+++ b/src/Orleans.Journaling/JournaledStateManager.cs
@@ -177,9 +177,42 @@ private async Task WorkLoop()
{
if (needsRecovery)
{
- await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true);
- needsRecovery = false;
- CompleteRecoveryTrigger();
+ bool fenceOnFailure;
+ lock (_lock)
+ {
+ fenceOnFailure = _state is not ManagerState.Unknown;
+ if (fenceOnFailure)
+ {
+ _state = ManagerState.Recovering;
+ }
+ }
+
+ try
+ {
+ await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true);
+ lock (_lock)
+ {
+ if (fenceOnFailure)
+ {
+ _state = ManagerState.Ready;
+ }
+ }
+
+ needsRecovery = false;
+ CompleteRecoveryTrigger();
+ }
+ catch
+ {
+ lock (_lock)
+ {
+ if (fenceOnFailure)
+ {
+ _state = ManagerState.Ready;
+ }
+ }
+
+ throw;
+ }
}
WorkItem workItem;
@@ -364,25 +397,8 @@ private async Task WorkLoop()
writeSequence = new ReadOnlySequence(debugPoisonBuffer, 0, debugPoisonLength);
#endif
- var writeCompleted = false;
try
{
- if (isSnapshot && hasBufferToConsume)
- {
- await AppendStorageAsync(bufferToConsume.AsReadOnlySequence(), _shutdownCancellation.Token).ConfigureAwait(true);
- lock (_lock)
- {
- _journalWriter.Consume(bufferToConsume);
- foreach (var state in _states.Values)
- {
- state.OnWriteCompleted();
- }
- }
-
- bufferToConsume.Dispose();
- hasBufferToConsume = false;
- }
-
if (isSnapshot)
{
await ReplaceStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true);
@@ -392,35 +408,25 @@ private async Task WorkLoop()
await AppendStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true);
}
- writeCompleted = true;
- }
- finally
- {
- try
+ lock (_lock)
{
- if (writeCompleted)
+ if (hasBufferToConsume)
{
- lock (_lock)
- {
- if (hasBufferToConsume)
- {
- _journalWriter.Consume(bufferToConsume);
- }
- }
+ _journalWriter.Consume(bufferToConsume);
}
}
- finally
+ }
+ finally
+ {
+ committedBuffer.Dispose();
+ if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer)
{
- committedBuffer.Dispose();
- if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer)
- {
- bufferToConsume.Dispose();
- }
+ bufferToConsume.Dispose();
+ }
#if DEBUG
- debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67);
- ArrayPool.Shared.Return(debugPoisonBuffer);
+ debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67);
+ ArrayPool.Shared.Return(debugPoisonBuffer);
#endif
- }
}
// Notify all states that the operation completed.
@@ -437,6 +443,7 @@ private async Task WorkLoop()
}
}
}
+
break;
}
@@ -461,6 +468,17 @@ private async Task WorkLoop()
break;
}
+ case RevertPendingChangesWorkItem:
+ {
+ await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true);
+ lock (_lock)
+ {
+ _state = ManagerState.Ready;
+ }
+
+ break;
+ }
+
case InitializeWorkItem:
{
lock (_lock)
@@ -528,11 +546,23 @@ private async Task WorkLoop()
Debug.Assert(recoveryTrigger is null);
recoveryTrigger = workItem;
recoveryTriggerException = exception;
+ lock (_lock)
+ {
+ _state = ManagerState.Recovering;
+ }
+
needsRecovery = true;
}
else
{
workItem.SetException(exception);
+ if (workItem is RevertPendingChangesWorkItem)
+ {
+ lock (_lock)
+ {
+ _state = ManagerState.Fenced;
+ }
+ }
}
}
finally
@@ -664,10 +694,12 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit
public async ValueTask DeleteStateAsync(CancellationToken cancellationToken)
{
+ cancellationToken.ThrowIfCancellationRequested();
Task task;
bool didEnqueue;
lock (_lock)
{
+ ThrowIfStateOperationsUnavailable();
task = EnqueueOrGetPendingWorkItem(out didEnqueue);
}
@@ -679,7 +711,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken)
var startTimestamp = _shared.TimeProvider.GetTimestamp();
try
{
- await task;
+ await task.WaitAsync(cancellationToken);
_shared.Instruments.OnStateDeleteRequest(_shared.TimeProvider.GetElapsedTime(startTimestamp), succeeded: true);
}
catch
@@ -842,6 +874,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken)
string operation;
lock (_lock)
{
+ ThrowIfStateOperationsUnavailable();
var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested;
operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend;
pendingWrite = isSnapshot
@@ -867,6 +900,49 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken)
}
}
+ public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ Task pendingRecovery;
+ bool didEnqueue;
+ lock (_lock)
+ {
+ _shutdownCancellation.Token.ThrowIfCancellationRequested();
+ if (_state is ManagerState.Unknown)
+ {
+ throw new InvalidOperationException("The journaled state manager has not been initialized.");
+ }
+
+ pendingRecovery = EnqueueOrGetPendingWorkItem(out didEnqueue);
+ _state = ManagerState.Recovering;
+ }
+
+ if (didEnqueue)
+ {
+ _workSignal.Signal();
+ }
+
+ await pendingRecovery.WaitAsync(cancellationToken);
+ }
+
+ private void ThrowIfStateOperationsUnavailable()
+ {
+ _shutdownCancellation.Token.ThrowIfCancellationRequested();
+ switch (_state)
+ {
+ case ManagerState.Unknown:
+ throw new InvalidOperationException("The journaled state manager has not been initialized.");
+ case ManagerState.Ready:
+ return;
+ case ManagerState.Recovering:
+ throw new InvalidOperationException("Journaled state operations are unavailable while recovery is in progress.");
+ case ManagerState.Fenced:
+ throw new InvalidOperationException("Journaled state operations are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery.");
+ default:
+ throw new UnreachableException();
+ }
+ }
+
private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
{
var startTimestamp = _shared.TimeProvider.GetTimestamp();
@@ -983,11 +1059,33 @@ private void BindState(string name, uint id)
private async Task StopAsync(CancellationToken cancellationToken)
{
- _shutdownCancellation.Cancel();
+ lock (_lock)
+ {
+ _shutdownCancellation.Cancel();
+ }
+
_workSignal.Signal();
- if (_workLoop is { } task)
+ try
+ {
+ if (_workLoop is { } task)
+ {
+ await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing);
+ }
+ }
+ finally
{
- await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing);
+ CancelQueuedWorkItems(_shutdownCancellation.Token);
+ }
+ }
+
+ private void CancelQueuedWorkItems(CancellationToken cancellationToken)
+ {
+ lock (_lock)
+ {
+ while (_workQueue.TryDequeue(out var workItem))
+ {
+ workItem.TrySetCanceled(cancellationToken);
+ }
}
}
@@ -1103,6 +1201,8 @@ private sealed class WriteSnapshotWorkItem : WorkItem;
private sealed class DeleteStateWorkItem : WorkItem;
+ private sealed class RevertPendingChangesWorkItem : WorkItem;
+
private sealed class RegisterStateWorkItem(string name) : WorkItem(name)
{
public string Name => (string)Task.AsyncState!;
@@ -1111,7 +1211,9 @@ private sealed class RegisterStateWorkItem(string name) : WorkItem(name)
private enum ManagerState : byte
{
Unknown,
- Ready
+ Ready,
+ Recovering,
+ Fenced
}
private sealed class StateDirectory(
diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs
index 83096227060..e7fcd7aa32d 100644
--- a/src/api/Orleans.Journaling/Orleans.Journaling.cs
+++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs
@@ -226,6 +226,7 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable
System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken);
System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken);
void RegisterState(string name, IJournaledState state);
+ System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken);
System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync();
bool TryGetState(string name, out IJournaledState? state);
System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken);
diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs
index bb5c47cc049..5a7a602c620 100644
--- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs
+++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs
@@ -138,6 +138,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta
public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default;
+ public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException();
+
public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default;
}
diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs
index 174b160d496..bdd53841f81 100644
--- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs
+++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs
@@ -205,6 +205,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st
public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default;
+ public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException();
+
public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default;
}
diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs
index 2b2e90641d3..5dff6e54cd0 100644
--- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs
+++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs
@@ -104,6 +104,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st
public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default;
+ public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException();
+
public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default;
}
diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs
index 0a157bf7149..ea45acdbdeb 100644
--- a/test/Orleans.Journaling.Tests/StateManagerTests.cs
+++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs
@@ -1,6 +1,7 @@
using System.Buffers;
using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
+using System.Runtime.ExceptionServices;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Time.Testing;
using Orleans.Serialization.Buffers;
@@ -86,6 +87,55 @@ public async Task StateManager_Initialize_ThrowsWhenCompletedReadLeavesData()
Assert.Contains("did not read the completed journal data", exception.InnerException.Message, StringComparison.Ordinal);
}
+ [Fact]
+ public async Task StateManager_WriteOperations_RequireInitialization()
+ {
+ var sut = CreateTestSystem();
+
+ var writeException = await Assert.ThrowsAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+ var deleteException = await Assert.ThrowsAsync(
+ () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask());
+
+ Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal);
+ Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task StateManager_WriteOperations_RequireSuccessfulInitialization()
+ {
+ var storage = new CapturingStorage
+ {
+ NextReadException = new IOException("Expected recovery failure.")
+ };
+ var sut = CreateTestSystem(storage: storage);
+
+ await Assert.ThrowsAsync(() => sut.Lifecycle.OnStart());
+
+ var writeException = await Assert.ThrowsAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+ var deleteException = await Assert.ThrowsAsync(
+ () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask());
+
+ Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal);
+ Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task StateManager_WriteOperations_ObserveShutdown()
+ {
+ var sut = CreateTestSystem();
+ await sut.Lifecycle.OnStart();
+ await sut.Lifecycle.OnStop(CancellationToken.None);
+
+ await Assert.ThrowsAnyAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+ await Assert.ThrowsAnyAsync(
+ () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask());
+ await Assert.ThrowsAnyAsync(
+ () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask());
+ }
+
[Fact]
public async Task StateManager_Recovery_PreservesUnknownStateEntry()
{
@@ -259,7 +309,7 @@ public async Task StateManager_BinarySnapshot_StoresBinaryVarUIntEntries()
await sut.Manager.WriteStateAsync(CancellationToken.None);
var replacement = Assert.Single(storage.Replaces);
- Assert.Single(storage.Appends);
+ Assert.Empty(storage.Appends);
AssertContainsRuntimeAndApplicationEntries(ReadBinaryEntries(replacement));
}
@@ -276,9 +326,8 @@ public async Task StateManager_StorageOperationBytesMetric_RecordsWritePayloadSi
value.Value = 42;
await sut.Manager.WriteStateAsync(CancellationToken.None);
- var append = Assert.Single(storage.Appends);
var replacement = Assert.Single(storage.Replaces);
- Assert.Contains(observedBytes, measurement => measurement.Operation == "append" && measurement.Status == "ok" && measurement.Value == append.Length);
+ Assert.Empty(storage.Appends);
Assert.Contains(observedBytes, measurement => measurement.Operation == "replace" && measurement.Status == "ok" && measurement.Value == replacement.Length);
}
@@ -382,8 +431,9 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData()
dictionary.Add("key", 1);
await sut.Manager.WriteStateAsync(CancellationToken.None);
- Assert.Single(storage.Appends);
+ Assert.Empty(storage.Appends);
Assert.Single(storage.Replaces);
+ Assert.Equal(0, sut.Manager.PendingWriteByteCount);
var recovered = CreateTestSystem(storage: storage, journalFormat: new TrackingJournalFormat(SessionPool));
var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec());
@@ -392,6 +442,58 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData()
Assert.Equal(1, recoveredDictionary["key"]);
}
+ [Fact]
+ public async Task StateManager_SnapshotFailure_PreservesMixedPendingMutationsForOneCommitRetry()
+ {
+ var storage = new CapturingStorage();
+ var sut = CreateTestSystem(storage: storage);
+ var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec());
+ var value = new DurableValue("value", sut.Manager, CreateValueCodec());
+ var notifications = new AlwaysWritingState();
+ sut.Manager.RegisterState("notifications", notifications);
+
+ await sut.Lifecycle.OnStart();
+ dictionary.Add("persisted", 1);
+ value.Value = 1;
+ await sut.Manager.WriteStateAsync(CancellationToken.None);
+ Assert.Equal(1, notifications.WriteCompletedCount);
+
+ storage.IsCompactionRequested = true;
+ dictionary.Add("pending", 2);
+ value.Value = 2;
+ var pendingBytes = sut.Manager.PendingWriteByteCount;
+ var expected = new IOException("Expected replace failure.");
+ storage.NextReplaceException = expected;
+
+ var exception = await Assert.ThrowsAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+
+ Assert.Same(expected, exception);
+ Assert.Single(storage.Appends);
+ Assert.Empty(storage.Replaces);
+ Assert.Equal(1, notifications.WriteCompletedCount);
+ Assert.Equal(pendingBytes, sut.Manager.PendingWriteByteCount);
+ Assert.Equal(2, dictionary["pending"]);
+ Assert.Equal(2, value.Value);
+
+ await sut.Manager.WriteStateAsync(CancellationToken.None);
+
+ Assert.Single(storage.Appends);
+ Assert.Single(storage.Replaces);
+ Assert.Equal(2, storage.ReplaceAttemptCount);
+ Assert.Equal(2, notifications.WriteCompletedCount);
+ Assert.Equal(0, sut.Manager.PendingWriteByteCount);
+
+ var recovered = CreateTestSystem(storage: storage);
+ var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec());
+ var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec());
+ await recovered.Lifecycle.OnStart();
+
+ Assert.Equal(1, recoveredDictionary["persisted"]);
+ Assert.Equal(2, recoveredDictionary["pending"]);
+ Assert.Equal(2, recoveredValue.Value);
+ }
+
[Fact]
public async Task StateManager_DirectWriteFailure_AbortsEntryBeforeMutation()
{
@@ -542,36 +644,127 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu
}
[Fact]
- public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites()
+ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState()
+ {
+ var storage = new CapturingStorage();
+ var sut = CreateTestSystem(storage: storage);
+ var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec());
+ var value = new DurableValue("value", sut.Manager, CreateValueCodec());
+
+ await sut.Lifecycle.OnStart();
+ dictionary.Add("persisted", 1);
+ value.Value = 1;
+ await sut.Manager.WriteStateAsync(CancellationToken.None);
+
+ dictionary.Add("pending", 2);
+ value.Value = 2;
+ Assert.True(sut.Manager.PendingWriteByteCount > 0);
+
+ await sut.Manager.RevertPendingChangesAsync(CancellationToken.None);
+
+ Assert.Equal(0, sut.Manager.PendingWriteByteCount);
+ Assert.Equal(1, dictionary["persisted"]);
+ Assert.False(dictionary.ContainsKey("pending"));
+ Assert.Equal(1, value.Value);
+ }
+
+ [Fact]
+ public async Task StateManager_FencesStateOperationsUntilFailedRevertIsRetriedSuccessfully()
{
var storage = new CapturingStorage();
var sut = CreateTestSystem(storage: storage);
+ var value = new DurableValue("value", sut.Manager, CreateValueCodec());
+
+ await sut.Lifecycle.OnStart();
+ value.Value = 1;
+ await sut.Manager.WriteStateAsync(CancellationToken.None);
+ value.Value = 2;
+
+ var expected = new IOException("Expected recovery failure.");
+ storage.NextReadException = expected;
+ var exception = await Assert.ThrowsAsync(
+ () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask());
+ Assert.Same(expected, exception);
+
+ var writeException = await Assert.ThrowsAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+ Assert.Contains("state operations are fenced", writeException.Message, StringComparison.Ordinal);
+ Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase);
+ Assert.Contains("Call RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal);
+ var deleteException = await Assert.ThrowsAsync(
+ () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask());
+ Assert.Contains("state operations are fenced", deleteException.Message, StringComparison.Ordinal);
+ Assert.Contains("Call RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal);
+
+ await sut.Manager.RevertPendingChangesAsync(CancellationToken.None);
+ Assert.Equal(1, value.Value);
+
+ value.Value = 3;
+ await sut.Manager.WriteStateAsync(CancellationToken.None);
+ Assert.Equal(3, value.Value);
+ }
+
+ [Fact]
+ public async Task StateManager_StateOperations_ReportRecoveryInProgress()
+ {
+ var storage = new BlockingRecoveryStorage();
+ var sut = CreateTestSystem(storage: storage);
+ await sut.Lifecycle.OnStart();
+
+ var recovery = sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask();
+ await storage.RecoveryReadStarted.Task.WaitAsync(TimeSpan.FromSeconds(10));
+
+ var writeException = await Assert.ThrowsAsync(
+ () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask());
+ var deleteException = await Assert.ThrowsAsync(
+ () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask());
+
+ Assert.Contains("state operations are unavailable while recovery is in progress", writeException.Message, StringComparison.Ordinal);
+ Assert.Contains("state operations are unavailable while recovery is in progress", deleteException.Message, StringComparison.Ordinal);
+ Assert.DoesNotContain("RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal);
+ Assert.DoesNotContain("RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal);
+
+ storage.AllowRecoveryRead.SetResult();
+ await recovery.WaitAsync(TimeSpan.FromSeconds(10));
+ }
+
+ [Fact]
+ public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites()
+ {
+ var storage = new BlockingAppendStorage();
+ var sut = CreateTestSystem(storage: storage);
var state = new AlwaysWritingState();
sut.Manager.RegisterState("state", state);
+ await sut.Lifecycle.OnStart();
var first = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask();
+ await storage.FirstAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10));
var second = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask();
+ var third = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask();
- await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10));
- await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10));
+ storage.AllowFirstAppend.SetResult();
+ await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10));
- Assert.Equal(1, state.AppendEntriesCount);
- Assert.Single(storage.Appends);
+ Assert.Equal(2, state.AppendEntriesCount);
+ Assert.Equal(2, storage.Appends.Count);
}
[Fact]
public async Task StateManager_DeleteStateAsync_CoalescesQueuedDeletes()
{
- var storage = new CapturingStorage();
+ var storage = new BlockingDeleteStorage();
var sut = CreateTestSystem(storage: storage);
+ await sut.Lifecycle.OnStart();
var first = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask();
+ await storage.FirstDeleteStarted.Task.WaitAsync(TimeSpan.FromSeconds(10));
var second = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask();
+ var third = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask();
- await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10));
- await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10));
+ storage.AllowFirstDelete.SetResult();
+ await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10));
- Assert.Equal(1, storage.DeleteCount);
+ Assert.Equal(2, storage.DeleteCount);
}
[Fact]
@@ -634,13 +827,13 @@ public async Task StateManager_SnapshotWrite_DoesNotDiscardActiveEntry()
await writeTask.WaitAsync(TimeSpan.FromSeconds(10));
var replacement = Assert.Single(storage.Replaces);
Assert.DoesNotContain(ReadBinaryEntries(replacement), entry => entry.StreamId.Value >= 8);
- Assert.Single(storage.Appends);
+ Assert.Empty(storage.Appends);
storage.IsCompactionRequested = false;
await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
- Assert.Equal(2, storage.Appends.Count);
- var append = storage.Appends[^1];
+ Assert.Single(storage.Appends);
+ var append = storage.Appends[0];
Assert.Contains(ReadBinaryEntries(append), entry => entry.StreamId.Value >= 8);
Task StartSnapshotAndCommitActiveEntry()
@@ -849,6 +1042,7 @@ public async Task StateManager_RecoveryRetry_ReplaysFixedStorage()
await Assert.ThrowsAsync(
() => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)));
+ await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
Assert.Equal(42, value.Value);
@@ -865,6 +1059,7 @@ public async Task StateManager_RecoveryRetry_PreservesUnknownStreamOnce()
await Assert.ThrowsAsync(
() => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)));
+ await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
var replacement = Assert.Single(storage.Replaces);
@@ -882,6 +1077,7 @@ public async Task StateManager_RecoveryRetry_RemovesStaleRetiredPlaceholder()
await Assert.ThrowsAsync(
() => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)));
+ await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10));
Assert.False(sut.Manager.TryGetState("stale", out _));
@@ -1540,6 +1736,66 @@ public async ValueTask AppendAsync(ReadOnlySequence value, CancellationTok
public ValueTask DeleteAsync(CancellationToken cancellationToken) => default;
}
+ private sealed class BlockingDeleteStorage : IJournalStorage
+ {
+ public TaskCompletionSource FirstDeleteStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public TaskCompletionSource AllowFirstDelete { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public int DeleteCount { get; private set; }
+
+ public bool IsCompactionRequested => false;
+
+ public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
+ {
+ consumer.Complete(metadata: null);
+ return default;
+ }
+
+ public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default;
+
+ public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default;
+
+ public async ValueTask DeleteAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ DeleteCount++;
+ if (DeleteCount == 1)
+ {
+ FirstDeleteStarted.SetResult();
+ await AllowFirstDelete.Task.WaitAsync(cancellationToken);
+ }
+ }
+ }
+
+ private sealed class BlockingRecoveryStorage : IJournalStorage
+ {
+ private int _readCount;
+
+ public TaskCompletionSource RecoveryReadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public TaskCompletionSource AllowRecoveryRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public bool IsCompactionRequested => false;
+
+ public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
+ {
+ if (Interlocked.Increment(ref _readCount) == 2)
+ {
+ RecoveryReadStarted.SetResult();
+ await AllowRecoveryRead.Task.WaitAsync(cancellationToken);
+ }
+
+ consumer.Complete(metadata: null);
+ }
+
+ public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default;
+
+ public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default;
+
+ public ValueTask DeleteAsync(CancellationToken cancellationToken) => default;
+ }
+
private sealed class CapturingStorage : IJournalStorage
{
private readonly List _segments = [];
@@ -1562,6 +1818,10 @@ private sealed class CapturingStorage : IJournalStorage
public TaskCompletionSource AllowBlockedRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public Exception? NextReplaceException { get; set; }
+
+ public int ReplaceAttemptCount { get; private set; }
+
public int ReadConsumeCount { get; private set; }
public void ResetReadConsumeCount() => ReadConsumeCount = 0;
@@ -1577,7 +1837,6 @@ private sealed class CapturingStorage : IJournalStorage
public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(consumer);
-
if (NextReadException is { } exception)
{
NextReadException = null;
@@ -1637,6 +1896,13 @@ IEnumerable> GetSegments()
public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
+ ReplaceAttemptCount++;
+ if (NextReplaceException is { } exception)
+ {
+ NextReplaceException = null;
+ ExceptionDispatchInfo.Throw(exception);
+ }
+
if (DelayReplace)
{
ReplaceStarted.SetResult();
@@ -1908,6 +2174,8 @@ private sealed class AlwaysWritingState : IJournaledState
{
public int AppendEntriesCount { get; private set; }
+ public int WriteCompletedCount { get; private set; }
+
void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) { }
public void Reset(JournalStreamWriter writer) { }
@@ -1923,6 +2191,8 @@ public void AppendEntries(JournalStreamWriter writer)
public void AppendSnapshot(JournalStreamWriter writer) => AppendEntries(writer);
+ public void OnWriteCompleted() => WriteCompletedCount++;
+
public IJournaledState DeepCopy() => throw new NotSupportedException();
}