diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index cc8fd7af3eb..2861d6e3deb 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -38,6 +38,13 @@ public interface IJournaledStateManager : IAsyncDisposable /// A which represents the operation. ValueTask WriteStateAsync(CancellationToken cancellationToken); + /// + /// Discards uncommitted mutations and reloads the last durable state. + /// + /// The cancellation token. + /// A which represents the operation. + ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken); + /// /// Resets this instance, removing any persistent state. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 591a988271d..658e3594695 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -177,9 +177,42 @@ private async Task WorkLoop() { if (needsRecovery) { - await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); - needsRecovery = false; - CompleteRecoveryTrigger(); + bool fenceOnFailure; + lock (_lock) + { + fenceOnFailure = _state is not ManagerState.Unknown; + if (fenceOnFailure) + { + _state = ManagerState.Recovering; + } + } + + try + { + await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); + lock (_lock) + { + if (fenceOnFailure) + { + _state = ManagerState.Ready; + } + } + + needsRecovery = false; + CompleteRecoveryTrigger(); + } + catch + { + lock (_lock) + { + if (fenceOnFailure) + { + _state = ManagerState.Ready; + } + } + + throw; + } } WorkItem workItem; @@ -364,25 +397,8 @@ private async Task WorkLoop() writeSequence = new ReadOnlySequence(debugPoisonBuffer, 0, debugPoisonLength); #endif - var writeCompleted = false; try { - if (isSnapshot && hasBufferToConsume) - { - await AppendStorageAsync(bufferToConsume.AsReadOnlySequence(), _shutdownCancellation.Token).ConfigureAwait(true); - lock (_lock) - { - _journalWriter.Consume(bufferToConsume); - foreach (var state in _states.Values) - { - state.OnWriteCompleted(); - } - } - - bufferToConsume.Dispose(); - hasBufferToConsume = false; - } - if (isSnapshot) { await ReplaceStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true); @@ -392,35 +408,25 @@ private async Task WorkLoop() await AppendStorageAsync(writeSequence, _shutdownCancellation.Token).ConfigureAwait(true); } - writeCompleted = true; - } - finally - { - try + lock (_lock) { - if (writeCompleted) + if (hasBufferToConsume) { - lock (_lock) - { - if (hasBufferToConsume) - { - _journalWriter.Consume(bufferToConsume); - } - } + _journalWriter.Consume(bufferToConsume); } } - finally + } + finally + { + committedBuffer.Dispose(); + if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer) { - committedBuffer.Dispose(); - if (hasBufferToConsume && !bufferToConsumeIsCommittedBuffer) - { - bufferToConsume.Dispose(); - } + bufferToConsume.Dispose(); + } #if DEBUG - debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67); - ArrayPool.Shared.Return(debugPoisonBuffer); + debugPoisonBuffer.AsSpan(0, debugPoisonLength).Fill(0x67); + ArrayPool.Shared.Return(debugPoisonBuffer); #endif - } } // Notify all states that the operation completed. @@ -437,6 +443,7 @@ private async Task WorkLoop() } } } + break; } @@ -461,6 +468,17 @@ private async Task WorkLoop() break; } + case RevertPendingChangesWorkItem: + { + await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); + lock (_lock) + { + _state = ManagerState.Ready; + } + + break; + } + case InitializeWorkItem: { lock (_lock) @@ -528,11 +546,23 @@ private async Task WorkLoop() Debug.Assert(recoveryTrigger is null); recoveryTrigger = workItem; recoveryTriggerException = exception; + lock (_lock) + { + _state = ManagerState.Recovering; + } + needsRecovery = true; } else { workItem.SetException(exception); + if (workItem is RevertPendingChangesWorkItem) + { + lock (_lock) + { + _state = ManagerState.Fenced; + } + } } } finally @@ -664,10 +694,12 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; lock (_lock) { + ThrowIfStateOperationsUnavailable(); task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -679,7 +711,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken) var startTimestamp = _shared.TimeProvider.GetTimestamp(); try { - await task; + await task.WaitAsync(cancellationToken); _shared.Instruments.OnStateDeleteRequest(_shared.TimeProvider.GetElapsedTime(startTimestamp), succeeded: true); } catch @@ -842,6 +874,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) string operation; lock (_lock) { + ThrowIfStateOperationsUnavailable(); var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -867,6 +900,49 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken) } } + public async ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task pendingRecovery; + bool didEnqueue; + lock (_lock) + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_state is ManagerState.Unknown) + { + throw new InvalidOperationException("The journaled state manager has not been initialized."); + } + + pendingRecovery = EnqueueOrGetPendingWorkItem(out didEnqueue); + _state = ManagerState.Recovering; + } + + if (didEnqueue) + { + _workSignal.Signal(); + } + + await pendingRecovery.WaitAsync(cancellationToken); + } + + private void ThrowIfStateOperationsUnavailable() + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + switch (_state) + { + case ManagerState.Unknown: + throw new InvalidOperationException("The journaled state manager has not been initialized."); + case ManagerState.Ready: + return; + case ManagerState.Recovering: + throw new InvalidOperationException("Journaled state operations are unavailable while recovery is in progress."); + case ManagerState.Fenced: + throw new InvalidOperationException("Journaled state operations are fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery."); + default: + throw new UnreachableException(); + } + } + private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { var startTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -983,11 +1059,33 @@ private void BindState(string name, uint id) private async Task StopAsync(CancellationToken cancellationToken) { - _shutdownCancellation.Cancel(); + lock (_lock) + { + _shutdownCancellation.Cancel(); + } + _workSignal.Signal(); - if (_workLoop is { } task) + try + { + if (_workLoop is { } task) + { + await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + } + } + finally { - await task.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.SuppressThrowing); + CancelQueuedWorkItems(_shutdownCancellation.Token); + } + } + + private void CancelQueuedWorkItems(CancellationToken cancellationToken) + { + lock (_lock) + { + while (_workQueue.TryDequeue(out var workItem)) + { + workItem.TrySetCanceled(cancellationToken); + } } } @@ -1103,6 +1201,8 @@ private sealed class WriteSnapshotWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; + private sealed class RevertPendingChangesWorkItem : WorkItem; + private sealed class RegisterStateWorkItem(string name) : WorkItem(name) { public string Name => (string)Task.AsyncState!; @@ -1111,7 +1211,9 @@ private sealed class RegisterStateWorkItem(string name) : WorkItem(name) private enum ManagerState : byte { Unknown, - Ready + Ready, + Recovering, + Fenced } private sealed class StateDirectory( diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 83096227060..e7fcd7aa32d 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -226,6 +226,7 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken); void RegisterState(string name, IJournaledState state); + System.Threading.Tasks.ValueTask RevertPendingChangesAsync(System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetState(string name, out IJournaledState? state); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index bb5c47cc049..5a7a602c620 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -138,6 +138,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState sta public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 174b160d496..bdd53841f81 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -205,6 +205,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index 2b2e90641d3..5dff6e54cd0 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -104,6 +104,8 @@ public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? st public ValueTask WriteStateAsync(CancellationToken cancellationToken) => default; + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => default; } diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 0a157bf7149..ea45acdbdeb 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Collections.Concurrent; using System.Diagnostics.Metrics; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; using Orleans.Serialization.Buffers; @@ -86,6 +87,55 @@ public async Task StateManager_Initialize_ThrowsWhenCompletedReadLeavesData() Assert.Contains("did not read the completed journal data", exception.InnerException.Message, StringComparison.Ordinal); } + [Fact] + public async Task StateManager_WriteOperations_RequireInitialization() + { + var sut = CreateTestSystem(); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StateManager_WriteOperations_RequireSuccessfulInitialization() + { + var storage = new CapturingStorage + { + NextReadException = new IOException("Expected recovery failure.") + }; + var sut = CreateTestSystem(storage: storage); + + await Assert.ThrowsAsync(() => sut.Lifecycle.OnStart()); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task StateManager_WriteOperations_ObserveShutdown() + { + var sut = CreateTestSystem(); + await sut.Lifecycle.OnStart(); + await sut.Lifecycle.OnStop(CancellationToken.None); + + await Assert.ThrowsAnyAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + await Assert.ThrowsAnyAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + await Assert.ThrowsAnyAsync( + () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); + } + [Fact] public async Task StateManager_Recovery_PreservesUnknownStateEntry() { @@ -259,7 +309,7 @@ public async Task StateManager_BinarySnapshot_StoresBinaryVarUIntEntries() await sut.Manager.WriteStateAsync(CancellationToken.None); var replacement = Assert.Single(storage.Replaces); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); AssertContainsRuntimeAndApplicationEntries(ReadBinaryEntries(replacement)); } @@ -276,9 +326,8 @@ public async Task StateManager_StorageOperationBytesMetric_RecordsWritePayloadSi value.Value = 42; await sut.Manager.WriteStateAsync(CancellationToken.None); - var append = Assert.Single(storage.Appends); var replacement = Assert.Single(storage.Replaces); - Assert.Contains(observedBytes, measurement => measurement.Operation == "append" && measurement.Status == "ok" && measurement.Value == append.Length); + Assert.Empty(storage.Appends); Assert.Contains(observedBytes, measurement => measurement.Operation == "replace" && measurement.Status == "ok" && measurement.Value == replacement.Length); } @@ -382,8 +431,9 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData() dictionary.Add("key", 1); await sut.Manager.WriteStateAsync(CancellationToken.None); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); Assert.Single(storage.Replaces); + Assert.Equal(0, sut.Manager.PendingWriteByteCount); var recovered = CreateTestSystem(storage: storage, journalFormat: new TrackingJournalFormat(SessionPool)); var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec()); @@ -392,6 +442,58 @@ public async Task StateManager_SnapshotFlush_ResetsPendingAppendData() Assert.Equal(1, recoveredDictionary["key"]); } + [Fact] + public async Task StateManager_SnapshotFailure_PreservesMixedPendingMutationsForOneCommitRetry() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + var notifications = new AlwaysWritingState(); + sut.Manager.RegisterState("notifications", notifications); + + await sut.Lifecycle.OnStart(); + dictionary.Add("persisted", 1); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(1, notifications.WriteCompletedCount); + + storage.IsCompactionRequested = true; + dictionary.Add("pending", 2); + value.Value = 2; + var pendingBytes = sut.Manager.PendingWriteByteCount; + var expected = new IOException("Expected replace failure."); + storage.NextReplaceException = expected; + + var exception = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + + Assert.Same(expected, exception); + Assert.Single(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(1, notifications.WriteCompletedCount); + Assert.Equal(pendingBytes, sut.Manager.PendingWriteByteCount); + Assert.Equal(2, dictionary["pending"]); + Assert.Equal(2, value.Value); + + await sut.Manager.WriteStateAsync(CancellationToken.None); + + Assert.Single(storage.Appends); + Assert.Single(storage.Replaces); + Assert.Equal(2, storage.ReplaceAttemptCount); + Assert.Equal(2, notifications.WriteCompletedCount); + Assert.Equal(0, sut.Manager.PendingWriteByteCount); + + var recovered = CreateTestSystem(storage: storage); + var recoveredDictionary = new DurableDictionary("dict", recovered.Manager, CreateDictionaryCodec()); + var recoveredValue = new DurableValue("value", recovered.Manager, CreateValueCodec()); + await recovered.Lifecycle.OnStart(); + + Assert.Equal(1, recoveredDictionary["persisted"]); + Assert.Equal(2, recoveredDictionary["pending"]); + Assert.Equal(2, recoveredValue.Value); + } + [Fact] public async Task StateManager_DirectWriteFailure_AbortsEntryBeforeMutation() { @@ -542,36 +644,127 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu } [Fact] - public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() + public async Task StateManager_RevertPendingChanges_RestoresLastDurableState() + { + var storage = new CapturingStorage(); + var sut = CreateTestSystem(storage: storage); + var dictionary = new DurableDictionary("dict", sut.Manager, CreateDictionaryCodec()); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(); + dictionary.Add("persisted", 1); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + + dictionary.Add("pending", 2); + value.Value = 2; + Assert.True(sut.Manager.PendingWriteByteCount > 0); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + + Assert.Equal(0, sut.Manager.PendingWriteByteCount); + Assert.Equal(1, dictionary["persisted"]); + Assert.False(dictionary.ContainsKey("pending")); + Assert.Equal(1, value.Value); + } + + [Fact] + public async Task StateManager_FencesStateOperationsUntilFailedRevertIsRetriedSuccessfully() { var storage = new CapturingStorage(); var sut = CreateTestSystem(storage: storage); + var value = new DurableValue("value", sut.Manager, CreateValueCodec()); + + await sut.Lifecycle.OnStart(); + value.Value = 1; + await sut.Manager.WriteStateAsync(CancellationToken.None); + value.Value = 2; + + var expected = new IOException("Expected recovery failure."); + storage.NextReadException = expected; + var exception = await Assert.ThrowsAsync( + () => sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, exception); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("state operations are fenced", writeException.Message, StringComparison.Ordinal); + Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Call RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("state operations are fenced", deleteException.Message, StringComparison.Ordinal); + Assert.Contains("Call RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal); + + await sut.Manager.RevertPendingChangesAsync(CancellationToken.None); + Assert.Equal(1, value.Value); + + value.Value = 3; + await sut.Manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(3, value.Value); + } + + [Fact] + public async Task StateManager_StateOperations_ReportRecoveryInProgress() + { + var storage = new BlockingRecoveryStorage(); + var sut = CreateTestSystem(storage: storage); + await sut.Lifecycle.OnStart(); + + var recovery = sut.Manager.RevertPendingChangesAsync(CancellationToken.None).AsTask(); + await storage.RecoveryReadStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + var writeException = await Assert.ThrowsAsync( + () => sut.Manager.WriteStateAsync(CancellationToken.None).AsTask()); + var deleteException = await Assert.ThrowsAsync( + () => sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask()); + + Assert.Contains("state operations are unavailable while recovery is in progress", writeException.Message, StringComparison.Ordinal); + Assert.Contains("state operations are unavailable while recovery is in progress", deleteException.Message, StringComparison.Ordinal); + Assert.DoesNotContain("RevertPendingChangesAsync", writeException.Message, StringComparison.Ordinal); + Assert.DoesNotContain("RevertPendingChangesAsync", deleteException.Message, StringComparison.Ordinal); + + storage.AllowRecoveryRead.SetResult(); + await recovery.WaitAsync(TimeSpan.FromSeconds(10)); + } + + [Fact] + public async Task StateManager_WriteStateAsync_CoalescesQueuedWrites() + { + var storage = new BlockingAppendStorage(); + var sut = CreateTestSystem(storage: storage); var state = new AlwaysWritingState(); sut.Manager.RegisterState("state", state); + await sut.Lifecycle.OnStart(); var first = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + await storage.FirstAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); var second = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); + var third = sut.Manager.WriteStateAsync(CancellationToken.None).AsTask(); - await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)); - await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + storage.AllowFirstAppend.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(1, state.AppendEntriesCount); - Assert.Single(storage.Appends); + Assert.Equal(2, state.AppendEntriesCount); + Assert.Equal(2, storage.Appends.Count); } [Fact] public async Task StateManager_DeleteStateAsync_CoalescesQueuedDeletes() { - var storage = new CapturingStorage(); + var storage = new BlockingDeleteStorage(); var sut = CreateTestSystem(storage: storage); + await sut.Lifecycle.OnStart(); var first = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); + await storage.FirstDeleteStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); var second = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); + var third = sut.Manager.DeleteStateAsync(CancellationToken.None).AsTask(); - await sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10)); - await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + storage.AllowFirstDelete.SetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(1, storage.DeleteCount); + Assert.Equal(2, storage.DeleteCount); } [Fact] @@ -634,13 +827,13 @@ public async Task StateManager_SnapshotWrite_DoesNotDiscardActiveEntry() await writeTask.WaitAsync(TimeSpan.FromSeconds(10)); var replacement = Assert.Single(storage.Replaces); Assert.DoesNotContain(ReadBinaryEntries(replacement), entry => entry.StreamId.Value >= 8); - Assert.Single(storage.Appends); + Assert.Empty(storage.Appends); storage.IsCompactionRequested = false; await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); - Assert.Equal(2, storage.Appends.Count); - var append = storage.Appends[^1]; + Assert.Single(storage.Appends); + var append = storage.Appends[0]; Assert.Contains(ReadBinaryEntries(append), entry => entry.StreamId.Value >= 8); Task StartSnapshotAndCommitActiveEntry() @@ -849,6 +1042,7 @@ public async Task StateManager_RecoveryRetry_ReplaysFixedStorage() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); Assert.Equal(42, value.Value); @@ -865,6 +1059,7 @@ public async Task StateManager_RecoveryRetry_PreservesUnknownStreamOnce() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); var replacement = Assert.Single(storage.Replaces); @@ -882,6 +1077,7 @@ public async Task StateManager_RecoveryRetry_RemovesStaleRetiredPlaceholder() await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart().WaitAsync(TimeSpan.FromSeconds(10))); + await sut.Manager.InitializeAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); await sut.Manager.WriteStateAsync(CancellationToken.None).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); Assert.False(sut.Manager.TryGetState("stale", out _)); @@ -1540,6 +1736,66 @@ public async ValueTask AppendAsync(ReadOnlySequence value, CancellationTok public ValueTask DeleteAsync(CancellationToken cancellationToken) => default; } + private sealed class BlockingDeleteStorage : IJournalStorage + { + public TaskCompletionSource FirstDeleteStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowFirstDelete { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int DeleteCount { get; private set; } + + public bool IsCompactionRequested => false; + + public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + consumer.Complete(metadata: null); + return default; + } + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public async ValueTask DeleteAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + DeleteCount++; + if (DeleteCount == 1) + { + FirstDeleteStarted.SetResult(); + await AllowFirstDelete.Task.WaitAsync(cancellationToken); + } + } + } + + private sealed class BlockingRecoveryStorage : IJournalStorage + { + private int _readCount; + + public TaskCompletionSource RecoveryReadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource AllowRecoveryRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool IsCompactionRequested => false; + + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _readCount) == 2) + { + RecoveryReadStarted.SetResult(); + await AllowRecoveryRead.Task.WaitAsync(cancellationToken); + } + + consumer.Complete(metadata: null); + } + + public ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask AppendAsync(ReadOnlySequence value, CancellationToken cancellationToken) => default; + + public ValueTask DeleteAsync(CancellationToken cancellationToken) => default; + } + private sealed class CapturingStorage : IJournalStorage { private readonly List _segments = []; @@ -1562,6 +1818,10 @@ private sealed class CapturingStorage : IJournalStorage public TaskCompletionSource AllowBlockedRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Exception? NextReplaceException { get; set; } + + public int ReplaceAttemptCount { get; private set; } + public int ReadConsumeCount { get; private set; } public void ResetReadConsumeCount() => ReadConsumeCount = 0; @@ -1577,7 +1837,6 @@ private sealed class CapturingStorage : IJournalStorage public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(consumer); - if (NextReadException is { } exception) { NextReadException = null; @@ -1637,6 +1896,13 @@ IEnumerable> GetSegments() public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + ReplaceAttemptCount++; + if (NextReplaceException is { } exception) + { + NextReplaceException = null; + ExceptionDispatchInfo.Throw(exception); + } + if (DelayReplace) { ReplaceStarted.SetResult(); @@ -1908,6 +2174,8 @@ private sealed class AlwaysWritingState : IJournaledState { public int AppendEntriesCount { get; private set; } + public int WriteCompletedCount { get; private set; } + void IJournaledState.ReplayEntry(JournalEntry entry, JournalReplayContext context) { } public void Reset(JournalStreamWriter writer) { } @@ -1923,6 +2191,8 @@ public void AppendEntries(JournalStreamWriter writer) public void AppendSnapshot(JournalStreamWriter writer) => AppendEntries(writer); + public void OnWriteCompleted() => WriteCompletedCount++; + public IJournaledState DeepCopy() => throw new NotSupportedException(); }