Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 41 additions & 3 deletions src/Orleans.Journaling/JournaledStateManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ internal sealed partial class JournaledStateManager : IJournaledStateManager, IJ
private readonly RetiredStateTracker _retirementTracker;
private Task? _workLoop;
private ManagerState _state;
private long _recoveryGeneration;
private bool _migrationSnapshotRequired;
private int _disposed;

Expand Down Expand Up @@ -138,6 +139,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken)
bool didEnqueue;
lock (_lock)
{
ThrowIfInitializationFenced();
if (_workLoop is null)
{
_workLoop = Start();
Expand Down Expand Up @@ -196,6 +198,8 @@ private async Task WorkLoop()
{
_state = ManagerState.Ready;
}

_recoveryGeneration++;
}

needsRecovery = false;
Expand All @@ -207,7 +211,7 @@ private async Task WorkLoop()
{
if (fenceOnFailure)
{
_state = ManagerState.Ready;
_state = ManagerState.Fenced;
}
}

Expand Down Expand Up @@ -248,6 +252,13 @@ private async Task WorkLoop()

try
{
if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem
&& workItem.RecoveryGeneration != Volatile.Read(ref _recoveryGeneration))
{
throw new InvalidOperationException(
"The journaled state operation was queued before recovery discarded its mutations. Retry the operation.");
}

// Note that the implementation of each command is inlined to avoid allocating unnecessary async states.
// We are ok sacrificing some code organization for performance in the inner loop.
switch (workItem)
Expand Down Expand Up @@ -464,6 +475,8 @@ private async Task WorkLoop()
var id = _journalStreamDirectory.GetNextJournalStreamId();
_journalStreamDirectory.Set(name, id);
}

_recoveryGeneration++;
}
break;
}
Expand All @@ -474,6 +487,7 @@ private async Task WorkLoop()
lock (_lock)
{
_state = ManagerState.Ready;
_recoveryGeneration++;
}

break;
Expand All @@ -483,6 +497,7 @@ private async Task WorkLoop()
{
lock (_lock)
{
ThrowIfInitializationFenced();
_state = ManagerState.Ready;
}
break;
Expand Down Expand Up @@ -541,7 +556,7 @@ private async Task WorkLoop()
}
}

if (IsRecoverySignal(exception))
if (workItem is not RevertPendingChangesWorkItem && IsRecoverySignal(exception))
Comment thread
Copilot marked this conversation as resolved.
Resolved
{
Debug.Assert(recoveryTrigger is null);
recoveryTrigger = workItem;
Expand Down Expand Up @@ -573,7 +588,10 @@ private async Task WorkLoop()
}
catch (Exception exception)
{
needsRecovery = true;
lock (_lock)
{
needsRecovery = _state is not ManagerState.Fenced;
}
Comment thread
Copilot marked this conversation as resolved.
Resolved
if (_shutdownCancellation.Token.IsCancellationRequested)
{
CompleteRecoveryTrigger();
Expand Down Expand Up @@ -742,6 +760,14 @@ private async Task RecoverAsync(CancellationToken cancellationToken)

lock (_lock)
{
foreach (var (name, state) in _states)
{
if (state is not RetiredState && !_journalStreamDirectory.ContainsKey(name))
{
_journalStreamDirectory.Set(name, _journalStreamDirectory.GetNextJournalStreamId());
}
Comment thread
Copilot marked this conversation as resolved.
Resolved
}

foreach ((var name, var state) in _states)
{
state.OnRecoveryCompleted();
Expand Down Expand Up @@ -943,6 +969,15 @@ private void ThrowIfStateOperationsUnavailable()
}
}

private void ThrowIfInitializationFenced()
{
if (_state is ManagerState.Fenced)
{
throw new InvalidOperationException(
"The journaled state manager is fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery.");
}
}

private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
{
var startTimestamp = _shared.TimeProvider.GetTimestamp();
Expand Down Expand Up @@ -1021,6 +1056,7 @@ private Task EnqueueOrGetPendingWorkItem<TWorkItem>(out bool didEnqueue)

var newWorkItem = new TWorkItem();
newWorkItem.EnqueuedTimestamp = _shared.TimeProvider.GetTimestamp();
newWorkItem.RecoveryGeneration = _recoveryGeneration;
Comment thread
Copilot marked this conversation as resolved.
Resolved
newWorkItem.RecordTraceContext();
_workQueue.Enqueue(newWorkItem);
didEnqueue = true;
Expand Down Expand Up @@ -1114,6 +1150,8 @@ private abstract class WorkItem : TaskCompletionSource

public long EnqueuedTimestamp { get; set; }

public long RecoveryGeneration { get; set; }

public ActivityContext TraceParent { get; private set; }

public IReadOnlyList<ActivityLink>? TraceLinks => _links;
Expand Down
99 changes: 95 additions & 4 deletions test/Orleans.Journaling.Tests/StateManagerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(
}

[Fact]
public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailures()
public async Task StateManager_WriteStateAsync_FencesWritesAfterRepeatedRecoveryFailures()
{
var storage = new CapturingStorage();
var sut = CreateTestSystem(storage: storage);
Expand All @@ -652,13 +652,22 @@ public async Task StateManager_WriteStateAsync_RetriesRecoveryAfterRepeatedFailu
var secondRecoveryFailure = new IOException("Expected second recovery failure.");
storage.NextReadException = secondRecoveryFailure;
var recoveryException = await Assert.ThrowsAsync<IOException>(
() => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()
() => sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask()
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken));
Assert.Same(secondRecoveryFailure, recoveryException);

await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
var writeException = await Assert.ThrowsAsync<InvalidOperationException>(
() => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken));
Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase);

var initializeException = await Assert.ThrowsAsync<InvalidOperationException>(
() => sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken));
Assert.Contains("fenced", initializeException.Message, StringComparison.OrdinalIgnoreCase);

await sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask()
.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
Assert.True(dictionary.ContainsKey("first"));
Assert.False(dictionary.ContainsKey("second"));
}
Expand Down Expand Up @@ -688,6 +697,82 @@ public async Task StateManager_RevertPendingChanges_RestoresLastDurableState()
Assert.Equal(1, value.Value);
}

[Fact]
public async Task StateManager_RecoveryRejectsWritesQueuedBeforeMutationsWereDiscarded()
{
var storage = new BlockingAppendStorage
{
FirstAppendException = new InconsistentStateException("Expected write conflict.")
};
var sut = CreateTestSystem(storage: storage);
var value = new DurableValue<int>("value", sut.Manager, CreateValueCodec<int>());
await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken);
value.Value = 1;

var conflictingWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask();
await storage.FirstAppendStarted.Task.WaitAsync(
TimeSpan.FromSeconds(10),
TestContext.Current.CancellationToken);
value.Value = 2;
var queuedWrite = sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask();

storage.AllowFirstAppend.TrySetResult();

await Assert.ThrowsAsync<InconsistentStateException>(
() => conflictingWrite.WaitAsync(
TimeSpan.FromSeconds(10),
TestContext.Current.CancellationToken));
var exception = await Assert.ThrowsAsync<InvalidOperationException>(
() => queuedWrite.WaitAsync(
TimeSpan.FromSeconds(10),
TestContext.Current.CancellationToken));

Assert.Contains("queued before recovery discarded its mutations", exception.Message, StringComparison.Ordinal);
Assert.Empty(storage.Appends);
}

[Fact]
public async Task StateManager_InconsistentStateDuringExplicitRevertRemainsFenced()
{
var storage = new CapturingStorage();
var sut = CreateTestSystem(storage: storage);
var value = new DurableValue<int>("value", sut.Manager, CreateValueCodec<int>());

await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken);
value.Value = 1;
await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken);
value.Value = 2;
storage.NextReadException = new InconsistentStateException("Expected explicit recovery conflict.");

await Assert.ThrowsAsync<InconsistentStateException>(
() => sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken).AsTask());
var writeException = await Assert.ThrowsAsync<InvalidOperationException>(
() => sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask());
Assert.Contains("fenced", writeException.Message, StringComparison.OrdinalIgnoreCase);

await sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken);
Assert.Equal(1, value.Value);
}

[Fact]
public async Task StateManager_RevertAgainstEmptyJournalResetsUncommittedState()
{
var storage = new CapturingStorage();
var sut = CreateTestSystem(storage: storage);
var value = new DurableValue<int>("value", sut.Manager, CreateValueCodec<int>());
await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken);
value.Value = 42;

await sut.Manager.RevertPendingChangesAsync(TestContext.Current.CancellationToken);

Assert.Equal(0, value.Value);
await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken);
var recovered = CreateTestSystem(storage: storage);
var recoveredValue = new DurableValue<int>("value", recovered.Manager, CreateValueCodec<int>());
await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken);
Assert.Equal(0, recoveredValue.Value);
}

[Fact]
public async Task StateManager_FencesStateOperationsUntilFailedRevertIsRetriedSuccessfully()
{
Expand Down Expand Up @@ -1905,6 +1990,8 @@ private sealed class BlockingAppendStorage : IJournalStorage

public TaskCompletionSource AllowFirstAppend { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);

public Exception? FirstAppendException { get; init; }

public List<byte[]> Appends { get; } = [];

public bool IsCompactionRequested => false;
Expand All @@ -1924,6 +2011,10 @@ public async ValueTask AppendAsync(ReadOnlySequence<byte> value, CancellationTok
{
FirstAppendStarted.SetResult();
await AllowFirstAppend.Task.WaitAsync(cancellationToken);
if (FirstAppendException is { } exception)
{
throw exception;
}
}

Appends.Add(value.ToArray());
Expand Down
Loading