Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 72 additions & 19 deletions src/Orleans.Journaling/JournaledStateManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ internal sealed partial class JournaledStateManager : IJournaledStateManager, IJ
private readonly RetiredStateTracker _retirementTracker;
private Task? _workLoop;
private ManagerState _state;
private long _recoveryGeneration;
private bool _migrationSnapshotRequired;
private int _disposed;

Expand Down Expand Up @@ -138,6 +139,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken)
bool didEnqueue;
lock (_lock)
{
ThrowIfInitializationFenced();
if (_workLoop is null)
{
_workLoop = Start();
Expand Down Expand Up @@ -196,18 +198,21 @@ private async Task WorkLoop()
{
_state = ManagerState.Ready;
}

_recoveryGeneration++;
}

needsRecovery = false;
CompleteRecoveryTrigger();
}
catch
catch (Exception exception)
{
lock (_lock)
{
if (fenceOnFailure)
{
_state = ManagerState.Ready;
_state = ManagerState.Fenced;
FaultQueuedWorkItemsUnderLock(exception);
}
}

Expand Down Expand Up @@ -248,6 +253,13 @@ private async Task WorkLoop()

try
{
if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem
&& workItem.RecoveryGeneration != Volatile.Read(ref _recoveryGeneration))
{
throw new InvalidOperationException(
"The journaled state operation was queued before recovery discarded its mutations. Retry the operation.");
}

// Note that the implementation of each command is inlined to avoid allocating unnecessary async states.
// We are ok sacrificing some code organization for performance in the inner loop.
switch (workItem)
Expand Down Expand Up @@ -464,6 +476,8 @@ private async Task WorkLoop()
var id = _journalStreamDirectory.GetNextJournalStreamId();
_journalStreamDirectory.Set(name, id);
}

_recoveryGeneration++;
}
break;
}
Expand All @@ -474,6 +488,7 @@ private async Task WorkLoop()
lock (_lock)
{
_state = ManagerState.Ready;
_recoveryGeneration++;
}

break;
Expand All @@ -483,6 +498,7 @@ private async Task WorkLoop()
{
lock (_lock)
{
ThrowIfInitializationFenced();
_state = ManagerState.Ready;
}
break;
Expand Down Expand Up @@ -541,7 +557,7 @@ private async Task WorkLoop()
}
}

if (IsRecoverySignal(exception))
if (workItem is not RevertPendingChangesWorkItem && IsRecoverySignal(exception))
Comment thread
Copilot marked this conversation as resolved.
Resolved
{
Debug.Assert(recoveryTrigger is null);
recoveryTrigger = workItem;
Expand All @@ -553,16 +569,19 @@ private async Task WorkLoop()

needsRecovery = true;
}
else
else if (workItem is RevertPendingChangesWorkItem)
{
workItem.SetException(exception);
if (workItem is RevertPendingChangesWorkItem)
lock (_lock)
{
lock (_lock)
{
_state = ManagerState.Fenced;
}
_state = ManagerState.Fenced;
FaultQueuedWorkItemsUnderLock(exception);
}

workItem.SetException(exception);
}
else
{
workItem.SetException(exception);
}
}
finally
Expand All @@ -573,7 +592,14 @@ private async Task WorkLoop()
}
catch (Exception exception)
{
needsRecovery = true;
lock (_lock)
{
needsRecovery = _state is not ManagerState.Fenced;
if (needsRecovery)
{
FaultQueuedWorkItemsUnderLock(exception);
}
}
if (_shutdownCancellation.Token.IsCancellationRequested)
{
CompleteRecoveryTrigger();
Expand All @@ -582,7 +608,6 @@ private async Task WorkLoop()

try
{
FaultQueuedWorkItems(exception);
LogErrorProcessingWorkItems(_shared.Logger, exception);
}
finally
Expand All @@ -605,15 +630,11 @@ void CompleteRecoveryTrigger()
}
}

private void FaultQueuedWorkItems(Exception exception)
private void FaultQueuedWorkItemsUnderLock(Exception exception)
{
lock (_lock)
while (_workQueue.TryDequeue(out var workItem))
{
while (_workQueue.TryDequeue(out var workItem))
{
workItem.TrySetException(exception);
}

workItem.TrySetException(exception);
}
}

Expand Down Expand Up @@ -742,6 +763,20 @@ private async Task RecoverAsync(CancellationToken cancellationToken)

lock (_lock)
{
foreach (var (name, state) in _states)
{
if (state is not RetiredState && !_journalStreamDirectory.ContainsKey(name))
{
var id = _journalStreamDirectory.GetNextJournalStreamId();
while (_statesMap.ContainsKey(id))
{
id = checked(id + 1);
}

_journalStreamDirectory.Set(name, id);
}
Comment thread
Copilot marked this conversation as resolved.
Resolved
}

foreach ((var name, var state) in _states)
{
state.OnRecoveryCompleted();
Expand Down Expand Up @@ -943,6 +978,15 @@ private void ThrowIfStateOperationsUnavailable()
}
}

private void ThrowIfInitializationFenced()
{
if (_state is ManagerState.Fenced)
{
throw new InvalidOperationException(
"The journaled state manager is fenced because recovery failed. Call RevertPendingChangesAsync to retry recovery.");
}
}

private async ValueTask ReadStorageAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken)
{
var startTimestamp = _shared.TimeProvider.GetTimestamp();
Expand Down Expand Up @@ -1013,6 +1057,12 @@ private Task EnqueueOrGetPendingWorkItem<TWorkItem>(out bool didEnqueue)
continue;
}

if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem
&& workItem.RecoveryGeneration != _recoveryGeneration)
{
continue;
}

workItem.RecordTraceContext();
workItem.AddCaller();
didEnqueue = false;
Expand All @@ -1021,6 +1071,7 @@ private Task EnqueueOrGetPendingWorkItem<TWorkItem>(out bool didEnqueue)

var newWorkItem = new TWorkItem();
newWorkItem.EnqueuedTimestamp = _shared.TimeProvider.GetTimestamp();
newWorkItem.RecoveryGeneration = _recoveryGeneration;
Comment thread
Copilot marked this conversation as resolved.
Resolved
newWorkItem.RecordTraceContext();
_workQueue.Enqueue(newWorkItem);
didEnqueue = true;
Expand Down Expand Up @@ -1114,6 +1165,8 @@ private abstract class WorkItem : TaskCompletionSource

public long EnqueuedTimestamp { get; set; }

public long RecoveryGeneration { get; set; }

public ActivityContext TraceParent { get; private set; }

public IReadOnlyList<ActivityLink>? TraceLinks => _links;
Expand Down
Loading
Loading