From 8d98f37ec58b388fc4e8d6bb2c3ade0a99c201f5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 08:43:15 -0700 Subject: [PATCH 01/22] feat(journaling): add operation prerequisite and completion hooks --- .../grains/journaling/runtime-behavior.md | 35 + .../IJournaledStateCaptureHook.cs | 15 + src/Orleans.Journaling/IJournaledStateHook.cs | 56 ++ .../IJournaledStateManager.cs | 23 +- src/Orleans.Journaling/JournaledStateHook.cs | 37 + .../JournaledStateManager.cs | 295 +++++++- .../JournaledStatePostCommitException.cs | 27 + .../JournaledStatePreCommitException.cs | 27 + src/Orleans.Journaling/README.md | 33 + .../Orleans.Journaling/Orleans.Journaling.cs | 108 +++ .../StateManagerHookTests.cs | 697 ++++++++++++++++++ 11 files changed, 1344 insertions(+), 9 deletions(-) create mode 100644 src/Orleans.Journaling/IJournaledStateCaptureHook.cs create mode 100644 src/Orleans.Journaling/IJournaledStateHook.cs create mode 100644 src/Orleans.Journaling/JournaledStateHook.cs create mode 100644 src/Orleans.Journaling/JournaledStatePostCommitException.cs create mode 100644 src/Orleans.Journaling/JournaledStatePreCommitException.cs create mode 100644 test/Orleans.Journaling.Tests/StateManagerHookTests.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index e513f837fc9..eff4032fb4d 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -68,6 +68,41 @@ Orleans executes that synchronous block on a single activation thread. Another g the operation awaits, so keep shared state safe to commit at each await. Any caller's write can include staged mutations from other calls. +## Journal operation hooks + +Activation-scoped features coordinate prerequisites and completion through +. The owner exposes a stable, lazily +allocated list of registrations. Inspect and +deduplicate feature registrations on the owner's logical execution context while persistence +is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list +mutation throughout each operation. + +Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside +the manager lock. At most one supplies the +final prerequisite. Its before callback runs last, and the work loop awaits it directly before +synchronous capture or storage deletion. Prerequisites cover changes staged during asynchronous +preparation, including the final hook's own I/O wait. Preserve operation-local bookkeeping for +the captured batch separately from changes staged later. + +Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. +All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers +share callbacks for the actual operation. adapts delegates, +executing the synchronous delegate before the asynchronous delegate in each phase. + +A failed prerequisite reports with pending +state retained for an explicit persistence retry after the prerequisite is restored. A failed after +callback reports with persistence completed. +Remaining after callbacks run, multiple failures are aggregated, and the manager stays usable. +The feature's durable recovery protocol resumes interrupted post-persistence work. Storage and +state-processing failures retain the manager's fencing and fresh-recovery behavior. + +Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait leaves the owned +operation running through its actual outcome. Disposal drains owned hooks and storage before releasing +journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers +share this completion. Recursive initialization, persistence, or disposal on the same owner from a +hook is rejected. For deletion, the feature owner stops admission and drains feature operations before +queuing the whole-journal reset. + ## Consistency and competing writers Orleans grain placement normally supplies a single active writer for a grain identity. Journal storage providers also use optimistic concurrency to protect the journal when a stale or competing writer reaches storage. diff --git a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs new file mode 100644 index 00000000000..cd6d0b42693 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs @@ -0,0 +1,15 @@ +namespace Orleans.Journaling; + +/// +/// Establishes the final prerequisite immediately before journal capture or deletion. +/// +/// +/// An owner admits at most one capture hook in . +/// Its before callback runs after all ordinary before callbacks. The work loop awaits it directly, +/// then captures state or starts deletion without a further asynchronous phase. +/// Its prerequisites cover changes staged while its own I/O awaited. After callbacks retain +/// normal list order. Features inspect and deduplicate this registration using the same hook list. +/// +public interface IJournaledStateCaptureHook : IJournaledStateHook +{ +} diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs new file mode 100644 index 00000000000..d0ffaae24f3 --- /dev/null +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -0,0 +1,56 @@ +namespace Orleans.Journaling; + +/// +/// Identifies a journal persistence operation. +/// +public enum JournaledStateOperation +{ + /// Appends pending journal entries. + Write, + + /// Replaces storage with a snapshot. + Snapshot, + + /// Deletes storage and resets registered states. + Delete +} + +/// +/// Participates in the prerequisites and completion of actual journal operations. +/// +/// +/// Ordinary before hooks run in list order on the owner's logical execution context, outside its lock. +/// An optional runs last immediately before capture or deletion. +/// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after +/// storage acknowledgement and state acknowledgement or reset, including successful writes which +/// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later +/// pending changes separate. Recursive operations on the same journal owner are rejected. +/// +public interface IJournaledStateHook +{ + /// + /// Establishes prerequisites before capture or storage deletion. + /// + /// The operation about to execute. + /// The token for the owned operation's lifetime. + /// A completion representing the prerequisite work. + /// + /// Failure reports and retains pending changes + /// for an explicit retry. All staged changes remain safe to commit. Full deletion requires the + /// owner to stop admission and drain feature operations before queuing deletion. + /// + ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; + + /// + /// Completes post-persistence work after the operation succeeds. + /// + /// The completed operation. + /// The token for the owned operation's lifetime. + /// A completion representing post-persistence work. + /// + /// Every after hook is invoked even when an earlier after hook fails. Failures are surfaced as + /// and leave the successfully persisted owner usable. + /// Durable feature state supplies recovery for interrupted post-persistence work. + /// + ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => default; +} diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index b6363a1b4cc..4f2af34920e 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -9,9 +9,24 @@ namespace Orleans.Journaling; /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. /// Disposing this manager stops journal processing and releases its journal resources. +/// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable { + /// + /// Gets the mutable, lazily allocated list of journal operation hooks. + /// + /// + /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while + /// persistence is quiescent. Mutation during an operation is rejected. Ordinary before hooks + /// and all after hooks execute in list order. The optional single + /// supplies the final prerequisite. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Custom owners implement this property to support hooks. + /// + /// The custom owner does not support hooks. + IList Hooks => throw new NotSupportedException("This journal owner does not support operation hooks."); + /// ValueTask IAsyncDisposable.DisposeAsync() => default; @@ -51,8 +66,11 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// Stage mutations only after establishing that they are safe to commit. Pending changes are shared /// by all callers using this manager. Storage acknowledgement establishes durability. - /// A failed journal operation fences the manager; recovery requires a new manager and state machine instances. + /// A storage or state-processing failure fences the manager; recovery requires a new manager and state machine instances. /// Cancellation stops the caller's wait; an already queued write continues to its storage outcome. + /// Before-hook failure reports and retains pending + /// changes for an explicit retry. After-hook failure reports + /// after successful persistence. /// /// The token used to cancel the caller's wait. /// A task representing the write acknowledgement. @@ -65,6 +83,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// Before-hook failure reports with state retained. + /// After-hook failure reports after storage deletion + /// and state reset succeed. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/JournaledStateHook.cs b/src/Orleans.Journaling/JournaledStateHook.cs new file mode 100644 index 00000000000..31e12d601f8 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStateHook.cs @@ -0,0 +1,37 @@ +namespace Orleans.Journaling; + +/// +/// Adapts synchronous and asynchronous delegates to journal operation hooks. +/// +/// +/// When both delegates for a phase are supplied, the synchronous delegate runs first. +/// Asynchronous delegates await their I/O outcome before completing their returned value task. +/// Custom implementations can carry feature identity and state +/// for inspection and deduplication in the owner's hook list. +/// +public sealed class JournaledStateHook : IJournaledStateHook +{ + /// Gets the synchronous prerequisite callback. + public Action? BeforeOperation { get; init; } + + /// Gets the asynchronous prerequisite callback. + public Func? BeforeOperationAsync { get; init; } + + /// Gets the synchronous completion callback. + public Action? AfterOperation { get; init; } + + /// Gets the asynchronous completion callback. + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index cbf6be752bc..4a852b65396 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,6 +1,8 @@ using System.Buffers; +using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using Microsoft.Extensions.Logging; using Orleans.Diagnostics; using Orleans.Serialization.Buffers; @@ -11,6 +13,7 @@ namespace Orleans.Journaling; internal partial class JournaledStateManager : IJournaledStateManager, IJournalStorageConsumer, ILifecycleParticipant, ILifecycleObserver, IDisposable { private const uint MinApplicationJournalStreamId = 8u; + private static readonly AsyncLocal CurrentHookOwner = new(); #if NET9_0_OR_GREATER private readonly Lock _lock = new(); #else @@ -33,6 +36,20 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private Exception? _failure; private bool _migrationSnapshotRequired; private int _disposed; + private Task? _disposeTask; + private HookCollection? _hooks; + private bool _hookOperationRunning; + + public IList Hooks + { + get + { + lock (_lock) + { + return _hooks ??= new(this); + } + } + } public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -155,6 +172,7 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -252,6 +270,7 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _hookOperationRunning = workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -273,6 +292,9 @@ private async Task WorkLoop() storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); } + JournaledStateOperation? hookOperation = null; + var beforeHookRunning = false; + var afterHookRunning = false; try { if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem @@ -282,6 +304,42 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } + hookOperation = workItem switch + { + AppendJournalWorkItem or WriteSnapshotWorkItem => + workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested + ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + if (hookOperation is { } operation && _hooks is { Count: > 0 }) + { + beforeHookRunning = true; + var captureHook = GetCaptureHook(); + if (captureHook is null || _hooks.Count > 1) + { + await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + + if (captureHook is not null) + { + // Await the final prerequisite in this frame so capture follows its completion directly. + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + try + { + await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } + finally + { + CurrentHookOwner.Value = previous; + } + } + + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + beforeHookRunning = false; + } + // Note that the implementation of each command is inlined to avoid allocating unnecessary async states. // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) @@ -291,9 +349,7 @@ private async Task WorkLoop() { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = workItem is WriteSnapshotWorkItem - || _migrationSnapshotRequired - || _storage.IsCompactionRequested; + var isSnapshot = hookOperation == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; @@ -540,13 +596,47 @@ private async Task WorkLoop() } } + if (hookOperation is { } completedOperation && _hooks is { Count: > 0 }) + { + afterHookRunning = true; + await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); + afterHookRunning = false; + } + if (recordQueueDuration && queueOperation is not null) { _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: true); } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetResult(); + } + } + catch (Exception exception) when (beforeHookRunning || afterHookRunning) + { + if (beforeHookRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) + { + workItem.TrySetCanceled(_shutdownCancellation.Token); + continue; + } + + LogOperationHookFailed(_shared.Logger, exception, hookOperation!.Value, afterHookRunning); + storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation hook failed."); + if (recordQueueDuration && queueOperation is not null) + { + _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); + } + + lock (_lock) + { + _hookOperationRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(hookOperation.Value, exception) + : new JournaledStatePreCommitException(hookOperation.Value, exception)); + } } catch (Exception exception) { @@ -580,6 +670,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _hookOperationRunning = false; + } + storageActivity?.Dispose(); } } @@ -596,6 +691,154 @@ private async Task WorkLoop() } } + private IJournaledStateCaptureHook? GetCaptureHook() + { + foreach (var hook in _hooks!) + { + if (hook is IJournaledStateCaptureHook captureHook) + { + return captureHook; + } + } + + return null; + } + + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + try + { + foreach (var hook in _hooks!) + { + if (hook is IJournaledStateCaptureHook) + { + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + finally + { + CurrentHookOwner.Value = previous; + } + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + var previous = CurrentHookOwner.Value; + CurrentHookOwner.Value = this; + List? failures = null; + try + { + foreach (var hook in _hooks!) + { + try + { + await hook.AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + } + finally + { + CurrentHookOwner.Value = previous; + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private void ThrowIfHookReentry() + { + if (ReferenceEquals(CurrentHookOwner.Value, this)) + { + throw new InvalidOperationException("Journal operation hooks cannot enqueue another operation on the same journal owner."); + } + } + + private sealed class HookCollection(JournaledStateManager owner) : Collection + { + protected override void InsertItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, replacingIndex: -1); + base.InsertItem(index, item); + } + } + + protected override void SetItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + ValidateHook(item, index); + base.SetItem(index, item); + } + } + + protected override void RemoveItem(int index) + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.RemoveItem(index); + } + } + + protected override void ClearItems() + { + lock (owner._lock) + { + EnsureMutationAllowed(); + base.ClearItems(); + } + } + + private void EnsureMutationAllowed() + { + ObjectDisposedException.ThrowIf(owner._disposed != 0, owner); + owner._shutdownCancellation.Token.ThrowIfCancellationRequested(); + owner.ThrowIfFenced(); + if (owner._hookOperationRunning) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + } + + private void ValidateHook(IJournaledStateHook item, int replacingIndex) + { + ArgumentNullException.ThrowIfNull(item); + if (item is IJournaledStateCaptureHook) + { + for (var i = 0; i < Count; i++) + { + if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) + { + throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); + } + } + } + } + } + private void Fence(Exception exception) { lock (_lock) @@ -699,6 +942,7 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -886,6 +1130,7 @@ private InvalidOperationException CreateRecoveryFormatException(Exception except public async ValueTask WriteStateAsync(CancellationToken cancellationToken = default) { + ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task pendingWrite; @@ -1062,9 +1307,23 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { + ThrowIfHookReentry(); + AggregateException? cancellationFailure = null; lock (_lock) { - _shutdownCancellation.Cancel(); + try + { + _shutdownCancellation.Cancel(); + } + catch (AggregateException exception) + { + cancellationFailure = exception; + } + } + + if (cancellationFailure is not null) + { + LogShutdownCancellationFailed(_shared.Logger, cancellationFailure); } _workSignal.Signal(); @@ -1079,6 +1338,11 @@ private async Task StopAsync(CancellationToken cancellationToken) { CancelQueuedWorkItems(_shutdownCancellation.Token); } + + if (cancellationFailure is not null) + { + ExceptionDispatchInfo.Capture(cancellationFailure).Throw(); + } } private void CancelQueuedWorkItems(CancellationToken cancellationToken) @@ -1097,13 +1361,18 @@ void IDisposable.Dispose() DisposeAsync().AsTask().GetAwaiter().GetResult(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) + ThrowIfHookReentry(); + lock (_lock) { - return; + return new(_disposeTask ??= DisposeCoreAsync()); } + } + private async Task DisposeCoreAsync() + { + _disposed = 1; try { await StopAsync(CancellationToken.None).ConfigureAwait(false); @@ -1352,6 +1621,16 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal {Operation} hook failed. Persistence completed: {Committed}.")] + private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] + private static partial void LogShutdownCancellationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs new file mode 100644 index 00000000000..c10a39e92b6 --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -0,0 +1,27 @@ +namespace Orleans.Journaling; + +/// +/// Reports a hook failure after the journal operation and its state acknowledgement or reset succeeded. +/// +/// +/// The journal owner remains usable. The caller handles the failed post-persistence work using +/// the feature's durable recovery protocol, preserving the completed business operation. +/// +[GenerateSerializer] +public sealed class JournaledStatePostCommitException : Exception +{ + /// + /// Initializes a post-persistence hook failure. + /// + /// The successfully completed journal operation. + /// The failure or aggregate of failures from after hooks. + public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) + { + Operation = operation; + } + + /// Gets the successfully completed journal operation. + [Id(0)] + public JournaledStateOperation Operation { get; } +} diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs new file mode 100644 index 00000000000..1fba60d1d3e --- /dev/null +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -0,0 +1,27 @@ +namespace Orleans.Journaling; + +/// +/// Reports a prerequisite hook failure which prevented the journal storage operation. +/// +/// +/// Pending changes remain staged and safe to commit. The caller can restore the prerequisite +/// and explicitly retry persistence, or retire its owner and recover from durable state. +/// +[GenerateSerializer] +public sealed class JournaledStatePreCommitException : Exception +{ + /// + /// Initializes a prerequisite hook failure. + /// + /// The journal operation prevented by the failed prerequisite. + /// The original prerequisite failure. + public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) + : base($"Journal operation '{operation}' was prevented by a prerequisite hook failure.", innerException) + { + Operation = operation; + } + + /// Gets the journal operation which was prevented. + [Id(0)] + public JournaledStateOperation Operation { get; } +} diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2f5046a30be..7c4f4ed865a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -264,6 +264,39 @@ the operation's token before use, or deliberately supply them through a registra lifecycle ownership. Creation through the explicit-journal factory keeps failure handling independent of the ambient grain context, including when the caller subsequently enrolls the manager in a lifecycle. +## Journal operation hooks + +`IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` +registrations. Features inspect and deduplicate their registrations on the owner's logical +execution context while persistence is quiescent. Registration persists through recovery and +whole-journal deletion. The standard manager rejects mutation of the list throughout an +operation and admits at most one `IJournaledStateCaptureHook`. + +For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside +the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously +captures the registered states or starts deletion. Prerequisites cover all changes staged during +asynchronous preparation, including changes arriving while the capture hook awaits its own I/O. +Keep operation-local bookkeeping for the captured batch separate from later pending changes. + +After hooks run in list order after storage acknowledgement and state acknowledgement or reset. +They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual +operation. `JournaledStateHook` adapts synchronous and asynchronous delegates; within each phase +the synchronous delegate executes first. + +| Outcome | Owner and caller behavior | +| --- | --- | +| Before hook fails | `JournaledStatePreCommitException` retains pending state for an explicit retry after restoring the prerequisite. | +| Storage or state processing fails | The original failure fences the manager; create a fresh owner and recover the durable outcome. | +| After hook fails | `JournaledStatePostCommitException` reports successful persistence. Every remaining after hook runs, failures are aggregated, and the manager stays usable. The feature's durable recovery protocol resumes interrupted post-persistence work. | + +Hooks receive the owner's shutdown token. Caller cancellation ends the caller's wait while +the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the +owner token and drains owned work before releasing journal resources, including when cancellation +callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. +Before whole-journal deletion, the feature owner stops admission and drains its own operations; +deletion completion follows storage deletion and registered-state reset. + ## State identity and retirement Preserve state names across activations and deployments. A stream absent from the setup declarations diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 396a1692cad..84a6099c001 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,8 +219,20 @@ public partial interface IDurableValue T? Value { get; set; } } + public partial interface IJournaledStateCaptureHook : IJournaledStateHook + { + } + + public partial interface IJournaledStateHook + { + System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.ValueTask BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); + } + public partial interface IJournaledStateManager : System.IAsyncDisposable { + System.Collections.Generic.IList Hooks { get; } + long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); @@ -416,6 +428,21 @@ public sealed partial class JournalCatalogListOptions public JournalId Prefix { get { throw null; } set { } } } + public sealed partial class JournaledStateHook : IJournaledStateHook + { + public System.Action? AfterOperation { get { throw null; } init { } } + + public System.Func? AfterOperationAsync { get { throw null; } init { } } + + public System.Action? BeforeOperation { get { throw null; } init { } } + + public System.Func? BeforeOperationAsync { get { throw null; } init { } } + + System.Threading.Tasks.ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } + + System.Threading.Tasks.ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } + } + public sealed partial class JournaledStateManagerOptions { public static readonly System.TimeSpan DEFAULT_RETIREMENT_GRACE_PERIOD; @@ -424,6 +451,31 @@ public sealed partial class JournaledStateManagerOptions public System.TimeSpan RetirementGracePeriod { get { throw null; } set { } } } + public enum JournaledStateOperation + { + Write = 0, + Snapshot = 1, + Delete = 2 + } + + [GenerateSerializer] + public sealed partial class JournaledStatePostCommitException : System.Exception + { + public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } + + [Id(0)] + public JournaledStateOperation Operation { get { throw null; } } + } + + [GenerateSerializer] + public sealed partial class JournaledStatePreCommitException : System.Exception + { + public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } + + [Id(0)] + public JournaledStateOperation Operation { get { throw null; } } + } + public readonly ref partial struct JournalEntry { private readonly object _dummy; @@ -800,10 +852,66 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. where TBufferWriter : System.Buffers.IBufferWriter { } } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePostCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePostCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePostCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec + { + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + + public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } + + public global::Orleans.Journaling.JournaledStatePreCommitException ReadValue(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Serialization.WireProtocol.Field field) { throw null; } + + public void Serialize(ref global::Orleans.Serialization.Buffers.Writer writer, global::Orleans.Journaling.JournaledStatePreCommitException instance) + where TBufferWriter : System.Buffers.IBufferWriter { } + + public void WriteField(ref global::Orleans.Serialization.Buffers.Writer writer, uint fieldIdDelta, System.Type expectedType, global::Orleans.Journaling.JournaledStatePreCommitException value) + where TBufferWriter : System.Buffers.IBufferWriter { } + } + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Copier_DurableTaskCompletionSourceState : global::Orleans.Serialization.Cloning.ShallowCopier> { } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + + public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommitException input, global::Orleans.Journaling.JournaledStatePostCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } + } + + [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] + public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier + { + public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } + + public override void DeepCopy(global::Orleans.Journaling.JournaledStatePreCommitException input, global::Orleans.Journaling.JournaledStatePreCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } + } } \ No newline at end of file diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs new file mode 100644 index 00000000000..29e2d9d60e5 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -0,0 +1,697 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Fact] + public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() + { + await using var manager = CreateTestSystem().Manager; + var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.Null(field.GetValue(manager)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(manager)); + + var hook = new JournaledStateHook(); + var hooks = manager.Hooks; + hooks.Add(hook); + Assert.True(hooks.Contains(hook)); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.True(hooks.Remove(hook)); + Assert.Empty(manager.Hooks); + Assert.Throws(() => hooks.Add(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot, BlockNextAppend = !snapshot, BlockNextReplace = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; + CancellationToken ownedToken = default; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.True(token.CanBeCanceled); + ownedToken = token; + events.Add("before-sync"); + state["hook"] = 2; + }, + BeforeOperationAsync = async (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("before-async"); + }, + AfterOperation = (operation, token) => + { + Assert.Equal(expected, operation); + Assert.Equal(ownedToken, token); + Assert.Single(snapshot ? storage.Replaces : storage.Appends); + events.Add("after-sync"); + }, + AfterOperationAsync = async (_, token) => + { + Assert.Equal(ownedToken, token); + await Task.Yield(); + events.Add("after-async"); + } + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (snapshot ? storage.ReplaceEntered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture" }, events); + Assert.False(write.IsCompleted); + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "before-sync", "before-async", "capture", "ack", "after-sync", "after-async" }, events); + } + finally + { + (snapshot ? storage.ReleaseReplace : storage.ReleaseAppend).TrySetResult(); + } + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["hook"]); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var capture = new AlwaysWritingState(); + manager.RegisterStateMachine("capture", capture); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; + var failure = new IOException("Prerequisite failed."); + var after = 0; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (_, _) => throw failure, + AfterOperation = (_, _) => after++ + }); + var caught = await Assert.ThrowsAsync(() => InvokeAsync()); + Assert.Same(failure, caught.InnerException); + Assert.Equal(operation, caught.Operation); + Assert.Equal(1, state["business"]); + Assert.Equal(pendingBytes, manager.PendingWriteByteCount); + Assert.Equal(0, capture.AppendEntriesCount); + Assert.Equal(0, capture.WriteCompletedCount); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + manager.Hooks.Clear(); + await InvokeAsync(); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + + Task InvokeAsync() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask(); + } + + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHooks(JournaledStateOperation operation) + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw first }); + manager.Hooks.Add(new JournaledStateHook + { + AfterOperationAsync = async (_, _) => + { + await Task.Yield(); + events.Add("second"); + throw second; + } + }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); + Assert.Equal(operation, caught.Operation); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + Assert.Equal(operation == JournaledStateOperation.Delete + ? new[] { "second", "last" } + : new[] { "capture", "ack", "second", "last" }, events); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + + await using var replayManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, recovered.Count); + if (operation != JournaledStateOperation.Delete) + { + Assert.Equal(1, recovered["business"]); + } + + state["later"] = 2; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, state["later"]); + } + + [Fact] + public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var after = 0; + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => after++ }); + state["business"] = 1; + Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + Assert.Equal(0, after); + var fenced = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinaryHook(bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + manager.Hooks.Add(new CaptureTestHook( + (_, _) => + { + Assert.Equal(3, state["late"]); + state["owner"] = 2; + events.Add("capture-prerequisite"); + return default; + }, + (_, _) => events.Add("capture-after"))); + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = async (_, token) => + { + events.Add("ordinary-before"); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + events.Add("ordinary-complete"); + }, + AfterOperation = (_, _) => events.Add("ordinary-after") + }); + state["business"] = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before" }, events); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + state["late"] = 3; + release.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "ordinary-before", "ordinary-complete", "capture-prerequisite", "capture", "ack", "capture-after", "ordinary-after" }, events); + } + finally + { + release.TrySetResult(); + } + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["owner"]); + Assert.Equal(3, recovered["late"]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(bool captureHook) + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken ownedToken = default; + Func before = async (_, token) => + { + ownedToken = token; + entered.TrySetResult(); + await release.Task.WaitAsync(token); + state["scheduled"] = 2; + }; + Action after = (_, token) => + { + Assert.Equal(ownedToken, token); + Assert.False(token.IsCancellationRequested); + Assert.Single(storage.Appends); + completed.TrySetResult(); + }; + manager.Hooks.Add(captureHook + ? new CaptureTestHook(before, after) + : new JournaledStateHook { BeforeOperationAsync = before, AfterOperation = after }); + using var caller = new CancellationTokenSource(); + state["business"] = 1; + var write = manager.WriteStateAsync(caller.Token).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.NotEqual(caller.Token, ownedToken); + state["late"] = 3; + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(ownedToken.IsCancellationRequested); + Assert.Empty(storage.Appends); + Assert.False(completed.Task.IsCompleted); + release.TrySetResult(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, state["scheduled"]); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Count); + Assert.Equal(1, recovered["business"]); + Assert.Equal(2, recovered["scheduled"]); + Assert.Equal(3, recovered["late"]); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) + { + await using var manager = CreateTestSystem().Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Func callback = async (_, token) => + { + await Assert.ThrowsAsync(() => manager.InitializeAsync(token).AsTask()); + await Assert.ThrowsAsync(() => manager.DeleteStateAsync(token).AsTask()); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + await manager.WriteStateAsync(token); + }; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }); + var failure = await Record.ExceptionAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken) + .AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + var cause = after + ? Assert.IsType(failure).InnerException + : Assert.IsType(failure).InnerException; + Assert.Contains("same journal owner", Assert.IsType(cause).Message, StringComparison.Ordinal); + manager.Hooks.Clear(); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Hooks_FinalPrerequisiteRejectsSameOwnerReentry() + { + await using var manager = CreateTestSystem().Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + manager.Hooks.Add(new CaptureTestHook((_, token) => manager.DeleteStateAsync(token))); + var failure = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Contains("same journal owner", Assert.IsType(failure.InnerException).Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() + { + await using var manager = CreateTestSystem().Manager; + var first = new CaptureTestHook((_, _) => default); + var second = new CaptureTestHook((_, _) => default); + var ordinary = new JournaledStateHook(); + manager.Hooks.Add(first); + manager.Hooks.Add(ordinary); + Assert.Throws(() => manager.Hooks.Add(second)); + Assert.Throws(() => manager.Hooks[1] = second); + Assert.Equal(new IJournaledStateHook[] { first, ordinary }, manager.Hooks); + manager.Hooks[0] = second; + Assert.Same(second, manager.Hooks[0]); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var after = 0; + manager.Hooks.Add(new CaptureTestHook( + async (operation, token) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(1, state.Value); + entered.TrySetResult(); + await release.Task.WaitAsync(token); + }, + (operation, _) => + { + Assert.Equal(JournaledStateOperation.Delete, operation); + Assert.Equal(0, state.Value); + Assert.Equal(1, storage.DeleteCount); + after++; + })); + var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, after); + Assert.False(deletion.IsCompleted); + release.TrySetResult(); + await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, after); + Assert.Single(manager.Hooks); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) + { + var storage = new CapturingStorage { BlockNextAppend = !after }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hook = new JournaledStateHook + { + BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), + AfterOperationAsync = after ? async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } : null + }; + manager.Hooks.Add(hook); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await (after ? entered : storage.BlockedAppendStarted).Task + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Throws(() => manager.Hooks.Add(new JournaledStateHook())); + Assert.Throws(() => manager.Hooks[0] = new JournaledStateHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var drained = false; + var failure = new IOException("Cancellation callback failed."); + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = async (_, token) => + { + using var registration = token.Register(() => throw failure); + entered.TrySetResult(); + await release.Task; + drained = true; + } + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + Assert.False(drained); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Same(failure, Assert.Single(caught.InnerExceptions)); + await Assert.ThrowsAnyAsync(() => write); + await Assert.ThrowsAnyAsync(() => queued); + Assert.True(drained); + Assert.Empty(storage.Appends); + Assert.Throws(() => manager.Hooks.Clear()); + } + finally + { + release.TrySetResult(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = new IOException("First cleanup failed."); + var second = new IOException("Second cleanup failed."); + List events = []; + manager.Hooks.Add(new JournaledStateHook + { + AfterOperationAsync = async (_, token) => + { + Assert.Single(storage.Appends); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + events.Add("first"); + throw first; + } + }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw second }); + manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var shutdown = manager.DisposeAsync().AsTask(); + var concurrentShutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, concurrentShutdown); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => + write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new[] { "first", "last" }, events); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await storage.BlockedAppendStarted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(0, after); + state["second"] = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + state["third"] = 3; + var third = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Task.WhenAll(first, second, third).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(2, before); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var before = 0; + var after = 0; + manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Single(storage.Appends); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool after) + { + Exception original = after + ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) + : new JournaledStatePreCommitException(JournaledStateOperation.Snapshot, new IOException("Scheduling failed.")); + var serializer = ServiceProvider.GetRequiredService(); + var copy = serializer.Deserialize(serializer.SerializeToArray(original)); + if (after) + { + Assert.Equal(JournaledStateOperation.Delete, Assert.IsType(copy).Operation); + } + else + { + Assert.Equal(JournaledStateOperation.Snapshot, Assert.IsType(copy).Operation); + } + + Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); + Assert.Equal(original.Message, copy.Message); + } + + [Fact] + public async Task Hooks_LegacyCustomOwnerKeepsExistingOperationsAndReportsUnsupportedHooks() + { + await using IJournaledStateManager manager = new LegacyHookTestOwner(); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Throws(() => manager.Hooks); + } + + private sealed class LegacyHookTestOwner : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken = default) => default; + public ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => default; + public ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) => default; + public void RegisterStateMachine(string name, IStateMachine stateMachine) { } + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? stateMachine) + { + stateMachine = null; + return false; + } + } + + private sealed class CaptureTestHook( + Func before, + Action? after = null) : IJournaledStateCaptureHook + { + public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => + before(operation, cancellationToken); + + public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + after?.Invoke(operation, cancellationToken); + return default; + } + } + + private sealed class HookAcknowledgementState(List events) : IStateMachine + { + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } + public void Reset(JournalStreamWriter writer) { } + public void WritePendingEntries(JournalStreamWriter writer) => events.Add("capture"); + public void WriteSnapshot(JournalStreamWriter writer) => events.Add("capture"); + public void OnWriteCompleted() => events.Add("ack"); + } +} From 2c301564bff4df0ad7f877279b360f022b3b1ec2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 10:55:30 -0700 Subject: [PATCH 02/22] fix(journaling): close work admission during hook shutdown --- .../grains/journaling/runtime-behavior.md | 4 +- .../IJournaledStateManager.cs | 3 +- .../JournaledStateManager.cs | 16 +- src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 146 ++++++++++++++++++ 5 files changed, 163 insertions(+), 8 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index eff4032fb4d..00d88df1c0e 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -100,7 +100,9 @@ Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait le operation running through its actual outcome. Disposal drains owned hooks and storage before releasing journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers share this completion. Recursive initialization, persistence, or disposal on the same owner from a -hook is rejected. For deletion, the feature owner stops admission and drains feature operations before +hook is rejected. Shutdown closes work admission and cancels queued operations while the current +operation drains to its actual storage and hook outcome. For deletion, the feature owner stops +admission and drains feature operations before queuing the whole-journal reset. ## Consistency and competing writers diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4f2af34920e..4b5598cdfce 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -8,7 +8,6 @@ namespace Orleans.Journaling; /// /// The owner registers state machines and initializes the journal before using recovered state. /// State machine instances and their dependencies retain the lifetime assigned by their caller. -/// Disposing this manager stops journal processing and releases its journal resources. /// Disposal drains owned storage and hook operations before releasing resources. /// public interface IJournaledStateManager : IAsyncDisposable @@ -82,7 +81,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. - /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. + /// A storage or state-reset failure permanently fences the manager and requests deactivation of its owning grain. /// Before-hook failure reports with state retained. /// After-hook failure reports after storage deletion /// and state reset succeed. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 4a852b65396..7cd2d160607 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -263,6 +263,11 @@ private async Task WorkLoop() WorkItem workItem; lock (_lock) { + if (_shutdownCancellation.IsCancellationRequested) + { + return; + } + if (!_workQueue.TryDequeue(out var dequeuedWorkItem)) { // Wait for the queue to be signaled again. @@ -693,9 +698,9 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo private IJournaledStateCaptureHook? GetCaptureHook() { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { - if (hook is IJournaledStateCaptureHook captureHook) + if (_hooks[i] is IJournaledStateCaptureHook captureHook) { return captureHook; } @@ -710,8 +715,9 @@ private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation CurrentHookOwner.Value = this; try { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { + var hook = _hooks[i]; if (hook is IJournaledStateCaptureHook) { continue; @@ -736,11 +742,11 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, List? failures = null; try { - foreach (var hook in _hooks!) + for (var i = 0; i < _hooks!.Count; i++) { try { - await hook.AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); } catch (Exception exception) { diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 7c4f4ed865a..ce02135b40c 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -293,6 +293,8 @@ Hooks receive the owner's shutdown token. Caller cancellation ends the caller's the owned prerequisite, capture, storage, and completion phases continue. Disposal cancels the owner token and drains owned work before releasing journal resources, including when cancellation callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. +Shutdown closes work admission and cancels queued operations while an already running operation +drains to its actual storage and hook outcome. Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 29e2d9d60e5..607878d692a 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -1,5 +1,7 @@ +using System.Buffers; using System.Reflection; using Microsoft.Extensions.DependencyInjection; +using NSubstitute; using Orleans.Serialization; using Xunit; @@ -526,6 +528,150 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDrains(bool after) + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + Func callback = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + }; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperationAsync = after ? null : callback, + AfterOperationAsync = after ? callback : null + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var initializing = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var deleting = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + Assert.False(initializing.IsCompleted); + Assert.False(deleting.IsCompleted); + release.TrySetResult(); + await WaitFor(shutdown); + if (after) + { + await WaitFor(write); + } + else + { + await Assert.ThrowsAnyAsync(() => WaitFor(write)); + } + + await Assert.ThrowsAnyAsync(() => WaitFor(initializing)); + await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); + Assert.True(initializing.IsCanceled); + Assert.True(deleting.IsCanceled); + Assert.Equal(after ? 1 : 0, state.CaptureCount); + Assert.Equal(after ? 1 : 0, state.WriteCompletedCount); + Assert.Equal(after ? 1 : 0, storage.Appends.Count); + Assert.Equal(0, storage.DeleteCount); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) + { + var storage = Substitute.For(); + storage.IsCompactionRequested.Returns(snapshot); + storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => + { + call.Arg().Complete(metadata: null); + return ValueTask.CompletedTask; + }); + storage.AppendAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + storage.ReplaceAsync(Arg.Any>(), Arg.Any()) + .Returns(call => WriteAsync(call.Arg>(), call.Arg())); + var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + CancellationTokenRegistration registration = default; + var completed = false; + var after = 0; + manager.Hooks.Add(new JournaledStateHook + { + BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), + AfterOperation = (_, _) => after++ + }); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.Same(shutdown, manager.DisposeAsync().AsTask()); + Assert.False(shutdown.IsCompleted); + Assert.False(completed); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Equal(0, after); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(shutdown)); + Assert.Same(cancellationFailure, Assert.Single(caught.InnerExceptions)); + Assert.True(completed); + if (storageFails) + { + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(storageFailure, await Assert.ThrowsAsync(() => WaitFor(queued))); + } + else + { + await WaitFor(write); + await Assert.ThrowsAnyAsync(() => WaitFor(queued)); + } + + Assert.Equal(storageFails ? 0 : 1, state.WriteCompletedCount); + Assert.Equal(storageFails ? 0 : 1, after); + } + finally + { + release.TrySetResult(); + registration.Dispose(); + await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); + } + + async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token) + { + var original = bytes.ToArray(); + entered.TrySetResult(); + await release.Task; + Assert.True(token.IsCancellationRequested); + Assert.Equal(original, bytes.ToArray()); + completed = true; + if (storageFails) + { + throw storageFailure; + } + } + } + [Fact] public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() { From 8b5b3e9e59542ab61fbbb0e4dcb60c412d7a5e41 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 11:45:21 -0700 Subject: [PATCH 03/22] test(journaling): initialize storage callback barriers before registration --- test/Orleans.Journaling.Tests/StateManagerHookTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 607878d692a..332a18c51cf 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -594,6 +594,11 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra [InlineData(true, true)] public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesItsOutcome(bool snapshot, bool storageFails) { + var entered = NewSignal(); + var release = NewSignal(); + var cancellationFailure = new IOException("Cancellation callback failed."); + var storageFailure = new IOException("Actual storage failure."); + var completed = false; var storage = Substitute.For(); storage.IsCompactionRequested.Returns(snapshot); storage.ReadAsync(Arg.Any(), Arg.Any()).Returns(call => @@ -609,12 +614,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); - var entered = NewSignal(); - var release = NewSignal(); - var cancellationFailure = new IOException("Cancellation callback failed."); - var storageFailure = new IOException("Actual storage failure."); CancellationTokenRegistration registration = default; - var completed = false; var after = 0; manager.Hooks.Add(new JournaledStateHook { From 2f21229ab95e30e52e81b126faaed5dd1e918fe9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 17:07:56 -0700 Subject: [PATCH 04/22] refactor(journaling): trim operation hook API and preparation --- .../grains/journaling/runtime-behavior.md | 4 +- src/Orleans.Journaling/JournaledStateHook.cs | 37 ---------- .../JournaledStateManager.cs | 26 ++----- src/Orleans.Journaling/README.md | 4 +- .../Orleans.Journaling/Orleans.Journaling.cs | 15 ----- .../StateManagerHookTests.cs | 67 ++++++++++++------- 6 files changed, 54 insertions(+), 99 deletions(-) delete mode 100644 src/Orleans.Journaling/JournaledStateHook.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 00d88df1c0e..68398471d80 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -86,8 +86,8 @@ the captured batch separately from changes staged later. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers -share callbacks for the actual operation. adapts delegates, -executing the synchronous delegate before the asynchronous delegate in each phase. +share callbacks for the actual operation. Features implement the before and after callbacks on +their own hook, retaining feature identity and operation-local bookkeeping there. A failed prerequisite reports with pending state retained for an explicit persistence retry after the prerequisite is restored. A failed after diff --git a/src/Orleans.Journaling/JournaledStateHook.cs b/src/Orleans.Journaling/JournaledStateHook.cs deleted file mode 100644 index 31e12d601f8..00000000000 --- a/src/Orleans.Journaling/JournaledStateHook.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Adapts synchronous and asynchronous delegates to journal operation hooks. -/// -/// -/// When both delegates for a phase are supplied, the synchronous delegate runs first. -/// Asynchronous delegates await their I/O outcome before completing their returned value task. -/// Custom implementations can carry feature identity and state -/// for inspection and deduplication in the owner's hook list. -/// -public sealed class JournaledStateHook : IJournaledStateHook -{ - /// Gets the synchronous prerequisite callback. - public Action? BeforeOperation { get; init; } - - /// Gets the asynchronous prerequisite callback. - public Func? BeforeOperationAsync { get; init; } - - /// Gets the synchronous completion callback. - public Action? AfterOperation { get; init; } - - /// Gets the asynchronous completion callback. - public Func? AfterOperationAsync { get; init; } - - ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - BeforeOperation?.Invoke(operation, cancellationToken); - return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } - - ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - AfterOperation?.Invoke(operation, cancellationToken); - return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; - } -} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 7cd2d160607..f7c22804500 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -320,11 +320,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo if (hookOperation is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - var captureHook = GetCaptureHook(); - if (captureHook is null || _hooks.Count > 1) - { - await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } + var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); if (captureHook is not null) { @@ -696,30 +692,19 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - private IJournaledStateCaptureHook? GetCaptureHook() - { - for (var i = 0; i < _hooks!.Count; i++) - { - if (_hooks[i] is IJournaledStateCaptureHook captureHook) - { - return captureHook; - } - } - - return null; - } - - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { var previous = CurrentHookOwner.Value; CurrentHookOwner.Value = this; + IJournaledStateCaptureHook? captureHook = null; try { for (var i = 0; i < _hooks!.Count; i++) { var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook) + if (hook is IJournaledStateCaptureHook capture) { + captureHook = capture; continue; } @@ -728,6 +713,7 @@ private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation } cancellationToken.ThrowIfCancellationRequested(); + return captureHook; } finally { diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index ce02135b40c..938b86ad946 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -280,8 +280,8 @@ Keep operation-local bookkeeping for the captured batch separate from later pend After hooks run in list order after storage acknowledgement and state acknowledgement or reset. They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual -operation. `JournaledStateHook` adapts synchronous and asynchronous delegates; within each phase -the synchronous delegate executes first. +operation. Features implement the before and after callbacks on their own hook, retaining their +identity and operation-local bookkeeping there. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 84a6099c001..11dc0ac3001 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -428,21 +428,6 @@ public sealed partial class JournalCatalogListOptions public JournalId Prefix { get { throw null; } set { } } } - public sealed partial class JournaledStateHook : IJournaledStateHook - { - public System.Action? AfterOperation { get { throw null; } init { } } - - public System.Func? AfterOperationAsync { get { throw null; } init { } } - - public System.Action? BeforeOperation { get { throw null; } init { } } - - public System.Func? BeforeOperationAsync { get { throw null; } init { } } - - System.Threading.Tasks.ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } - - System.Threading.Tasks.ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken) { throw null; } - } - public sealed partial class JournaledStateManagerOptions { public static readonly System.TimeSpan DEFAULT_RETIREMENT_GRACE_PERIOD; diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 332a18c51cf..9e81a5e5604 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -19,7 +19,7 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(manager)); - var hook = new JournaledStateHook(); + var hook = new DelegateTestHook(); var hooks = manager.Hooks; hooks.Add(hook); Assert.True(hooks.Contains(hook)); @@ -43,7 +43,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo await manager.InitializeAsync(TestContext.Current.CancellationToken); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (operation, token) => { @@ -115,7 +115,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var pendingBytes = manager.PendingWriteByteCount; var failure = new IOException("Prerequisite failed."); var after = 0; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => throw failure, AfterOperation = (_, _) => after++ @@ -158,8 +158,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo state["business"] = 1; var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw first }); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw first }); + manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -168,7 +168,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw second; } }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); @@ -204,7 +204,7 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); await manager.InitializeAsync(TestContext.Current.CancellationToken); var after = 0; - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); Assert.Equal(0, after); @@ -234,7 +234,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar return default; }, (_, _) => events.Add("capture-after"))); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -301,7 +301,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b }; manager.Hooks.Add(captureHook ? new CaptureTestHook(before, after) - : new JournaledStateHook { BeforeOperationAsync = before, AfterOperation = after }); + : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); using var caller = new CancellationTokenSource(); state["business"] = 1; var write = manager.WriteStateAsync(caller.Token).AsTask(); @@ -346,7 +346,7 @@ public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); await manager.WriteStateAsync(token); }; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null @@ -379,7 +379,7 @@ public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() await using var manager = CreateTestSystem().Manager; var first = new CaptureTestHook((_, _) => default); var second = new CaptureTestHook((_, _) => default); - var ordinary = new JournaledStateHook(); + var ordinary = new DelegateTestHook(); manager.Hooks.Add(first); manager.Hooks.Add(ordinary); Assert.Throws(() => manager.Hooks.Add(second)); @@ -448,14 +448,15 @@ public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) state.Value = 1; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hook = new JournaledStateHook + var hook = new DelegateTestHook { BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), AfterOperationAsync = after ? async (_, _) => { entered.TrySetResult(); await release.Task; - } : null + } + : null }; manager.Hooks.Add(hook); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -463,8 +464,8 @@ public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) { await (after ? entered : storage.BlockedAppendStarted).Task .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks.Add(new JournaledStateHook())); - Assert.Throws(() => manager.Hooks[0] = new JournaledStateHook()); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); Assert.Throws(() => manager.Hooks.Remove(hook)); Assert.Throws(() => manager.Hooks.Clear()); Assert.Same(hook, Assert.Single(manager.Hooks)); @@ -491,7 +492,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -545,7 +546,7 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra entered.TrySetResult(); await release.Task; }; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null @@ -616,7 +617,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt await manager.InitializeAsync(TestContext.Current.CancellationToken); CancellationTokenRegistration registration = default; var after = 0; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ @@ -685,7 +686,7 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); List events = []; - manager.Hooks.Add(new JournaledStateHook + manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -697,8 +698,8 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() throw first; } }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => throw second }); - manager.Hooks.Add(new JournaledStateHook { AfterOperation = (_, _) => events.Add("last") }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); + manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -731,7 +732,7 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() await manager.InitializeAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; - manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -766,7 +767,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() await manager.WriteStateAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; - manager.Hooks.Add(new JournaledStateHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); @@ -832,6 +833,26 @@ public ValueTask AfterOperationAsync(JournaledStateOperation operation, Cancella } } + private sealed class DelegateTestHook : IJournaledStateHook + { + public Action? BeforeOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } + public Action? AfterOperation { get; init; } + public Func? AfterOperationAsync { get; init; } + + ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + BeforeOperation?.Invoke(operation, cancellationToken); + return BeforeOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + + ValueTask IJournaledStateHook.AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + AfterOperation?.Invoke(operation, cancellationToken); + return AfterOperationAsync?.Invoke(operation, cancellationToken) ?? default; + } + } + private sealed class HookAcknowledgementState(List events) : IStateMachine { public void ReplayEntry(JournalEntry entry, JournalReplayContext context) { } From 33749ac593c6376dcde1600ff38064a235025e0e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 17:46:43 -0700 Subject: [PATCH 05/22] fix(journaling): preserve hooks for queued persistence --- .../grains/journaling/runtime-behavior.md | 2 +- .../IJournaledStateManager.cs | 2 +- .../JournaledStateManager.cs | 8 +++ src/Orleans.Journaling/README.md | 4 +- .../StateManagerHookTests.cs | 62 +++++++++++++++++++ 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 68398471d80..7b4433ed7e1 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -75,7 +75,7 @@ Activation-scoped features coordinate prerequisites and completion through allocated list of registrations. Inspect and deduplicate feature registrations on the owner's logical execution context while persistence is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list -mutation throughout each operation. +mutation while persistence is queued or running. Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside the manager lock. At most one supplies the diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4b5598cdfce..4b31a647282 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,7 +17,7 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while - /// persistence is quiescent. Mutation during an operation is rejected. Ordinary before hooks + /// persistence is quiescent. Mutation while persistence is queued or running is rejected. Ordinary before hooks /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index f7c22804500..47784d90a86 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -813,6 +813,14 @@ private void EnsureMutationAllowed() { throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); } + + foreach (var workItem in owner._workQueue) + { + if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem) + { + throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); + } + } } private void ValidateHook(IJournaledStateHook item, int replacingIndex) diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 938b86ad946..37d2f763d6e 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -269,8 +269,8 @@ of the ambient grain context, including when the caller subsequently enrolls the `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` registrations. Features inspect and deduplicate their registrations on the owner's logical execution context while persistence is quiescent. Registration persists through recovery and -whole-journal deletion. The standard manager rejects mutation of the list throughout an -operation and admits at most one `IJournaledStateCaptureHook`. +whole-journal deletion. The standard manager rejects mutation of the list while persistence is +queued or running and admits at most one `IJournaledStateCaptureHook`. For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 9e81a5e5604..cbbca0546a5 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -436,6 +436,68 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() } } + [Theory] + [InlineData(JournaledStateOperation.Write)] + [InlineData(JournaledStateOperation.Snapshot)] + [InlineData(JournaledStateOperation.Delete)] + public async Task Hooks_MutationIsRejectedWhilePersistenceIsQueued(JournaledStateOperation operation) + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var before = 0; + var after = 0; + var hook = new CaptureTestHook( + (actual, _) => + { + Assert.Equal(operation, actual); + before++; + return default; + }, + (actual, _) => + { + Assert.Equal(operation, actual); + after++; + }); + manager.Hooks.Add(hook); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + var canceledWaiter = InvokeAsync(caller.Token); + var remainingWaiter = InvokeAsync(TestContext.Current.CancellationToken); + caller.Cancel(); + + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, storage.DeleteCount); + Assert.Equal(0, before); + Assert.False(remainingWaiter.IsCompleted); + Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); + Assert.Throws(() => manager.Hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); + Assert.Throws(() => manager.Hooks.Remove(hook)); + Assert.Throws(() => manager.Hooks.Clear()); + Assert.Same(hook, Assert.Single(manager.Hooks)); + + await Assert.ThrowsAnyAsync(() => canceledWaiter); + await remainingWaiter; + Assert.Equal(1, before); + Assert.Equal(1, after); + Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.CaptureCount); + Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); + Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); + Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); + manager.Hooks.Clear(); + Assert.Empty(manager.Hooks); + + Task InvokeAsync(CancellationToken token) => (operation == JournaledStateOperation.Delete + ? manager.DeleteStateAsync(token) + : manager.WriteStateAsync(token)).AsTask(); + }); + } + [Theory] [InlineData(false)] [InlineData(true)] From 43b25a0b356e977a87104bf3de447e2d83eed48e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 18:31:13 -0700 Subject: [PATCH 06/22] feat(journaling)!: require journal operation hook support --- .../grains/journaling/runtime-behavior.md | 2 ++ .../CompatibilitySuppressions.xml | 14 ++++++++++++ .../IJournaledStateManager.cs | 5 ++--- src/Orleans.Journaling/README.md | 2 ++ .../DurableListJournalBenchmarks.cs | 2 ++ .../JournaledJobShardDiscoveryTests.cs | 1 + .../DurableCollectionDirectWriteTests.cs | 2 ++ .../DurableListDirectWriteTests.cs | 2 ++ .../KeyedJournalingRegistrationTests.cs | 1 + .../StateManagerHookTests.cs | 22 ------------------- 10 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 7b4433ed7e1..0b0d25201b7 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -76,6 +76,8 @@ allocated list of registrations. I deduplicate feature registrations on the owner's logical execution context while persistence is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list mutation while persistence is queued or running. +Every journal owner implementation provides this list and runs its registered callbacks at the +operation boundaries below. Delegating owners forward the list to the inner owner. Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside the manager lock. At most one supplies the diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index eb687e7cf91..ed87a41fc9b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -267,6 +267,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey @@ -302,6 +309,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + P:Orleans.Journaling.IJournaledStateManager.Hooks + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 4b31a647282..ffd59075f22 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -21,10 +21,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. - /// Custom owners implement this property to support hooks. + /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// - /// The custom owner does not support hooks. - IList Hooks => throw new NotSupportedException("This journal owner does not support operation hooks."); + IList Hooks { get; } /// ValueTask IAsyncDisposable.DisposeAsync() => default; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 37d2f763d6e..45ae95fcf62 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -271,6 +271,8 @@ registrations. Features inspect and deduplicate their registrations on the owner execution context while persistence is quiescent. Registration persists through recovery and whole-journal deletion. The standard manager rejects mutation of the list while persistence is queued or running and admits at most one `IJournaledStateCaptureHook`. +Every journal owner implementation provides the hook list and invokes registered callbacks at +the operation boundaries described below. Delegating owners forward the list to their inner owner. For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index c94bd6045e2..f5931b7e85b 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,6 +126,8 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(buffer.CreateJournalStreamWriter(streamId)); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 7a9862c5cab..c6829908e9a 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,6 +727,7 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 26cf16aac46..4e46d24cde7 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,6 +193,8 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index e1cc3708672..bb3648d69fa 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,6 +92,8 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { + public IList Hooks { get; } = []; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; public void RegisterStateMachine(string name, IStateMachine state) => state.Reset(writer.CreateWriter()); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index e9822dd9557..97b65742677 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,6 +746,7 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { + public IList Hooks => inner.Hooks; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index cbbca0546a5..7cb585e922e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -859,28 +859,6 @@ public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool aft Assert.Equal(original.Message, copy.Message); } - [Fact] - public async Task Hooks_LegacyCustomOwnerKeepsExistingOperationsAndReportsUnsupportedHooks() - { - await using IJournaledStateManager manager = new LegacyHookTestOwner(); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks); - } - - private sealed class LegacyHookTestOwner : IJournaledStateManager - { - public ValueTask InitializeAsync(CancellationToken cancellationToken = default) => default; - public ValueTask WriteStateAsync(CancellationToken cancellationToken = default) => default; - public ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) => default; - public void RegisterStateMachine(string name, IStateMachine stateMachine) { } - public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? stateMachine) - { - stateMachine = null; - return false; - } - } - private sealed class CaptureTestHook( Func before, Action? after = null) : IJournaledStateCaptureHook From 659766b01433e1493bd2c9f2f5c399a1ace7735e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 10 Oct 2026 19:40:35 -0700 Subject: [PATCH 07/22] refactor(journaling): simplify hook execution and failure state --- .../grains/journaling/runtime-behavior.md | 5 +- src/Orleans.Journaling/IJournaledStateHook.cs | 4 +- .../JournaledStateManager.cs | 75 +++++-------------- .../JournaledStatePostCommitException.cs | 5 -- .../JournaledStatePreCommitException.cs | 5 -- src/Orleans.Journaling/README.md | 3 +- .../Orleans.Journaling/Orleans.Journaling.cs | 14 +--- .../StateManagerHookTests.cs | 51 ++----------- 8 files changed, 32 insertions(+), 130 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 0b0d25201b7..6b245288bb5 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -101,8 +101,9 @@ state-processing failures retain the manager's fencing and fresh-recovery behavi Hook callbacks receive the owner's shutdown token. Cancelling a caller's wait leaves the owned operation running through its actual outcome. Disposal drains owned hooks and storage before releasing journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers -share this completion. Recursive initialization, persistence, or disposal on the same owner from a -hook is rejected. Shutdown closes work admission and cancels queued operations while the current +share this completion. Hook implementations complete without calling initialization, persistence, +or disposal on their own owner. Awaiting an operation serialized behind the current callback creates +a circular dependency. Shutdown closes work admission and cancels queued operations while the current operation drains to its actual storage and hook outcome. For deletion, the feature owner stops admission and drains feature operations before queuing the whole-journal reset. diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index d0ffaae24f3..3639b6c658a 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -24,7 +24,9 @@ public enum JournaledStateOperation /// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after /// storage acknowledgement and state acknowledgement or reset, including successful writes which /// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later -/// pending changes separate. Recursive operations on the same journal owner are rejected. +/// pending changes separate. Hook implementations must complete without calling initialization, +/// persistence, or disposal on their own journal owner. Those operations are serialized behind the +/// current operation, so awaiting them from a hook would create a circular dependency. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 47784d90a86..6f8d8479d36 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -13,7 +13,6 @@ namespace Orleans.Journaling; internal partial class JournaledStateManager : IJournaledStateManager, IJournalStorageConsumer, ILifecycleParticipant, ILifecycleObserver, IDisposable { private const uint MinApplicationJournalStreamId = 8u; - private static readonly AsyncLocal CurrentHookOwner = new(); #if NET9_0_OR_GREATER private readonly Lock _lock = new(); #else @@ -172,7 +171,6 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -325,16 +323,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo if (captureHook is not null) { // Await the final prerequisite in this frame so capture follows its completion directly. - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; - try - { - await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } - finally - { - CurrentHookOwner.Value = previous; - } + await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); } _shutdownCancellation.Token.ThrowIfCancellationRequested(); @@ -694,55 +683,37 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; IJournaledStateCaptureHook? captureHook = null; - try + for (var i = 0; i < _hooks!.Count; i++) { - for (var i = 0; i < _hooks!.Count; i++) + var hook = _hooks[i]; + if (hook is IJournaledStateCaptureHook capture) { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + captureHook = capture; + continue; } cancellationToken.ThrowIfCancellationRequested(); - return captureHook; - } - finally - { - CurrentHookOwner.Value = previous; + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); } + + cancellationToken.ThrowIfCancellationRequested(); + return captureHook; } private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - var previous = CurrentHookOwner.Value; - CurrentHookOwner.Value = this; List? failures = null; - try + for (var i = 0; i < _hooks!.Count; i++) { - for (var i = 0; i < _hooks!.Count; i++) + try { - try - { - await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - catch (Exception exception) - { - (failures ??= []).Add(exception); - } + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); } - } - finally - { - CurrentHookOwner.Value = previous; } if (failures is { Count: 1 }) @@ -756,14 +727,6 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private void ThrowIfHookReentry() - { - if (ReferenceEquals(CurrentHookOwner.Value, this)) - { - throw new InvalidOperationException("Journal operation hooks cannot enqueue another operation on the same journal owner."); - } - } - private sealed class HookCollection(JournaledStateManager owner) : Collection { protected override void InsertItem(int index, IJournaledStateHook item) @@ -942,7 +905,6 @@ private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWrit public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task task; bool didEnqueue; @@ -1130,7 +1092,6 @@ private InvalidOperationException CreateRecoveryFormatException(Exception except public async ValueTask WriteStateAsync(CancellationToken cancellationToken = default) { - ThrowIfHookReentry(); cancellationToken.ThrowIfCancellationRequested(); Task pendingWrite; @@ -1307,7 +1268,6 @@ void ILifecycleParticipant.Participate(IGrainLifecycle observer private async Task StopAsync(CancellationToken cancellationToken) { - ThrowIfHookReentry(); AggregateException? cancellationFailure = null; lock (_lock) { @@ -1363,7 +1323,6 @@ void IDisposable.Dispose() public ValueTask DisposeAsync() { - ThrowIfHookReentry(); lock (_lock) { return new(_disposeTask ??= DisposeCoreAsync()); diff --git a/src/Orleans.Journaling/JournaledStatePostCommitException.cs b/src/Orleans.Journaling/JournaledStatePostCommitException.cs index c10a39e92b6..8274c8726d6 100644 --- a/src/Orleans.Journaling/JournaledStatePostCommitException.cs +++ b/src/Orleans.Journaling/JournaledStatePostCommitException.cs @@ -18,10 +18,5 @@ public sealed class JournaledStatePostCommitException : Exception public JournaledStatePostCommitException(JournaledStateOperation operation, Exception innerException) : base($"Journal operation '{operation}' completed, but a post-persistence hook failed.", innerException) { - Operation = operation; } - - /// Gets the successfully completed journal operation. - [Id(0)] - public JournaledStateOperation Operation { get; } } diff --git a/src/Orleans.Journaling/JournaledStatePreCommitException.cs b/src/Orleans.Journaling/JournaledStatePreCommitException.cs index 1fba60d1d3e..fe8d2eb79d4 100644 --- a/src/Orleans.Journaling/JournaledStatePreCommitException.cs +++ b/src/Orleans.Journaling/JournaledStatePreCommitException.cs @@ -18,10 +18,5 @@ public sealed class JournaledStatePreCommitException : Exception public JournaledStatePreCommitException(JournaledStateOperation operation, Exception innerException) : base($"Journal operation '{operation}' was prevented by a prerequisite hook failure.", innerException) { - Operation = operation; } - - /// Gets the journal operation which was prevented. - [Id(0)] - public JournaledStateOperation Operation { get; } } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 45ae95fcf62..0444dfcb9e3 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -297,7 +297,8 @@ owner token and drains owned work before releasing journal resources, including callbacks or after-hook cleanup fail. Concurrent disposal callers share that drain and its outcome. Shutdown closes work admission and cancels queued operations while an already running operation drains to its actual storage and hook outcome. -Recursive initialization, persistence, or disposal on the same owner from a hook is rejected. +Hook implementations complete without calling initialization, persistence, or disposal on their +own owner: awaiting an operation serialized behind the current callback creates a circular dependency. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 11dc0ac3001..76ff42769dd 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -447,18 +447,12 @@ public enum JournaledStateOperation public sealed partial class JournaledStatePostCommitException : System.Exception { public JournaledStatePostCommitException(JournaledStateOperation operation, System.Exception innerException) { } - - [Id(0)] - public JournaledStateOperation Operation { get { throw null; } } } [GenerateSerializer] public sealed partial class JournaledStatePreCommitException : System.Exception { public JournaledStatePreCommitException(JournaledStateOperation operation, System.Exception innerException) { } - - [Id(0)] - public JournaledStateOperation Operation { get { throw null; } } } public readonly ref partial struct JournalEntry @@ -842,7 +836,7 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Codec_JournaledStatePostCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec { - public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Codec_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePostCommitException instance) { } @@ -860,7 +854,7 @@ public void WriteField(ref global::Orleans.Serialization.Buffers. [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] public sealed partial class Codec_JournaledStatePreCommitException : global::Orleans.Serialization.Codecs.IFieldCodec, global::Orleans.Serialization.Codecs.IFieldCodec { - public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator, global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) { } + public Codec_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.IBaseCodec _baseTypeSerializer, global::Orleans.Serialization.Activators.IActivator _activator) { } public void Deserialize(ref global::Orleans.Serialization.Buffers.Reader reader, global::Orleans.Journaling.JournaledStatePreCommitException instance) { } @@ -886,8 +880,6 @@ public sealed partial class Copier_DurableTaskCompletionSourceState : global: public sealed partial class Copier_JournaledStatePostCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier { public Copier_JournaledStatePostCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } - - public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommitException input, global::Orleans.Journaling.JournaledStatePostCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } } [System.CodeDom.Compiler.GeneratedCode("OrleansCodeGen", "10.0.0.0")] @@ -896,7 +888,5 @@ public override void DeepCopy(global::Orleans.Journaling.JournaledStatePostCommi public sealed partial class Copier_JournaledStatePreCommitException : global::Orleans.Serialization.GeneratedCodeHelpers.OrleansGeneratedCodeHelper.ExceptionCopier { public Copier_JournaledStatePreCommitException(global::Orleans.Serialization.Serializers.ICodecProvider codecProvider) : base(default(Serialization.Serializers.ICodecProvider)!) { } - - public override void DeepCopy(global::Orleans.Journaling.JournaledStatePreCommitException input, global::Orleans.Journaling.JournaledStatePreCommitException output, global::Orleans.Serialization.Cloning.CopyContext context) { } } } \ No newline at end of file diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 7cb585e922e..dff2a601174 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -122,7 +122,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta }); var caught = await Assert.ThrowsAsync(() => InvokeAsync()); Assert.Same(failure, caught.InnerException); - Assert.Equal(operation, caught.Operation); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); Assert.Equal(1, state["business"]); Assert.Equal(pendingBytes, manager.PendingWriteByteCount); Assert.Equal(0, capture.AppendEntriesCount); @@ -172,7 +172,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); - Assert.Equal(operation, caught.Operation); + Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); Assert.Equal(new Exception[] { first, second }, Assert.IsType(caught.InnerException).InnerExceptions); Assert.Equal(operation == JournaledStateOperation.Delete ? new[] { "second", "last" } @@ -332,47 +332,6 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b } } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_SameOwnerReentryIsRejectedWithoutDeadlock(bool after) - { - await using var manager = CreateTestSystem().Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - Func callback = async (_, token) => - { - await Assert.ThrowsAsync(() => manager.InitializeAsync(token).AsTask()); - await Assert.ThrowsAsync(() => manager.DeleteStateAsync(token).AsTask()); - await Assert.ThrowsAsync(() => manager.DisposeAsync().AsTask()); - await manager.WriteStateAsync(token); - }; - manager.Hooks.Add(new DelegateTestHook - { - BeforeOperationAsync = after ? null : callback, - AfterOperationAsync = after ? callback : null - }); - var failure = await Record.ExceptionAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken) - .AsTask().WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - var cause = after - ? Assert.IsType(failure).InnerException - : Assert.IsType(failure).InnerException; - Assert.Contains("same journal owner", Assert.IsType(cause).Message, StringComparison.Ordinal); - manager.Hooks.Clear(); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - } - - [Fact] - public async Task Hooks_FinalPrerequisiteRejectsSameOwnerReentry() - { - await using var manager = CreateTestSystem().Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - manager.Hooks.Add(new CaptureTestHook((_, token) => manager.DeleteStateAsync(token))); - var failure = await Assert.ThrowsAsync(() => - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - Assert.Contains("same journal owner", Assert.IsType(failure.InnerException).Message, StringComparison.Ordinal); - } - [Fact] public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() { @@ -839,7 +798,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() [Theory] [InlineData(false)] [InlineData(true)] - public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool after) + public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) { Exception original = after ? new JournaledStatePostCommitException(JournaledStateOperation.Delete, new IOException("Cleanup failed.")) @@ -848,11 +807,11 @@ public void Hooks_FailureOutcomesRoundTripWithOperationAndOriginalCause(bool aft var copy = serializer.Deserialize(serializer.SerializeToArray(original)); if (after) { - Assert.Equal(JournaledStateOperation.Delete, Assert.IsType(copy).Operation); + Assert.IsType(copy); } else { - Assert.Equal(JournaledStateOperation.Snapshot, Assert.IsType(copy).Operation); + Assert.IsType(copy); } Assert.Equal(original.InnerException!.Message, Assert.IsType(copy.InnerException).Message); From 104fdab6811c042db77b1746181828a05ac47fb7 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 08:37:20 -0700 Subject: [PATCH 08/22] refactor(journaling): fix hook registration at initialization --- .../grains/journaling/runtime-behavior.md | 10 +- src/Orleans.Journaling/IJournaledStateHook.cs | 1 + .../IJournaledStateManager.cs | 4 +- .../JournaledStateManager.Hooks.cs | 122 ++++++ .../JournaledStateManager.cs | 187 ++------- src/Orleans.Journaling/README.md | 8 +- .../DurableStateManagerTests.cs | 2 +- .../StateManagerHookTests.cs | 361 +++++++++++------- 8 files changed, 387 insertions(+), 308 deletions(-) create mode 100644 src/Orleans.Journaling/JournaledStateManager.Hooks.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 6b245288bb5..4f739a24284 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -72,10 +72,9 @@ staged mutations from other calls. Activation-scoped features coordinate prerequisites and completion through . The owner exposes a stable, lazily -allocated list of registrations. Inspect and -deduplicate feature registrations on the owner's logical execution context while persistence -is quiescent. Registration survives recovery and deletion; the standard manager rejects hook-list -mutation while persistence is queued or running. +allocated list of registrations. Register and +deduplicate feature hooks during setup before initialization begins, alongside state machines. +The hook list then remains fixed through recovery, persistence and deletion. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. @@ -103,7 +102,8 @@ operation running through its actual outcome. Disposal drains owned hooks and st journal resources, including when cancellation callbacks or cleanup fail. Concurrent disposal callers share this completion. Hook implementations complete without calling initialization, persistence, or disposal on their own owner. Awaiting an operation serialized behind the current callback creates -a circular dependency. Shutdown closes work admission and cancels queued operations while the current +a circular dependency. Dependencies between journal owners also remain acyclic. +Shutdown closes work admission and cancels queued operations while the current operation drains to its actual storage and hook outcome. For deletion, the feature owner stops admission and drains feature operations before queuing the whole-journal reset. diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 3639b6c658a..29d245799d6 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -27,6 +27,7 @@ public enum JournaledStateOperation /// pending changes separate. Hook implementations must complete without calling initialization, /// persistence, or disposal on their own journal owner. Those operations are serialized behind the /// current operation, so awaiting them from a hook would create a circular dependency. +/// Dependencies between journal owners must also remain acyclic. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index ffd59075f22..77149dc9500 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -16,8 +16,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// Gets the mutable, lazily allocated list of journal operation hooks. /// /// - /// Inspect, add, remove, and deduplicate hooks on the owner's logical execution context while - /// persistence is quiescent. Mutation while persistence is queued or running is rejected. Ordinary before hooks + /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. + /// Registration remains fixed for this manager's initialized lifetime. Ordinary before hooks /// and all after hooks execute in list order. The optional single /// supplies the final prerequisite. /// Registration is independent of state-machine registration and persists through recovery and deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs new file mode 100644 index 00000000000..1c9502a772f --- /dev/null +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -0,0 +1,122 @@ +using System.Collections.ObjectModel; +using System.Runtime.ExceptionServices; + +namespace Orleans.Journaling; + +internal partial class JournaledStateManager +{ + private HookCollection? _hooks; + + public IList Hooks + { + get + { + lock (_lock) + { + return _hooks ??= new(this); + } + } + } + + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + IJournaledStateCaptureHook? captureHook = null; + for (var i = 0; i < _hooks!.Count; i++) + { + var hook = _hooks[i]; + if (hook is IJournaledStateCaptureHook capture) + { + captureHook = capture; + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + + cancellationToken.ThrowIfCancellationRequested(); + return captureHook; + } + + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + { + List? failures = null; + for (var i = 0; i < _hooks!.Count; i++) + { + try + { + await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); + } + catch (Exception exception) + { + (failures ??= []).Add(exception); + } + } + + if (failures is { Count: 1 }) + { + ExceptionDispatchInfo.Capture(failures[0]).Throw(); + } + + if (failures is not null) + { + throw new AggregateException(failures); + } + } + + private sealed class HookCollection(JournaledStateManager owner) : Collection + { + protected override void InsertItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + ValidateHook(item, replacingIndex: -1); + base.InsertItem(index, item); + } + } + + protected override void SetItem(int index, IJournaledStateHook item) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + ValidateHook(item, index); + base.SetItem(index, item); + } + } + + protected override void RemoveItem(int index) + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + base.RemoveItem(index); + } + } + + protected override void ClearItems() + { + lock (owner._lock) + { + owner.EnsureRegistrationAllowed(); + base.ClearItems(); + } + } + + private void ValidateHook(IJournaledStateHook item, int replacingIndex) + { + ArgumentNullException.ThrowIfNull(item); + if (item is IJournaledStateCaptureHook) + { + for (var i = 0; i < Count; i++) + { + if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) + { + throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); + } + } + } + } + } +} diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6f8d8479d36..0260219f6be 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.ExceptionServices; @@ -36,19 +35,6 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private bool _migrationSnapshotRequired; private int _disposed; private Task? _disposeTask; - private HookCollection? _hooks; - private bool _hookOperationRunning; - - public IList Hooks - { - get - { - lock (_lock) - { - return _hooks ??= new(this); - } - } - } public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) : this(shared, CreateStorage(storageProvider, CreateJournalId(grainContext)), grainContext) @@ -143,7 +129,7 @@ protected void EnsureRegistrationAllowed() _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is not null) { - throw new InvalidOperationException("New states cannot be registered after journaled state manager initialization has begun."); + throw new InvalidOperationException("Journal registrations cannot be changed after journaled state manager initialization has begun."); } } } @@ -273,7 +259,6 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; - _hookOperationRunning = workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -599,11 +584,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } storageActivity?.SetStatus(ActivityStatusCode.Ok); - lock (_lock) - { - _hookOperationRunning = false; - workItem.SetResult(); - } + workItem.SetResult(); } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { @@ -620,13 +601,9 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); } - lock (_lock) - { - _hookOperationRunning = false; - workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(hookOperation.Value, exception) - : new JournaledStatePreCommitException(hookOperation.Value, exception)); - } + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(hookOperation.Value, exception) + : new JournaledStatePreCommitException(hookOperation.Value, exception)); } catch (Exception exception) { @@ -660,11 +637,6 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } finally { - lock (_lock) - { - _hookOperationRunning = false; - } - storageActivity?.Dispose(); } } @@ -681,127 +653,6 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - IJournaledStateCaptureHook? captureHook = null; - for (var i = 0; i < _hooks!.Count; i++) - { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - - cancellationToken.ThrowIfCancellationRequested(); - return captureHook; - } - - private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - List? failures = null; - for (var i = 0; i < _hooks!.Count; i++) - { - try - { - await _hooks[i].AfterOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - catch (Exception exception) - { - (failures ??= []).Add(exception); - } - } - - if (failures is { Count: 1 }) - { - ExceptionDispatchInfo.Capture(failures[0]).Throw(); - } - - if (failures is not null) - { - throw new AggregateException(failures); - } - } - - private sealed class HookCollection(JournaledStateManager owner) : Collection - { - protected override void InsertItem(int index, IJournaledStateHook item) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - ValidateHook(item, replacingIndex: -1); - base.InsertItem(index, item); - } - } - - protected override void SetItem(int index, IJournaledStateHook item) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - ValidateHook(item, index); - base.SetItem(index, item); - } - } - - protected override void RemoveItem(int index) - { - lock (owner._lock) - { - EnsureMutationAllowed(); - base.RemoveItem(index); - } - } - - protected override void ClearItems() - { - lock (owner._lock) - { - EnsureMutationAllowed(); - base.ClearItems(); - } - } - - private void EnsureMutationAllowed() - { - ObjectDisposedException.ThrowIf(owner._disposed != 0, owner); - owner._shutdownCancellation.Token.ThrowIfCancellationRequested(); - owner.ThrowIfFenced(); - if (owner._hookOperationRunning) - { - throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); - } - - foreach (var workItem in owner._workQueue) - { - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem or DeleteStateWorkItem) - { - throw new InvalidOperationException("Journal operation hooks can be changed only while persistence is quiescent."); - } - } - } - - private void ValidateHook(IJournaledStateHook item, int replacingIndex) - { - ArgumentNullException.ThrowIfNull(item); - if (item is IJournaledStateCaptureHook) - { - for (var i = 0; i < Count; i++) - { - if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) - { - throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); - } - } - } - } - } - private void Fence(Exception exception) { lock (_lock) @@ -1325,21 +1176,37 @@ public ValueTask DisposeAsync() { lock (_lock) { - return new(_disposeTask ??= DisposeCoreAsync()); + if (_disposeTask is null) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _disposeTask = completion.Task; + _ = DisposeCoreAsync(completion); + } + + return new(_disposeTask); } } - private async Task DisposeCoreAsync() + private async Task DisposeCoreAsync(TaskCompletionSource completion) { _disposed = 1; try { - await StopAsync(CancellationToken.None).ConfigureAwait(false); + try + { + await StopAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + _shutdownCancellation.Dispose(); + _journalWriter.Dispose(); + } + + completion.SetResult(); } - finally + catch (Exception exception) { - _shutdownCancellation.Dispose(); - _journalWriter.Dispose(); + completion.SetException(exception); } } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 0444dfcb9e3..2d7f0f36200 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -267,10 +267,9 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` -registrations. Features inspect and deduplicate their registrations on the owner's logical -execution context while persistence is quiescent. Registration persists through recovery and -whole-journal deletion. The standard manager rejects mutation of the list while persistence is -queued or running and admits at most one `IJournaledStateCaptureHook`. +registrations. Features register and deduplicate their hooks during setup before initialization +begins, alongside state-machine registration. The hook list then remains fixed through recovery, +persistence and whole-journal deletion. The standard manager admits at most one `IJournaledStateCaptureHook`. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. @@ -299,6 +298,7 @@ Shutdown closes work admission and cancels queued operations while an already ru drains to its actual storage and hook outcome. Hook implementations complete without calling initialization, persistence, or disposal on their own owner: awaiting an operation serialized behind the current callback creates a circular dependency. +Dependencies between journal owners also remain acyclic. Before whole-journal deletion, the feature owner stops admission and drains its own operations; deletion completion follows storage deletion and registered-state reset. diff --git a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs index d1b0b4b797f..e85899b2bdb 100644 --- a/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/DurableStateManagerTests.cs @@ -592,7 +592,7 @@ private static void AssertIncompatible( private static void AssertAdmissionRejected(Action action) { var error = Assert.Throws(action); - Assert.Equal("New states cannot be registered after journaled state manager initialization has begun.", error.Message); + Assert.Equal("Journal registrations cannot be changed after journaled state manager initialization has begun.", error.Message); } private static void AssertFenced(InvalidOperationException error, IOException cause) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index dff2a601174..6c49ce2ede1 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -15,9 +15,6 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await using var manager = CreateTestSystem().Manager; var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Null(field.GetValue(manager)); var hook = new DelegateTestHook(); var hooks = manager.Hooks; @@ -28,6 +25,67 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() Assert.True(hooks.Remove(hook)); Assert.Empty(manager.Hooks); Assert.Throws(() => hooks.Add(null!)); + hooks.Add(hook); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.Throws(() => hooks.Clear()); + + await using var withoutHooks = CreateTestSystem().Manager; + await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); + await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Null(field.GetValue(withoutHooks)); + } + + [Fact] + public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFailure() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + var failure = new IOException("Recovery failed."); + var fail = true; + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var hook = new DelegateTestHook(); + var hooks = manager.Hooks; + hooks.Add(hook); + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + Assert.Throws(() => hooks.Add(new DelegateTestHook())); + Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => hooks[0] = new DelegateTestHook()); + Assert.Throws(() => hooks.Remove(hook)); + Assert.Throws(() => hooks.Clear()); + await WaitFor(entered.Task); + Assert.False(initialization.IsCompleted); + release.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); + Assert.Throws(() => hooks.Clear()); + fail = false; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Throws(() => hooks.Clear()); + Assert.Same(hook, Assert.Single(hooks)); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + if (fail) + { + throw failure; + } + + consumer.Complete(metadata: null); + } } [Theory] @@ -40,7 +98,6 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; manager.Hooks.Add(new DelegateTestHook @@ -74,6 +131,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo events.Add("after-async"); } }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -110,16 +168,23 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var capture = new AlwaysWritingState(); manager.RegisterStateMachine("capture", capture); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; - var pendingBytes = manager.PendingWriteByteCount; var failure = new IOException("Prerequisite failed."); + var fail = true; var after = 0; manager.Hooks.Add(new DelegateTestHook { - BeforeOperation = (_, _) => throw failure, + BeforeOperation = (_, _) => + { + if (fail) + { + throw failure; + } + }, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; + var pendingBytes = manager.PendingWriteByteCount; var caught = await Assert.ThrowsAsync(() => InvokeAsync()); Assert.Same(failure, caught.InnerException); Assert.Contains($"'{operation}'", caught.Message, StringComparison.Ordinal); @@ -131,8 +196,9 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta Assert.Empty(storage.Replaces); Assert.Equal(0, storage.DeleteCount); Assert.Equal(0, after); - manager.Hooks.Clear(); + fail = false; await InvokeAsync(); + Assert.Equal(1, after); Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); @@ -154,21 +220,34 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state["business"] = 1; var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw first }); + var fail = true; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperation = (_, _) => + { + if (fail) + { + throw first; + } + } + }); manager.Hooks.Add(new DelegateTestHook { AfterOperationAsync = async (_, _) => { await Task.Yield(); events.Add("second"); - throw second; + if (fail) + { + throw second; + } } }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["business"] = 1; var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) : manager.WriteStateAsync(TestContext.Current.CancellationToken)).AsTask()); @@ -179,7 +258,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo : new[] { "capture", "ack", "second", "last" }, events); Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.Count); Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - manager.Hooks.Clear(); + fail = false; await using var replayManager = CreateTestSystem(storage).Manager; var recovered = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); @@ -202,9 +281,9 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() var storage = new CapturingStorage { NextAppendException = failure }; await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var after = 0; manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); Assert.Equal(0, after); @@ -222,7 +301,6 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); manager.Hooks.Add(new CaptureTestHook( @@ -245,6 +323,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar }, AfterOperation = (_, _) => events.Add("ordinary-after") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -280,7 +359,6 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -302,6 +380,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b manager.Hooks.Add(captureHook ? new CaptureTestHook(before, after) : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; var write = manager.WriteStateAsync(caller.Token).AsTask(); @@ -356,15 +435,19 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var deleting = false; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; manager.Hooks.Add(new CaptureTestHook( async (operation, token) => { + if (!deleting) + { + Assert.Equal(JournaledStateOperation.Write, operation); + return; + } + Assert.Equal(JournaledStateOperation.Delete, operation); Assert.Equal(1, state.Value); entered.TrySetResult(); @@ -372,11 +455,20 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() }, (operation, _) => { + if (!deleting) + { + return; + } + Assert.Equal(JournaledStateOperation.Delete, operation); Assert.Equal(0, state.Value); Assert.Equal(1, storage.DeleteCount); after++; })); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + deleting = true; var deletion = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -395,120 +487,11 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() } } - [Theory] - [InlineData(JournaledStateOperation.Write)] - [InlineData(JournaledStateOperation.Snapshot)] - [InlineData(JournaledStateOperation.Delete)] - public async Task Hooks_MutationIsRejectedWhilePersistenceIsQueued(JournaledStateOperation operation) - { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var storage = new CapturingStorage { IsCompactionRequested = operation == JournaledStateOperation.Snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new LifecycleState(); - manager.RegisterStateMachine("state", state); - var before = 0; - var after = 0; - var hook = new CaptureTestHook( - (actual, _) => - { - Assert.Equal(operation, actual); - before++; - return default; - }, - (actual, _) => - { - Assert.Equal(operation, actual); - after++; - }); - manager.Hooks.Add(hook); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - using var caller = new CancellationTokenSource(); - var canceledWaiter = InvokeAsync(caller.Token); - var remainingWaiter = InvokeAsync(TestContext.Current.CancellationToken); - caller.Cancel(); - - Assert.Equal(0, state.CaptureCount); - Assert.Equal(0, storage.DeleteCount); - Assert.Equal(0, before); - Assert.False(remainingWaiter.IsCompleted); - Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); - Assert.Throws(() => manager.Hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); - Assert.Throws(() => manager.Hooks.Remove(hook)); - Assert.Throws(() => manager.Hooks.Clear()); - Assert.Same(hook, Assert.Single(manager.Hooks)); - - await Assert.ThrowsAnyAsync(() => canceledWaiter); - await remainingWaiter; - Assert.Equal(1, before); - Assert.Equal(1, after); - Assert.Equal(operation == JournaledStateOperation.Delete ? 0 : 1, state.CaptureCount); - Assert.Equal(operation == JournaledStateOperation.Write ? 1 : 0, storage.Appends.Count); - Assert.Equal(operation == JournaledStateOperation.Snapshot ? 1 : 0, storage.Replaces.Count); - Assert.Equal(operation == JournaledStateOperation.Delete ? 1 : 0, storage.DeleteCount); - manager.Hooks.Clear(); - Assert.Empty(manager.Hooks); - - Task InvokeAsync(CancellationToken token) => (operation == JournaledStateOperation.Delete - ? manager.DeleteStateAsync(token) - : manager.WriteStateAsync(token)).AsTask(); - }); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_MutationIsRejectedThroughoutOwnedOperation(bool after) - { - var storage = new CapturingStorage { BlockNextAppend = !after }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var hook = new DelegateTestHook - { - BeforeOperation = (_, _) => Assert.Throws(() => manager.Hooks.Clear()), - AfterOperationAsync = after ? async (_, _) => - { - entered.TrySetResult(); - await release.Task; - } - : null - }; - manager.Hooks.Add(hook); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - try - { - await (after ? entered : storage.BlockedAppendStarted).Task - .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - Assert.Throws(() => manager.Hooks.Add(new DelegateTestHook())); - Assert.Throws(() => manager.Hooks[0] = new DelegateTestHook()); - Assert.Throws(() => manager.Hooks.Remove(hook)); - Assert.Throws(() => manager.Hooks.Clear()); - Assert.Same(hook, Assert.Single(manager.Hooks)); - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - manager.Hooks.Clear(); - Assert.Empty(manager.Hooks); - } - finally - { - release.TrySetResult(); - storage.ReleaseAppend.TrySetResult(); - } - } - [Fact] public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() { var storage = new CapturingStorage(); var manager = CreateTestSystem(storage).Manager; - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; @@ -523,6 +506,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() drained = true; } }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -559,7 +543,6 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra var manager = CreateTestSystem(storage).Manager; var state = new LifecycleState(); manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var entered = NewSignal(); var release = NewSignal(); Func callback = async (_, _) => @@ -572,6 +555,7 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -635,7 +619,6 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt var manager = CreateTestSystem(storage).Manager; var state = new LifecycleState(); manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); CancellationTokenRegistration registration = default; var after = 0; manager.Hooks.Add(new DelegateTestHook @@ -643,6 +626,7 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -695,13 +679,56 @@ async ValueTask WriteAsync(ReadOnlySequence bytes, CancellationToken token } [Fact] - public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() { var storage = new CapturingStorage(); var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + Task? reenteredDisposal = null; + var callbackCount = 0; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, token) => + { + using var registration = token.Register(() => + { + callbackCount++; + reenteredDisposal = manager.DisposeAsync().AsTask(); + }); + entered.TrySetResult(); + await release.Task; + } + }); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var disposal = manager.DisposeAsync().AsTask(); + Assert.Equal(1, callbackCount); + Assert.Same(disposal, reenteredDisposal); + Assert.False(disposal.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, disposal)); + Assert.Same(disposal, manager.DisposeAsync().AsTask()); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + await manager.DisposeAsync(); + } + } + + [Fact] + public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() + { + var storage = new CapturingStorage(); + var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var first = new IOException("First cleanup failed."); @@ -721,6 +748,8 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try { @@ -750,10 +779,10 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() var storage = new CapturingStorage { BlockNextAppend = true }; await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -777,18 +806,78 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() } } + [Fact] + public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var after = 0; + manager.Hooks.Add(new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + if (++after == 1) + { + entered.TrySetResult(); + await release.Task; + } + } + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state["first"] = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.Single(storage.Appends); + Assert.False(first.IsCompleted); + state["later"] = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(next.IsCompleted); + + await using (var beforeNext = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", beforeNext, CreateDictionaryCodec()); + await beforeNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(recovered).Value); + } + + release.TrySetResult(); + await WaitFor(Task.WhenAll(first, next)); + Assert.Equal(2, after); + Assert.Equal(2, storage.Appends.Count); + await using var afterNext = CreateTestSystem(storage).Manager; + var persisted = new DurableDictionary("state", afterNext, CreateDictionaryCodec()); + await afterNext.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, persisted.Count); + Assert.Equal(1, persisted["first"]); + Assert.Equal(2, persisted["later"]); + } + finally + { + release.TrySetResult(); + } + } + [Fact] public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new DurableValue("state", manager, CreateValueCodec()); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - state.Value = 1; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); var before = 0; var after = 0; manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + before = 0; + after = 0; await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); From 16843f0438bc497e3c2778ae8ff70332517fdab1 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 08:39:57 -0700 Subject: [PATCH 09/22] refactor(journaling): name the persistence operation type --- src/Orleans.Journaling/JournaledStateManager.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 0260219f6be..5bf644a328c 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -280,7 +280,7 @@ private async Task WorkLoop() storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); } - JournaledStateOperation? hookOperation = null; + JournaledStateOperation? operationType = null; var beforeHookRunning = false; var afterHookRunning = false; try @@ -292,7 +292,7 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } - hookOperation = workItem switch + operationType = workItem switch { AppendJournalWorkItem or WriteSnapshotWorkItem => workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested @@ -300,7 +300,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; - if (hookOperation is { } operation && _hooks is { Count: > 0 }) + if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); @@ -324,7 +324,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. - var isSnapshot = hookOperation == JournaledStateOperation.Snapshot; + var isSnapshot = operationType == JournaledStateOperation.Snapshot; var operationLabel = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; @@ -571,7 +571,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } } - if (hookOperation is { } completedOperation && _hooks is { Count: > 0 }) + if (operationType is { } completedOperation && _hooks is { Count: > 0 }) { afterHookRunning = true; await InvokeAfterHooksAsync(completedOperation, _shutdownCancellation.Token).ConfigureAwait(true); @@ -594,7 +594,7 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo continue; } - LogOperationHookFailed(_shared.Logger, exception, hookOperation!.Value, afterHookRunning); + LogOperationHookFailed(_shared.Logger, exception, operationType!.Value, afterHookRunning); storageActivity?.SetStatus(ActivityStatusCode.Error, "Journal operation hook failed."); if (recordQueueDuration && queueOperation is not null) { @@ -602,8 +602,8 @@ workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCo } workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(hookOperation.Value, exception) - : new JournaledStatePreCommitException(hookOperation.Value, exception)); + ? new JournaledStatePostCommitException(operationType.Value, exception) + : new JournaledStatePreCommitException(operationType.Value, exception)); } catch (Exception exception) { From a3523e77ffbfa0e5d4b9305a1ad75315c0ba3586 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:15:59 -0700 Subject: [PATCH 10/22] refactor(journaling): select explicit snapshot work directly --- src/Orleans.Journaling/JournaledStateManager.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 5bf644a328c..a5a5314da64 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -294,8 +294,9 @@ private async Task WorkLoop() operationType = workItem switch { - AppendJournalWorkItem or WriteSnapshotWorkItem => - workItem is WriteSnapshotWorkItem || _migrationSnapshotRequired || _storage.IsCompactionRequested + WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, + AppendJournalWorkItem => + _migrationSnapshotRequired || _storage.IsCompactionRequested ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null From 363c04831cb4a698e4507b27ae1543b84f395e9a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:17:06 -0700 Subject: [PATCH 11/22] refactor(journaling): guard snapshot selection with when --- src/Orleans.Journaling/JournaledStateManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index a5a5314da64..d59c7546eb1 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -295,9 +295,8 @@ private async Task WorkLoop() operationType = workItem switch { WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem => - _migrationSnapshotRequired || _storage.IsCompactionRequested - ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write, + AppendJournalWorkItem when _migrationSnapshotRequired || _storage.IsCompactionRequested => JournaledStateOperation.Snapshot, + AppendJournalWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; From 07be377d0a55834846c85e86f15049e035be0431 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:17:56 -0700 Subject: [PATCH 12/22] refactor(journaling): centralize snapshot requirement --- src/Orleans.Journaling/JournaledStateManager.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index d59c7546eb1..36e2d3767ff 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -100,6 +100,8 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; + private bool IsSnapshotRequired => _migrationSnapshotRequired || _storage.IsCompactionRequested; + internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) @@ -295,7 +297,7 @@ private async Task WorkLoop() operationType = workItem switch { WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem when _migrationSnapshotRequired || _storage.IsCompactionRequested => JournaledStateOperation.Snapshot, + AppendJournalWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, AppendJournalWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null @@ -951,7 +953,7 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; + var isSnapshot = IsSnapshotRequired; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot ? EnqueueOrGetPendingWorkItem(out didEnqueue) From 46f1e99d8a9f8382bbdaede3f434129bbdec27e6 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 09:45:18 -0700 Subject: [PATCH 13/22] refactor(journaling): coalesce pending journal writes --- .../grains/journaling/runtime-behavior.md | 5 +- .../JournaledStateManager.cs | 58 ++++++------- src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 84 +++++++++++++++++++ 4 files changed, 119 insertions(+), 30 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 4f739a24284..d23ad5f0141 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -54,7 +54,10 @@ Durable collections encode their operation before applying it to the in-memory c - **Append** adds the encoded operation batch atomically. - **Snapshot replacement** writes the current state of every registered stream and atomically publishes it as the new journal generation. -Concurrent calls made while the same kind of write is queued can share that queued operation. Each caller observes its completion or failure. Calls made after a storage operation starts are processed by a later operation. +Concurrent calls made while a write is queued share that queued operation. A snapshot requirement +from any caller upgrades the queued write to a snapshot and remains set until that operation runs. +Each caller observes its completion or failure. Calls made after a storage operation starts are +processed by a later operation with its own snapshot requirement. > [!IMPORTANT] > In-memory mutation is visible before storage acknowledgement. Return success to a caller only after the required `WriteStateAsync` completes. Recovery reconstructs durable state in a new activation. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 36e2d3767ff..f685a41c337 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -32,7 +32,8 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - private bool _migrationSnapshotRequired; + // Recovery format changes require a snapshot; queued writes retain their own snapshot requests. + private bool _snapshotRequired; private int _disposed; private Task? _disposeTask; @@ -100,7 +101,7 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal string WriteJournalFormatKey => _shared.JournalFormatKey; - private bool IsSnapshotRequired => _migrationSnapshotRequired || _storage.IsCompactionRequested; + private bool IsSnapshotRequired => _snapshotRequired || _storage.IsCompactionRequested; internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; @@ -171,7 +172,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def _workLoop = Start(); } - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -287,7 +288,7 @@ private async Task WorkLoop() var afterHookRunning = false; try { - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + if (workItem is WriteStateWorkItem && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) { throw new InvalidOperationException( @@ -296,9 +297,8 @@ private async Task WorkLoop() operationType = workItem switch { - WriteSnapshotWorkItem => JournaledStateOperation.Snapshot, - AppendJournalWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, - AppendJournalWorkItem => JournaledStateOperation.Write, + WriteStateWorkItem write when write.SnapshotRequired || IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, DeleteStateWorkItem => JournaledStateOperation.Delete, _ => null }; @@ -321,8 +321,7 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case AppendJournalWorkItem: - case WriteSnapshotWorkItem: + case WriteStateWorkItem writeState: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. @@ -332,9 +331,9 @@ private async Task WorkLoop() : JournalingInstruments.OperationAppend; if (isSnapshot) { - var compactionReason = workItem is WriteSnapshotWorkItem + var compactionReason = writeState.SnapshotRequired ? JournalingInstruments.CompactionReasonUserSnapshot - : _migrationSnapshotRequired + : _snapshotRequired ? JournalingInstruments.CompactionReasonMigration : JournalingInstruments.CompactionReasonStorageRequested; _shared.Instruments.OnCompactionTriggered(compactionReason); @@ -358,7 +357,7 @@ private async Task WorkLoop() RetireOrResurectStates(); } - if (_migrationSnapshotRequired) + if (_snapshotRequired) { ThrowIfMigrationBlockedByRetiredStates(); } @@ -506,7 +505,7 @@ private async Task WorkLoop() if (isSnapshot) { - _migrationSnapshotRequired = false; + _snapshotRequired = false; } } } @@ -764,7 +763,7 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); - task = EnqueueOrGetPendingWorkItem(out didEnqueue); + task = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -839,7 +838,7 @@ private async Task RecoverAsync(CancellationToken cancellationToken) private void ResetForRecovery() { _journalWriter.Reset(); - _migrationSnapshotRequired = false; + _snapshotRequired = false; _statesMap.Clear(); _statesMap[StateDirectory.Id] = _journalStreamDirectory; _statesMap[RetiredStateTracker.Id] = _retirementTracker; @@ -910,7 +909,7 @@ private void ProcessRecoveryBuffer(JournalBufferReader buffer, IJournalMetadata? { if (!string.Equals(journalFormatKey, _shared.JournalFormatKey, StringComparison.Ordinal)) { - _migrationSnapshotRequired = true; + _snapshotRequired = true; } var journalFormat = GetJournalFormat(journalFormatKey); @@ -955,9 +954,9 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def ThrowIfStateOperationsUnavailable(); var isSnapshot = IsSnapshotRequired; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - pendingWrite = isSnapshot - ? EnqueueOrGetPendingWorkItem(out didEnqueue) - : EnqueueOrGetPendingWorkItem(out didEnqueue); + var workItem = EnqueueOrGetPendingWorkItem(out didEnqueue); + workItem.SnapshotRequired |= isSnapshot; + pendingWrite = workItem.Task; } if (didEnqueue) @@ -1058,17 +1057,17 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) } } - private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) + private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { foreach (var workItem in _workQueue) { - if (workItem.GetType() != typeof(TWorkItem)) + if (workItem is not TWorkItem pending) { continue; } - if (workItem is AppendJournalWorkItem or WriteSnapshotWorkItem + if (workItem is WriteStateWorkItem && workItem.DeletionGeneration != _deletionGeneration) { continue; @@ -1077,7 +1076,7 @@ private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) workItem.RecordTraceContext(); workItem.AddCaller(); didEnqueue = false; - return workItem.Task; + return pending; } var newWorkItem = new TWorkItem(); @@ -1086,7 +1085,7 @@ private Task EnqueueOrGetPendingWorkItem(out bool didEnqueue) newWorkItem.RecordTraceContext(); _workQueue.Enqueue(newWorkItem); didEnqueue = true; - return newWorkItem.Task; + return newWorkItem; } private void BindState(string name, uint id) @@ -1293,17 +1292,18 @@ public void RecordTraceContext() private static string? GetStorageQueueOperation(WorkItem workItem) => workItem switch { - AppendJournalWorkItem => JournalingInstruments.OperationAppend, - WriteSnapshotWorkItem => JournalingInstruments.OperationSnapshot, + WriteStateWorkItem { SnapshotRequired: true } => JournalingInstruments.OperationSnapshot, + WriteStateWorkItem => JournalingInstruments.OperationAppend, DeleteStateWorkItem => JournalingInstruments.OperationDelete, _ => null }; private sealed class InitializeWorkItem : WorkItem; - private sealed class AppendJournalWorkItem : WorkItem; - - private sealed class WriteSnapshotWorkItem : WorkItem; + private sealed class WriteStateWorkItem : WorkItem + { + public bool SnapshotRequired { get; set; } + } private sealed class DeleteStateWorkItem : WorkItem; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2d7f0f36200..745c04d707a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -283,6 +283,8 @@ After hooks run in list order after storage acknowledgement and state acknowledg They also run for a successful zero-byte write. Coalesced callers share the hooks for their actual operation. Features implement the before and after callbacks on their own hook, retaining their identity and operation-local bookkeeping there. +Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that +queued operation to a snapshot, even if the storage compaction signal clears before it runs. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 6c49ce2ede1..a9a455cd91d 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -773,6 +773,90 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() } } + [Fact] + public async Task Hooks_QueuedWritesCoalesceAndRetainSnapshotRequirement() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = true; + state.Value = 2; + var snapshot = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + state.Value = 3; + var anotherAppend = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + Assert.Empty(before); + await Task.WhenAll(append, snapshot, anotherAppend); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, recovered.Value); + }); + } + + [Fact] + public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() + { + var storage = new CapturingStorage { IsCompactionRequested = true, BlockNextReplace = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (operation, _) => before.Add(operation), + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.ReplaceEntered.Task); + state.Value = 2; + var second = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.IsCompactionRequested = false; + Assert.Empty(storage.Replaces); + Assert.Equal([JournaledStateOperation.Snapshot], before); + Assert.Empty(after); + storage.ReleaseReplace.TrySetResult(); + await WaitFor(Task.WhenAll(first, second)); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Equal(2, storage.Replaces.Count); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + storage.ReleaseReplace.TrySetResult(); + } + } + [Fact] public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() { From 6b77e5578386c7d914b7619f0761e7981200205a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 13:08:13 -0700 Subject: [PATCH 14/22] refactor(journaling): consume pending snapshot requirements --- .../grains/journaling/runtime-behavior.md | 6 +- .../JournaledStateManager.cs | 142 +++++++++--------- src/Orleans.Journaling/README.md | 4 +- .../CodecRecoveryTests.cs | 24 +-- .../StateManagerHookTests.cs | 111 ++++++++++++++ 5 files changed, 199 insertions(+), 88 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index d23ad5f0141..36435a498e7 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -55,9 +55,11 @@ Durable collections encode their operation before applying it to the in-memory c - **Snapshot replacement** writes the current state of every registered stream and atomically publishes it as the new journal generation. Concurrent calls made while a write is queued share that queued operation. A snapshot requirement -from any caller upgrades the queued write to a snapshot and remains set until that operation runs. +from any caller upgrades pending work to a snapshot. The manager consumes the requirement when +selecting a write and restores it if a prerequisite hook fails. Storage acknowledgement leaves +newer pending requirements untouched. Each caller observes its completion or failure. Calls made after a storage operation starts are -processed by a later operation with its own snapshot requirement. +processed by a later operation. > [!IMPORTANT] > In-memory mutation is visible before storage acknowledgement. Return success to a caller only after the required `WriteStateAsync` completes. Recovery reconstructs durable state in a new activation. diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index f685a41c337..89791f81443 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -32,7 +32,7 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private ManagerState _state; private long _deletionGeneration; private Exception? _failure; - // Recovery format changes require a snapshot; queued writes retain their own snapshot requests. + // A snapshot requirement not yet claimed by a write. Consume under the lock before preparation awaits. private bool _snapshotRequired; private int _disposed; private Task? _disposeTask; @@ -265,43 +265,44 @@ private async Task WorkLoop() } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); - var queueOperation = GetStorageQueueOperation(workItem); - var queueDuration = queueOperation is null - ? default - : _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + string? queueOperation = null; + TimeSpan queueDuration = default; var recordQueueDuration = workItem is DeleteStateWorkItem; - Activity? storageActivity = queueOperation is null - ? null - : ActivitySources.StorageGrainSource.StartActivity( - $"journal {queueOperation}", - ActivityKind.Internal, - workItem.TraceParent, - tags: null, - links: workItem.TraceLinks); - if (storageActivity is not null) - { - storageActivity.SetTag(ActivityTagKeys.JournalStorageOperation, queueOperation); - } + Activity? storageActivity = null; JournaledStateOperation? operationType = null; + string? compactionReason = null; var beforeHookRunning = false; var afterHookRunning = false; try { - if (workItem is WriteStateWorkItem - && workItem.DeletionGeneration != Volatile.Read(ref _deletionGeneration)) + lock (_lock) { - throw new InvalidOperationException( - "The journaled state operation was queued before deletion reset its state."); + if (workItem is WriteStateWorkItem + && workItem.DeletionGeneration != _deletionGeneration) + { + throw new InvalidOperationException( + "The journaled state operation was queued before deletion reset its state."); + } + + operationType = workItem switch + { + WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + if (operationType == JournaledStateOperation.Snapshot) + { + compactionReason = _snapshotRequired + ? JournalingInstruments.CompactionReasonUserSnapshot + : JournalingInstruments.CompactionReasonStorageRequested; + _snapshotRequired = false; + } } - operationType = workItem switch - { - WriteStateWorkItem write when write.SnapshotRequired || IsSnapshotRequired => JournaledStateOperation.Snapshot, - WriteStateWorkItem => JournaledStateOperation.Write, - DeleteStateWorkItem => JournaledStateOperation.Delete, - _ => null - }; + (queueOperation, queueDuration, storageActivity) = GetStorageQueueOperation(workItem, operationType, processingTimestamp); + if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; @@ -321,7 +322,7 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { - case WriteStateWorkItem writeState: + case WriteStateWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. @@ -331,12 +332,7 @@ private async Task WorkLoop() : JournalingInstruments.OperationAppend; if (isSnapshot) { - var compactionReason = writeState.SnapshotRequired - ? JournalingInstruments.CompactionReasonUserSnapshot - : _snapshotRequired - ? JournalingInstruments.CompactionReasonMigration - : JournalingInstruments.CompactionReasonStorageRequested; - _shared.Instruments.OnCompactionTriggered(compactionReason); + _shared.Instruments.OnCompactionTriggered(compactionReason!); } _shared.Instruments.OnWriteCoalesced(operationLabel, workItem.CallerCount); var gatherStartTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -357,11 +353,6 @@ private async Task WorkLoop() RetireOrResurectStates(); } - if (_snapshotRequired) - { - ThrowIfMigrationBlockedByRetiredStates(); - } - // The map of state ids is itself stored as a durable state with the id 0. // This must be stored first, since it includes the identities of all other states, which are needed when replaying the journal. // If we removed retired states, this snapshot will persist that change. @@ -502,11 +493,6 @@ private async Task WorkLoop() { state.OnWriteCompleted(); } - - if (isSnapshot) - { - _snapshotRequired = false; - } } } @@ -589,6 +575,14 @@ private async Task WorkLoop() } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { + if (beforeHookRunning && operationType == JournaledStateOperation.Snapshot) + { + lock (_lock) + { + _snapshotRequired = true; + } + } + if (beforeHookRunning && exception is OperationCanceledException && _shutdownCancellation.IsCancellationRequested) { workItem.TrySetCanceled(_shutdownCancellation.Token); @@ -726,20 +720,6 @@ private void RetireOrResurectStates() } } - private void ThrowIfMigrationBlockedByRetiredStates() - { - foreach (var state in _statesMap.Values) - { - if (state is RetiredStateMachine { PreservedEntries.Count: > 0 } retiredState) - { - throw new InvalidOperationException( - $"Cannot migrate journal to format key '{_shared.JournalFormatKey}' because stream " + - $"{retiredState.StreamId.Value} belongs to a state which is not currently registered. Register the state so it can be decoded " + - "and snapshotted in the configured format, or keep the previous journal format configured until the state can be retired."); - } - } - } - private JournalStreamWriter CreateJournalStreamWriter(JournalStreamId streamId) => _journalWriter.CreateJournalStreamWriter(streamId); private static void AppendUpdatesOrSnapshotState(JournalBufferWriter journalWriter, bool isSnapshot, uint id, IStateMachine state) @@ -952,11 +932,9 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - var isSnapshot = IsSnapshotRequired; - operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; - var workItem = EnqueueOrGetPendingWorkItem(out didEnqueue); - workItem.SnapshotRequired |= isSnapshot; - pendingWrite = workItem.Task; + _snapshotRequired = IsSnapshotRequired; + operation = _snapshotRequired ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; + pendingWrite = EnqueueOrGetPendingWorkItem(out didEnqueue).Task; } if (didEnqueue) @@ -1289,21 +1267,37 @@ public void RecordTraceContext() } } - private static string? GetStorageQueueOperation(WorkItem workItem) => - workItem switch + private (string? Operation, TimeSpan QueueDuration, Activity? Activity) GetStorageQueueOperation( + WorkItem workItem, + JournaledStateOperation? operationType, + long processingTimestamp) + { + var operation = operationType switch { - WriteStateWorkItem { SnapshotRequired: true } => JournalingInstruments.OperationSnapshot, - WriteStateWorkItem => JournalingInstruments.OperationAppend, - DeleteStateWorkItem => JournalingInstruments.OperationDelete, + JournaledStateOperation.Snapshot => JournalingInstruments.OperationSnapshot, + JournaledStateOperation.Write => JournalingInstruments.OperationAppend, + JournaledStateOperation.Delete => JournalingInstruments.OperationDelete, _ => null }; + if (operation is null) + { + return default; + } + + var queueDuration = _shared.TimeProvider.GetElapsedTime(workItem.EnqueuedTimestamp, processingTimestamp); + var activity = ActivitySources.StorageGrainSource.StartActivity( + $"journal {operation}", + ActivityKind.Internal, + workItem.TraceParent, + tags: null, + links: workItem.TraceLinks); + activity?.SetTag(ActivityTagKeys.JournalStorageOperation, operation); + return (operation, queueDuration, activity); + } private sealed class InitializeWorkItem : WorkItem; - private sealed class WriteStateWorkItem : WorkItem - { - public bool SnapshotRequired { get; set; } - } + private sealed class WriteStateWorkItem : WorkItem; private sealed class DeleteStateWorkItem : WorkItem; @@ -1427,8 +1421,6 @@ private sealed class RetiredStateMachine(JournalStreamId streamId) : IStateMachi public JournalStreamId StreamId { get; } = streamId; - public IReadOnlyList PreservedEntries => _preservedEntries; - void IStateMachine.ReplayEntry(JournalEntry entry, JournalReplayContext context) => _preservedEntries.Add(new PreservedJournalEntry(entry.FormatKey, entry.Reader)); diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 745c04d707a..46b80082885 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -284,7 +284,9 @@ They also run for a successful zero-byte write. Coalesced callers share the hook operation. Features implement the before and after callbacks on their own hook, retaining their identity and operation-local bookkeeping there. Queued writes coalesce into one operation. A snapshot requirement from any caller upgrades that -queued operation to a snapshot, even if the storage compaction signal clears before it runs. +pending work to a snapshot, even if the storage compaction signal clears before it runs. +The manager consumes the pending requirement when selecting a write and restores it when a before +hook fails. Storage acknowledgement leaves newer snapshot requests untouched. | Outcome | Owner and caller behavior | | --- | --- | diff --git a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs index f1cd56ccf37..5217532cad1 100644 --- a/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs +++ b/test/Orleans.Journaling.Json.Tests/CodecRecoveryTests.cs @@ -293,27 +293,31 @@ public async Task Recovery_EmptyJournalWithStaleMetadata_WritesConfiguredFormat( Assert.Contains("""[8,["set","alpha",1]]""", Encoding.UTF8.GetString(storage.Segments.Single()), StringComparison.Ordinal); } - [Fact] - public async Task Migration_WithUnregisteredRetiredState_ThrowsClearError() + [Theory] + [InlineData(OrleansBinaryJournalFormat.JournalFormatKey, JsonLinesJournalFormat.JournalFormatKey)] + [InlineData(JsonLinesJournalFormat.JournalFormatKey, OrleansBinaryJournalFormat.JournalFormatKey)] + public async Task Migration_WithUnregisteredRetiredState_PreservesStorage(string sourceFormat, string targetFormat) { - var storage = new VolatileJournalStorage(OrleansBinaryJournalFormat.JournalFormatKey); - using var first = CreateFormatAwareTestSystem(storage, OrleansBinaryJournalFormat.JournalFormatKey); - var dict = CreateFormatAwareDictionary(first, OrleansBinaryJournalFormat.JournalFormatKey, "dict"); + var storage = new VolatileJournalStorage(sourceFormat); + using var first = CreateFormatAwareTestSystem(storage, sourceFormat); + var dict = CreateFormatAwareDictionary(first, sourceFormat, "dict"); await first.Lifecycle.OnStart(TestContext.Current.CancellationToken); dict.Add("alpha", 1); await first.Manager.WriteStateAsync(CancellationToken.None); - storage.SetConfiguredJournalFormatKey(JsonLinesJournalFormat.JournalFormatKey); - using var recovered = CreateFormatAwareTestSystem(storage, JsonLinesJournalFormat.JournalFormatKey); - var other = CreateFormatAwareDictionary(recovered, JsonLinesJournalFormat.JournalFormatKey, "other"); + var originalBytes = Assert.Single(storage.Segments).ToArray(); + storage.SetConfiguredJournalFormatKey(targetFormat); + using var recovered = CreateFormatAwareTestSystem(storage, targetFormat); + var other = CreateFormatAwareDictionary(recovered, targetFormat, "other"); await recovered.Lifecycle.OnStart(TestContext.Current.CancellationToken); other.Add("beta", 2); var exception = await Assert.ThrowsAsync( () => recovered.Manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Contains("Cannot migrate journal", exception.Message, StringComparison.Ordinal); - Assert.Contains("not currently registered", exception.Message, StringComparison.Ordinal); + Assert.Contains("cannot append preserved entry", exception.Message, StringComparison.Ordinal); + Assert.Equal(sourceFormat, storage.StoredJournalFormatKey); + Assert.Equal(originalBytes, Assert.Single(storage.Segments)); } [Theory] diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index a9a455cd91d..b8426953b24 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -1,7 +1,10 @@ using System.Buffers; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Reflection; using Microsoft.Extensions.DependencyInjection; using NSubstitute; +using Orleans.Diagnostics; using Orleans.Serialization; using Xunit; @@ -9,6 +12,58 @@ namespace Orleans.Journaling.Tests; public partial class StateManagerTests { + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorageOutcome(bool snapshot, bool storageFails) + { + using var parent = new Activity("journal queue test").SetIdFormat(ActivityIdFormat.W3C).Start(); + var stopped = new ConcurrentBag(); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == ActivitySources.StorageActivitySourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => + { + if (activity.ParentSpanId == parent.SpanId) + { + stopped.Add(activity); + } + } + }; + ActivitySource.AddActivityListener(listener); + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage + { + IsCompactionRequested = snapshot, + NextAppendException = storageFails && !snapshot ? failure : null, + NextReplaceException = storageFails && snapshot ? failure : null + }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + if (storageFails) + { + Assert.Same(failure, await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + } + else + { + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + await manager.DisposeAsync(); + var activity = Assert.Single(stopped); + var operation = snapshot ? "snapshot" : "append"; + Assert.Equal($"journal {operation}", activity.OperationName); + Assert.Equal(operation, activity.GetTagItem(ActivityTagKeys.JournalStorageOperation)); + Assert.Equal(storageFails ? ActivityStatusCode.Error : ActivityStatusCode.Ok, activity.Status); + Assert.Equal(storageFails ? failure.Message : null, activity.StatusDescription); + } + [Fact] public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() { @@ -196,6 +251,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta Assert.Empty(storage.Replaces); Assert.Equal(0, storage.DeleteCount); Assert.Equal(0, after); + storage.IsCompactionRequested = false; fail = false; await InvokeAsync(); Assert.Equal(1, after); @@ -815,6 +871,61 @@ await context.Run(async () => }); } + [Fact] + public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() + { + var storage = new CapturingStorage { IsCompactionRequested = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var failure = new IOException("Snapshot prerequisite failed."); + List before = []; + List after = []; + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperationAsync = async (operation, _) => + { + before.Add(operation); + if (before.Count == 1) + { + entered.TrySetResult(); + await release.Task; + throw failure; + } + }, + AfterOperation = (operation, _) => after.Add(operation) + }); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + storage.IsCompactionRequested = false; + state.Value = 2; + var retry = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Empty(storage.Appends); + Assert.Empty(storage.Replaces); + release.TrySetResult(); + var caught = await Assert.ThrowsAsync(() => WaitFor(first)); + Assert.Same(failure, caught.InnerException); + await WaitFor(retry); + Assert.Equal([JournaledStateOperation.Snapshot, JournaledStateOperation.Snapshot], before); + Assert.Equal([JournaledStateOperation.Snapshot], after); + Assert.Empty(storage.Appends); + Assert.Single(storage.Replaces); + await using var recoveredManager = CreateTestSystem(storage).Manager; + var recovered = new DurableValue("state", recoveredManager, CreateValueCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, recovered.Value); + } + finally + { + release.TrySetResult(); + } + } + [Fact] public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() { From 158fcf35da8209f2ca9bb643773ee477b92951dc Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 13:58:10 -0700 Subject: [PATCH 15/22] refactor(journaling): order hook preparation by registration --- .../grains/journaling/runtime-behavior.md | 9 ++- .../IJournaledStateCaptureHook.cs | 15 ---- src/Orleans.Journaling/IJournaledStateHook.cs | 7 +- .../IJournaledStateManager.cs | 6 +- .../JournaledStateManager.Hooks.cs | 33 ++------- .../JournaledStateManager.cs | 8 +-- src/Orleans.Journaling/README.md | 10 +-- .../Orleans.Journaling/Orleans.Journaling.cs | 4 -- .../StateManagerHookTests.cs | 68 +++++++++---------- 9 files changed, 53 insertions(+), 107 deletions(-) delete mode 100644 src/Orleans.Journaling/IJournaledStateCaptureHook.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 36435a498e7..c8b8554f3ce 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -83,11 +83,10 @@ The hook list then remains fixed through recovery, persistence and deletion. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. -Each actual append, snapshot, or deletion runs ordinary before callbacks in list order, outside -the manager lock. At most one supplies the -final prerequisite. Its before callback runs last, and the work loop awaits it directly before -synchronous capture or storage deletion. Prerequisites cover changes staged during asynchronous -preparation, including the final hook's own I/O wait. Preserve operation-local bookkeeping for +Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside +the manager lock. Register the final prerequisite at index zero so its callback runs last before +synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged +during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. diff --git a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs b/src/Orleans.Journaling/IJournaledStateCaptureHook.cs deleted file mode 100644 index cd6d0b42693..00000000000 --- a/src/Orleans.Journaling/IJournaledStateCaptureHook.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Orleans.Journaling; - -/// -/// Establishes the final prerequisite immediately before journal capture or deletion. -/// -/// -/// An owner admits at most one capture hook in . -/// Its before callback runs after all ordinary before callbacks. The work loop awaits it directly, -/// then captures state or starts deletion without a further asynchronous phase. -/// Its prerequisites cover changes staged while its own I/O awaited. After callbacks retain -/// normal list order. Features inspect and deduplicate this registration using the same hook list. -/// -public interface IJournaledStateCaptureHook : IJournaledStateHook -{ -} diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 29d245799d6..95b8cb4b3b1 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -19,9 +19,10 @@ public enum JournaledStateOperation /// Participates in the prerequisites and completion of actual journal operations. /// /// -/// Ordinary before hooks run in list order on the owner's logical execution context, outside its lock. -/// An optional runs last immediately before capture or deletion. -/// Prerequisites must cover changes staged during asynchronous preparation. After hooks run after +/// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. +/// The hook at index zero supplies the final prerequisite before capture or deletion. +/// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous +/// preparation and while the journal work loop resumes. After hooks run in forward list order after /// storage acknowledgement and state acknowledgement or reset, including successful writes which /// produce no storage bytes. Hooks retain operation-local data across these boundaries and keep later /// pending changes separate. Hook implementations must complete without calling initialization, diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 77149dc9500..c5f169d6d3d 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,9 +17,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. - /// Registration remains fixed for this manager's initialized lifetime. Ordinary before hooks - /// and all after hooks execute in list order. The optional single - /// supplies the final prerequisite. + /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in + /// reverse list order; after hooks execute in forward list order. Register the final capture + /// prerequisite at index zero so it runs last before capture or deletion. /// Registration is independent of state-machine registration and persists through recovery and deletion. /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 1c9502a772f..789a06ead9f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -18,24 +18,15 @@ public IList Hooks } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { - IJournaledStateCaptureHook? captureHook = null; - for (var i = 0; i < _hooks!.Count; i++) + for (var i = _hooks!.Count - 1; i >= 0; i--) { - var hook = _hooks[i]; - if (hook is IJournaledStateCaptureHook capture) - { - captureHook = capture; - continue; - } - cancellationToken.ThrowIfCancellationRequested(); - await hook.BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); + await _hooks[i].BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); } cancellationToken.ThrowIfCancellationRequested(); - return captureHook; } private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) @@ -71,7 +62,7 @@ protected override void InsertItem(int index, IJournaledStateHook item) lock (owner._lock) { owner.EnsureRegistrationAllowed(); - ValidateHook(item, replacingIndex: -1); + ArgumentNullException.ThrowIfNull(item); base.InsertItem(index, item); } } @@ -81,7 +72,7 @@ protected override void SetItem(int index, IJournaledStateHook item) lock (owner._lock) { owner.EnsureRegistrationAllowed(); - ValidateHook(item, index); + ArgumentNullException.ThrowIfNull(item); base.SetItem(index, item); } } @@ -104,19 +95,5 @@ protected override void ClearItems() } } - private void ValidateHook(IJournaledStateHook item, int replacingIndex) - { - ArgumentNullException.ThrowIfNull(item); - if (item is IJournaledStateCaptureHook) - { - for (var i = 0; i < Count; i++) - { - if (i != replacingIndex && this[i] is IJournaledStateCaptureHook) - { - throw new InvalidOperationException("A journal owner supports one final capture prerequisite hook. Inspect and deduplicate the hook list before registration."); - } - } - } - } } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 89791f81443..6790b90e51f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -306,13 +306,7 @@ private async Task WorkLoop() if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - var captureHook = await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - - if (captureHook is not null) - { - // Await the final prerequisite in this frame so capture follows its completion directly. - await captureHook.BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); - } + await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); _shutdownCancellation.Token.ThrowIfCancellationRequested(); beforeHookRunning = false; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 46b80082885..51eb00c3f6b 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -269,14 +269,14 @@ of the ambient grain context, including when the caller subsequently enrolls the `IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` registrations. Features register and deduplicate their hooks during setup before initialization begins, alongside state-machine registration. The hook list then remains fixed through recovery, -persistence and whole-journal deletion. The standard manager admits at most one `IJournaledStateCaptureHook`. +persistence and whole-journal deletion. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. -For each actual write, snapshot, or deletion, ordinary before hooks run in list order outside -the manager lock. The capture hook runs last: the work loop awaits it directly, then synchronously -captures the registered states or starts deletion. Prerequisites cover all changes staged during -asynchronous preparation, including changes arriving while the capture hook awaits its own I/O. +For each actual write, snapshot, or deletion, before hooks run in reverse list order outside +the manager lock. Register the final prerequisite at index zero so it runs last before the work loop +captures the registered states or starts deletion. Prerequisites remain valid through capture or +deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Keep operation-local bookkeeping for the captured batch separate from later pending changes. After hooks run in list order after storage acknowledgement and state acknowledgement or reset. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 76ff42769dd..7502cdeb322 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -219,10 +219,6 @@ public partial interface IDurableValue T? Value { get; set; } } - public partial interface IJournaledStateCaptureHook : IJournaledStateHook - { - } - public partial interface IJournaledStateHook { System.Threading.Tasks.ValueTask AfterOperationAsync(JournaledStateOperation operation, System.Threading.CancellationToken cancellationToken); diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index b8426953b24..649a3fac00e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -359,7 +359,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - manager.Hooks.Add(new CaptureTestHook( + manager.Hooks.Add(new DelegateTestHook( (_, _) => { Assert.Equal(3, state["late"]); @@ -407,10 +407,8 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar Assert.Equal(3, recovered["late"]); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(bool captureHook) + [Fact] + public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; @@ -433,9 +431,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b Assert.Single(storage.Appends); completed.TrySetResult(); }; - manager.Hooks.Add(captureHook - ? new CaptureTestHook(before, after) - : new DelegateTestHook { BeforeOperationAsync = before, AfterOperation = after }); + manager.Hooks.Add(new DelegateTestHook(before, after)); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; @@ -468,21 +464,31 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion(b } [Fact] - public async Task Hooks_RegistrationEnforcesSingleCapturePrerequisite() + public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() { await using var manager = CreateTestSystem().Manager; - var first = new CaptureTestHook((_, _) => default); - var second = new CaptureTestHook((_, _) => default); - var ordinary = new DelegateTestHook(); - manager.Hooks.Add(first); - manager.Hooks.Add(ordinary); - Assert.Throws(() => manager.Hooks.Add(second)); - Assert.Throws(() => manager.Hooks[1] = second); - Assert.Equal(new IJournaledStateHook[] { first, ordinary }, manager.Hooks); - manager.Hooks[0] = second; - Assert.Same(second, manager.Hooks[0]); + List events = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("second-before"), + AfterOperation = (_, _) => events.Add("second-after") + }); + manager.Hooks.Add(new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("third-before"), + AfterOperation = (_, _) => events.Add("third-after") + }); + manager.Hooks.Insert(0, new DelegateTestHook + { + BeforeOperation = (_, _) => events.Add("first-before"), + AfterOperation = (_, _) => events.Add("first-after") + }); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal( + ["third-before", "second-before", "first-before", "capture", "ack", "first-after", "second-after", "third-after"], + events); } [Fact] @@ -495,7 +501,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; - manager.Hooks.Add(new CaptureTestHook( + manager.Hooks.Add(new DelegateTestHook( async (operation, token) => { if (!deleting) @@ -1102,25 +1108,13 @@ public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) Assert.Equal(original.Message, copy.Message); } - private sealed class CaptureTestHook( - Func before, - Action? after = null) : IJournaledStateCaptureHook - { - public ValueTask BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) => - before(operation, cancellationToken); - - public ValueTask AfterOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - after?.Invoke(operation, cancellationToken); - return default; - } - } - - private sealed class DelegateTestHook : IJournaledStateHook + private sealed class DelegateTestHook( + Func? before = null, + Action? after = null) : IJournaledStateHook { public Action? BeforeOperation { get; init; } - public Func? BeforeOperationAsync { get; init; } - public Action? AfterOperation { get; init; } + public Func? BeforeOperationAsync { get; init; } = before; + public Action? AfterOperation { get; init; } = after; public Func? AfterOperationAsync { get; init; } ValueTask IJournaledStateHook.BeforeOperationAsync(JournaledStateOperation operation, CancellationToken cancellationToken) From 470ec86827a3d478470e0735445ff33ead1010ef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:08:33 -0700 Subject: [PATCH 16/22] refactor(journaling): decouple hook registration freeze --- .../grains/journaling/runtime-behavior.md | 2 + .../IJournaledStateManager.cs | 2 + .../JournaledStateManager.Hooks.cs | 47 ++++++++++--------- .../JournaledStateManager.cs | 2 + src/Orleans.Journaling/README.md | 2 + .../StateManagerHookTests.cs | 46 ++++++++++++++---- 6 files changed, 69 insertions(+), 32 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index c8b8554f3ce..1533355c815 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -80,6 +80,8 @@ Activation-scoped features coordinate prerequisites and completion through allocated list of registrations. Register and deduplicate feature hooks during setup before initialization begins, alongside state machines. The hook list then remains fixed through recovery, persistence and deletion. +Keep registration sequential with initialization and shutdown. Both transitions make the list +read-only, including when the hook list is first accessed afterward. Every journal owner implementation provides this list and runs its registered callbacks at the operation boundaries below. Delegating owners forward the list to the inner owner. diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index c5f169d6d3d..f6c25ecc739 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -17,6 +17,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// /// /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. + /// The caller keeps hook registration sequential with initialization and shutdown. + /// Initialization or shutdown makes the list read-only. /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in /// reverse list order; after hooks execute in forward list order. Register the final capture /// prerequisite at index zero so it runs last before capture or deletion. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 789a06ead9f..f9b61c08791 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -13,7 +13,7 @@ public IList Hooks { lock (_lock) { - return _hooks ??= new(this); + return _hooks ??= new(_workLoop is not null || _shutdownCancellation.IsCancellationRequested); } } } @@ -55,45 +55,46 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection(JournaledStateManager owner) : Collection + private sealed class HookCollection(bool isReadOnly) : Collection, IList { + private bool _isReadOnly = isReadOnly; + + bool ICollection.IsReadOnly => _isReadOnly; + + public void Freeze() => _isReadOnly = true; + protected override void InsertItem(int index, IJournaledStateHook item) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - ArgumentNullException.ThrowIfNull(item); - base.InsertItem(index, item); - } + ThrowIfReadOnly(); + ArgumentNullException.ThrowIfNull(item); + base.InsertItem(index, item); } protected override void SetItem(int index, IJournaledStateHook item) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - ArgumentNullException.ThrowIfNull(item); - base.SetItem(index, item); - } + ThrowIfReadOnly(); + ArgumentNullException.ThrowIfNull(item); + base.SetItem(index, item); } protected override void RemoveItem(int index) { - lock (owner._lock) - { - owner.EnsureRegistrationAllowed(); - base.RemoveItem(index); - } + ThrowIfReadOnly(); + base.RemoveItem(index); } protected override void ClearItems() { - lock (owner._lock) + ThrowIfReadOnly(); + base.ClearItems(); + } + + private void ThrowIfReadOnly() + { + if (_isReadOnly) { - owner.EnsureRegistrationAllowed(); - base.ClearItems(); + throw new NotSupportedException("Journal operation hook registrations are read-only."); } } - } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6790b90e51f..6cdf5ebf108 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -169,6 +169,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { + _hooks?.Freeze(); _workLoop = Start(); } @@ -1095,6 +1096,7 @@ private async Task StopAsync(CancellationToken cancellationToken) AggregateException? cancellationFailure = null; lock (_lock) { + _hooks?.Freeze(); try { _shutdownCancellation.Cancel(); diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 51eb00c3f6b..df2af6af17a 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -270,6 +270,8 @@ of the ambient grain context, including when the caller subsequently enrolls the registrations. Features register and deduplicate their hooks during setup before initialization begins, alongside state-machine registration. The hook list then remains fixed through recovery, persistence and whole-journal deletion. +Keep hook registration sequential with initialization and shutdown. Either lifecycle transition +makes the list read-only; its collection owns this state independently of the journal owner. Every journal owner implementation provides the hook list and invokes registered callbacks at the operation boundaries described below. Delegating owners forward the list to their inner owner. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 649a3fac00e..b0c1a84b39e 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -73,6 +73,7 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() var hook = new DelegateTestHook(); var hooks = manager.Hooks; + Assert.False(hooks.IsReadOnly); hooks.Add(hook); Assert.True(hooks.Contains(hook)); Assert.Same(hooks, manager.Hooks); @@ -85,12 +86,38 @@ public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.Throws(() => hooks.Clear()); + Assert.True(hooks.IsReadOnly); + Assert.Throws(() => hooks.Clear()); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(withoutHooks)); + Assert.True(withoutHooks.Hooks.IsReadOnly); + Assert.Throws(() => withoutHooks.Hooks.Add(hook)); + } + + [Fact] + public async Task Hooks_ShutdownMakesSetupRegistrationsReadOnlyBeforeInitialization() + { + var manager = CreateTestSystem().Manager; + var hooks = manager.Hooks; + var hook = new DelegateTestHook(); + hooks.Add(hook); + Assert.False(hooks.IsReadOnly); + + await manager.DisposeAsync(); + + Assert.True(hooks.IsReadOnly); + Assert.Same(hooks, manager.Hooks); + Assert.Same(hook, Assert.Single(hooks)); + Assert.Throws(() => hooks.Add(hook)); + Assert.Throws(() => hooks.Clear()); + + var withoutHooks = CreateTestSystem().Manager; + await withoutHooks.DisposeAsync(); + Assert.True(withoutHooks.Hooks.IsReadOnly); + Assert.Throws(() => withoutHooks.Hooks.Add(hook)); } [Fact] @@ -110,19 +137,20 @@ public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFai var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); try { - Assert.Throws(() => hooks.Add(new DelegateTestHook())); - Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => hooks[0] = new DelegateTestHook()); - Assert.Throws(() => hooks.Remove(hook)); - Assert.Throws(() => hooks.Clear()); + Assert.True(hooks.IsReadOnly); + Assert.Throws(() => hooks.Add(new DelegateTestHook())); + Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); + Assert.Throws(() => hooks[0] = new DelegateTestHook()); + Assert.Throws(() => hooks.Remove(hook)); + Assert.Throws(() => hooks.Clear()); await WaitFor(entered.Task); Assert.False(initialization.IsCompleted); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); - Assert.Throws(() => hooks.Clear()); + Assert.Throws(() => hooks.Clear()); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => hooks.Clear()); + Assert.Throws(() => hooks.Clear()); Assert.Same(hook, Assert.Single(hooks)); } finally @@ -587,7 +615,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await Assert.ThrowsAnyAsync(() => queued); Assert.True(drained); Assert.Empty(storage.Appends); - Assert.Throws(() => manager.Hooks.Clear()); + Assert.Throws(() => manager.Hooks.Clear()); } finally { From 065c3da253dfb47f1b809ab6adf7c188f75cd77f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 16:43:52 -0700 Subject: [PATCH 17/22] feat(journaling)!: serialize dynamic hook registration --- .../grains/journaling/runtime-behavior.md | 19 +- .../CompatibilitySuppressions.xml | 28 ++ src/Orleans.Journaling/IJournaledStateHook.cs | 2 + .../IJournaledStateManager.cs | 39 +- .../JournaledStateManager.Hooks.cs | 100 ++-- .../JournaledStateManager.cs | 94 +++- src/Orleans.Journaling/README.md | 20 +- .../Orleans.Journaling/Orleans.Journaling.cs | 4 +- .../DurableListJournalBenchmarks.cs | 19 +- .../JournaledJobShardDiscoveryTests.cs | 6 +- .../DurableCollectionDirectWriteTests.cs | 19 +- .../DurableListDirectWriteTests.cs | 19 +- .../KeyedJournalingRegistrationTests.cs | 6 +- .../StateManagerHookTests.cs | 457 ++++++++++++++---- 14 files changed, 675 insertions(+), 157 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 1533355c815..7230ffd7315 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -77,16 +77,19 @@ staged mutations from other calls. Activation-scoped features coordinate prerequisites and completion through . The owner exposes a stable, lazily -allocated list of registrations. Register and -deduplicate feature hooks during setup before initialization begins, alongside state machines. -The hook list then remains fixed through recovery, persistence and deletion. -Keep registration sequential with initialization and shutdown. Both transitions make the list -read-only, including when the hook list is first accessed afterward. -Every journal owner implementation provides this list and runs its registered callbacks at the -operation boundaries below. Delegating owners forward the list to the inner owner. +allocated read-only view of registrations. +Use and + to change the set. Idle changes +apply synchronously; busy changes queue between complete operations, after current after callbacks. +The hook set stays stable throughout each operation. Queued changes also separate coalesced writes, +so writes admitted after a change observe that change. +Cancellation ends only the caller's wait while the manager owns admitted changes. +Every owner implements the registration methods and callback boundaries; delegating owners forward +them to their inner owner. Synchronous state callbacks can request a queued change and observe its +completion outside the callback. Standalone callers coordinate view inspection with changes. Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside -the manager lock. Register the final prerequisite at index zero so its callback runs last before +the manager lock. Add the final prerequisite at index zero so its callback runs last before synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index ed87a41fc9b..3b379c6860f 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -239,6 +239,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -246,6 +253,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net10.0/Orleans.Journaling.dll + lib/net10.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) @@ -281,6 +295,13 @@ lib/net10.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.AddHookAsync(Orleans.Journaling.IJournaledStateHook,System.Nullable{System.Int32},System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.RegisterStateMachine(System.String,Orleans.Journaling.IStateMachine) @@ -288,6 +309,13 @@ lib/net8.0/Orleans.Journaling.dll true + + CP0006 + M:Orleans.Journaling.IJournaledStateManager.RemoveHookAsync(Orleans.Journaling.IJournaledStateHook,System.Threading.CancellationToken) + lib/net8.0/Orleans.Journaling.dll + lib/net8.0/Orleans.Journaling.dll + true + CP0006 M:Orleans.Journaling.IJournaledStateManager.TryGetStateMachine(System.String,Orleans.Journaling.IStateMachine@) diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 95b8cb4b3b1..10c8d5eb9f1 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -29,6 +29,8 @@ public enum JournaledStateOperation /// persistence, or disposal on their own journal owner. Those operations are serialized behind the /// current operation, so awaiting them from a hook would create a circular dependency. /// Dependencies between journal owners must also remain acyclic. +/// Callbacks can request a hook registration change, observing its queued completion outside the +/// callback rather than awaiting work which follows the current operation. /// public interface IJournaledStateHook { diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index f6c25ecc739..761eb1d50cd 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -13,19 +13,46 @@ namespace Orleans.Journaling; public interface IJournaledStateManager : IAsyncDisposable { /// - /// Gets the mutable, lazily allocated list of journal operation hooks. + /// Gets the read-only view of registered journal operation hooks. /// /// - /// Inspect, add, remove, and deduplicate hooks during setup, before initialization begins. - /// The caller keeps hook registration sequential with initialization and shutdown. - /// Initialization or shutdown makes the list read-only. - /// Registration remains fixed for this manager's initialized lifetime. Before hooks execute in + /// Inspect hooks on the owner's logical execution context, coordinating standalone inspection + /// with registration changes. Use and + /// to change registrations. Before hooks execute in /// reverse list order; after hooks execute in forward list order. Register the final capture /// prerequisite at index zero so it runs last before capture or deletion. /// Registration is independent of state-machine registration and persists through recovery and deletion. /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. /// - IList Hooks { get; } + IReadOnlyList Hooks { get; } + + /// + /// Adds a hook before the next persistence operation which follows this registration. + /// + /// The hook to register. + /// The insertion index, or to append. + /// The token used to cancel the caller's wait. + /// A completion representing application of the registration. + /// + /// Idle registration applies synchronously. Otherwise it is queued between owned operations, + /// after the current operation's after hooks finish. Queued registration is a write-coalescing barrier. + /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. + /// Synchronous state callbacks can request registration and observe its completion outside the callback. + /// Await registration before staging changes whose safety depends on the added hook. + /// + ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default); + + /// + /// Removes a hook between persistence operations. + /// + /// The registered hook to remove. + /// The token used to cancel the caller's wait. + /// A completion representing application of the removal. + /// The hook is not registered when the removal is applied. + /// + /// Removal has the same ownership and ordering guarantees as . + /// + ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default); /// ValueTask IAsyncDisposable.DisposeAsync() => default; diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index f9b61c08791..8cb1fe842cf 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -1,4 +1,4 @@ -using System.Collections.ObjectModel; +using System.Collections; using System.Runtime.ExceptionServices; namespace Orleans.Journaling; @@ -7,17 +7,61 @@ internal partial class JournaledStateManager { private HookCollection? _hooks; - public IList Hooks + public IReadOnlyList Hooks { get { lock (_lock) { - return _hooks ??= new(_workLoop is not null || _shutdownCancellation.IsCancellationRequested); + return _hooks ??= new(); } } } + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index, remove: false, cancellationToken); + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + MutateHookAsync(hook, index: null, remove: true, cancellationToken); + + private ValueTask MutateHookAsync(IJournaledStateHook hook, int? index, bool remove, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(hook); + cancellationToken.ThrowIfCancellationRequested(); + Task task; + lock (_lock) + { + ThrowIfFenced(); + ObjectDisposedException.ThrowIf(_disposed != 0, this); + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + if (_workLoop is null || (!_workItemRunning && _workQueue.Count == 0)) + { + ApplyHookMutation(hook, index, remove); + return default; + } + + var workItem = new HookMutationWorkItem(hook, index, remove); + _workQueue.Enqueue(workItem); + task = workItem.Task; + } + + _workSignal.Signal(); + return new(task.WaitAsync(cancellationToken)); + } + + private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove) + { + var hooks = _hooks ??= new(); + if (remove) + { + hooks.Remove(hook); + } + else + { + hooks.Insert(index, hook); + } + } + private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { for (var i = _hooks!.Count - 1; i >= 0; i--) @@ -55,46 +99,30 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection(bool isReadOnly) : Collection, IList + private sealed class HookCollection : IReadOnlyList { - private bool _isReadOnly = isReadOnly; - - bool ICollection.IsReadOnly => _isReadOnly; + private readonly List _items = []; - public void Freeze() => _isReadOnly = true; + public int Count => _items.Count; + public IJournaledStateHook this[int index] => _items[index]; + public IEnumerator GetEnumerator() => _items.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - protected override void InsertItem(int index, IJournaledStateHook item) - { - ThrowIfReadOnly(); - ArgumentNullException.ThrowIfNull(item); - base.InsertItem(index, item); - } + public void Insert(int? index, IJournaledStateHook hook) => _items.Insert(index ?? _items.Count, hook); - protected override void SetItem(int index, IJournaledStateHook item) + public void Remove(IJournaledStateHook hook) { - ThrowIfReadOnly(); - ArgumentNullException.ThrowIfNull(item); - base.SetItem(index, item); - } - - protected override void RemoveItem(int index) - { - ThrowIfReadOnly(); - base.RemoveItem(index); - } - - protected override void ClearItems() - { - ThrowIfReadOnly(); - base.ClearItems(); - } - - private void ThrowIfReadOnly() - { - if (_isReadOnly) + if (!_items.Remove(hook)) { - throw new NotSupportedException("Journal operation hook registrations are read-only."); + throw new InvalidOperationException("The journal operation hook is not registered."); } } } + + private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem + { + public IJournaledStateHook Hook { get; } = hook; + public int? Index { get; } = index; + public bool Remove { get; } = remove; + } } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6cdf5ebf108..1f3e6c18d62 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -29,6 +29,8 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private readonly StateDirectory _journalStreamDirectory; private readonly RetiredStateTracker _retirementTracker; private Task? _workLoop; + // Under _lock, covers recovery and dequeued work through the last after callback. + private bool _workItemRunning; private ManagerState _state; private long _deletionGeneration; private Exception? _failure; @@ -169,7 +171,7 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def ThrowIfFenced(); if (_workLoop is null) { - _hooks?.Freeze(); + _workItemRunning = true; _workLoop = Start(); } @@ -195,6 +197,11 @@ private async Task WorkLoop() await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); while (!_shutdownCancellation.IsCancellationRequested) { + lock (_lock) + { + _workItemRunning = true; + } + try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -216,9 +223,17 @@ private async Task WorkLoop() lock (_lock) { FaultQueuedWorkItemsUnderLock(exception); + _workItemRunning = false; } } } + finally + { + lock (_lock) + { + _workItemRunning = false; + } + } // Signals can remain from the failed attempt. Retry only for newly queued initialization work. while (true) @@ -231,7 +246,17 @@ private async Task WorkLoop() lock (_lock) { - if (_workQueue.Count > 0) + var retryRequested = false; + foreach (var workItem in _workQueue) + { + if (workItem is InitializeWorkItem) + { + retryRequested = true; + break; + } + } + + if (retryRequested) { break; } @@ -263,6 +288,7 @@ private async Task WorkLoop() } workItem = dequeuedWorkItem; + _workItemRunning = true; } var processingTimestamp = _shared.TimeProvider.GetTimestamp(); @@ -317,6 +343,15 @@ private async Task WorkLoop() // We are ok sacrificing some code organization for performance in the inner loop. switch (workItem) { + case HookMutationWorkItem mutation: + { + lock (_lock) + { + ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); + } + break; + } + case WriteStateWorkItem: { // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. @@ -566,7 +601,11 @@ private async Task WorkLoop() } storageActivity?.SetStatus(ActivityStatusCode.Ok); - workItem.SetResult(); + lock (_lock) + { + _workItemRunning = false; + workItem.SetResult(); + } } catch (Exception exception) when (beforeHookRunning || afterHookRunning) { @@ -591,9 +630,22 @@ private async Task WorkLoop() _shared.Instruments.OnStorageOperationQueued(queueOperation, queueDuration, succeeded: afterHookRunning); } - workItem.SetException(afterHookRunning - ? new JournaledStatePostCommitException(operationType.Value, exception) - : new JournaledStatePreCommitException(operationType.Value, exception)); + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(afterHookRunning + ? new JournaledStatePostCommitException(operationType.Value, exception) + : new JournaledStatePreCommitException(operationType.Value, exception)); + } + } + catch (Exception exception) when (workItem is HookMutationWorkItem) + { + LogHookMutationFailed(_shared.Logger, exception); + lock (_lock) + { + _workItemRunning = false; + workItem.SetException(exception); + } } catch (Exception exception) { @@ -627,6 +679,11 @@ private async Task WorkLoop() } finally { + lock (_lock) + { + _workItemRunning = false; + } + storageActivity?.Dispose(); } } @@ -1033,9 +1090,17 @@ private async ValueTask DeleteStorageAsync(CancellationToken cancellationToken) private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) where TWorkItem : WorkItem, new() { + TWorkItem? pending = null; foreach (var workItem in _workQueue) { - if (workItem is not TWorkItem pending) + if (workItem is HookMutationWorkItem) + { + // Later callers must not coalesce into work which precedes a registration change. + pending = null; + continue; + } + + if (pending is not null || workItem is not TWorkItem candidate) { continue; } @@ -1046,8 +1111,13 @@ private TWorkItem EnqueueOrGetPendingWorkItem(out bool didEnqueue) continue; } - workItem.RecordTraceContext(); - workItem.AddCaller(); + pending = candidate; + } + + if (pending is not null) + { + pending.RecordTraceContext(); + pending.AddCaller(); didEnqueue = false; return pending; } @@ -1096,7 +1166,6 @@ private async Task StopAsync(CancellationToken cancellationToken) AggregateException? cancellationFailure = null; lock (_lock) { - _hooks?.Freeze(); try { _shutdownCancellation.Cancel(); @@ -1442,6 +1511,11 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Journal {Operation} hook failed. Persistence completed: {Committed}.")] private static partial void LogOperationHookFailed(ILogger logger, Exception exception, JournaledStateOperation operation, bool committed); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Journal operation hook registration change failed.")] + private static partial void LogHookMutationFailed(ILogger logger, Exception exception); + [LoggerMessage( Level = LogLevel.Error, Message = "Journal shutdown cancellation callback failed; owned operations are drained before resources are released.")] diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index df2af6af17a..4e43c8c8334 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -266,17 +266,21 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks -`IJournaledStateManager.Hooks` is a lazily allocated, stable list of `IJournaledStateHook` -registrations. Features register and deduplicate their hooks during setup before initialization -begins, alongside state-machine registration. The hook list then remains fixed through recovery, -persistence and whole-journal deletion. -Keep hook registration sequential with initialization and shutdown. Either lifecycle transition -makes the list read-only; its collection owns this state independently of the journal owner. +`IJournaledStateManager.Hooks` is a lazily allocated, stable read-only view of `IJournaledStateHook` +registrations. Use `AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to change registrations. +Changes apply synchronously when idle, or queue between complete journal operations while busy. +Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. +Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. +Caller cancellation ends only its wait for an admitted change; the manager retains ownership. +Await registration before staging changes whose safety depends on the new hook. Every journal owner implementation provides the hook list and invokes registered callbacks at -the operation boundaries described below. Delegating owners forward the list to their inner owner. +the operation boundaries described below. Delegating owners forward registration methods and the +read-only view to their inner owner. Synchronous state callbacks can request a queued registration +and observe completion outside the callback. Standalone callers coordinate view inspection with +registration changes. For each actual write, snapshot, or deletion, before hooks run in reverse list order outside -the manager lock. Register the final prerequisite at index zero so it runs last before the work loop +the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop captures the registered states or starts deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Keep operation-local bookkeeping for the captured batch separate from later pending changes. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 7502cdeb322..969a8c672d1 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -227,13 +227,15 @@ public partial interface IJournaledStateHook public partial interface IJournaledStateManager : System.IAsyncDisposable { - System.Collections.Generic.IList Hooks { get; } + System.Collections.Generic.IReadOnlyList Hooks { get; } long PendingWriteByteCount { get; } + System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); + System.Threading.Tasks.ValueTask RemoveHookAsync(IJournaledStateHook hook, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); bool TryGetStateMachine(string name, out IStateMachine? stateMachine); System.Threading.Tasks.ValueTask WriteStateAsync(System.Threading.CancellationToken cancellationToken = default); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index f5931b7e85b..fa2f4658687 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -126,7 +126,24 @@ private void ReplayEncodedJournalData() private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter buffer, JournalStreamId streamId) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index c6829908e9a..c06a4576ec7 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,7 +727,11 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IList Hooks => inner.Hooks; + public IReadOnlyList Hooks => inner.Hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public int DisposeCalls { get; private set; } public long PendingWriteByteCount => inner.PendingWriteByteCount; public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 4e46d24cde7..94fc6b4ff09 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -193,7 +193,24 @@ public void Collections_UseDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index bb3648d69fa..bd4176f6403 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -92,7 +92,24 @@ public void Add_UsesDirectEntryWriter() private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJournaledStateManager { - public IList Hooks { get; } = []; + private readonly List _hooks = []; + + public IReadOnlyList Hooks => _hooks; + + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) + { + _hooks.Insert(index ?? _hooks.Count, hook); + return default; + } + + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) + { + if (!_hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } + return default; + } public ValueTask InitializeAsync(CancellationToken cancellationToken) => default; diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 97b65742677..5d91cafdd8b 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,7 +746,11 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IList Hooks => inner.Hooks; + public IReadOnlyList Hooks => inner.Hooks; + public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => + inner.AddHookAsync(hook, index, cancellationToken); + public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => + inner.RemoveHookAsync(hook, cancellationToken); public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index b0c1a84b39e..4166f54ab9a 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -65,93 +65,125 @@ public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorag } [Fact] - public async Task Hooks_ListIsLazyStableAndSupportsInPlaceDeduplication() + public async Task Hooks_IdleRegistrationAppliesSynchronouslyAndExposesStableView() { await using var manager = CreateTestSystem().Manager; + _ = new DurableValue("state", manager, CreateValueCodec()); var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); var hook = new DelegateTestHook(); + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); var hooks = manager.Hooks; - Assert.False(hooks.IsReadOnly); - hooks.Add(hook); - Assert.True(hooks.Contains(hook)); + Assert.False(hooks is IList); + Assert.Contains(hook, hooks); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.True(hooks.Remove(hook)); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); Assert.Empty(manager.Hooks); - Assert.Throws(() => hooks.Add(null!)); - hooks.Add(hook); + await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.True(hooks.IsReadOnly); - Assert.Throws(() => hooks.Clear()); + Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); await withoutHooks.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Null(field.GetValue(withoutHooks)); - Assert.True(withoutHooks.Hooks.IsReadOnly); - Assert.Throws(() => withoutHooks.Hooks.Add(hook)); } [Fact] - public async Task Hooks_ShutdownMakesSetupRegistrationsReadOnlyBeforeInitialization() + public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() { var manager = CreateTestSystem().Manager; - var hooks = manager.Hooks; var hook = new DelegateTestHook(); - hooks.Add(hook); - Assert.False(hooks.IsReadOnly); + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); + var hooks = manager.Hooks; await manager.DisposeAsync(); - Assert.True(hooks.IsReadOnly); Assert.Same(hooks, manager.Hooks); Assert.Same(hook, Assert.Single(hooks)); - Assert.Throws(() => hooks.Add(hook)); - Assert.Throws(() => hooks.Clear()); + await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); + } - var withoutHooks = CreateTestSystem().Manager; - await withoutHooks.DisposeAsync(); - Assert.True(withoutHooks.Hooks.IsReadOnly); - Assert.Throws(() => withoutHooks.Hooks.Add(hook)); + [Fact] + public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence() + { + var entered = NewSignal(); + var release = NewSignal(); + var storage = Substitute.For(); + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg(), call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var before = 0; + var after = 0; + var hook = new DelegateTestHook + { + BeforeOperation = (_, _) => before++, + AfterOperation = (_, _) => after++ + }; + var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var registration = manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + Assert.False(initialization.IsCompleted); + Assert.False(registration.IsCompleted); + Assert.Empty(manager.Hooks); + release.TrySetResult(); + await WaitFor(Task.WhenAll(initialization, registration)); + Assert.Same(hook, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); + Assert.Equal(1, after); + } + finally + { + release.TrySetResult(); + } + + async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + consumer.Complete(metadata: null); + } } [Fact] - public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFailure() + public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrationDoesNotRetryRecovery() { var entered = NewSignal(); var release = NewSignal(); var storage = Substitute.For(); var failure = new IOException("Recovery failed."); var fail = true; + var reads = 0; storage.ReadAsync(Arg.Any(), Arg.Any()) .Returns(call => ReadAsync(call.Arg(), call.Arg())); await using var manager = CreateTestSystem(storage).Manager; - var hook = new DelegateTestHook(); - var hooks = manager.Hooks; - hooks.Add(hook); var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); + var failedRegistration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); try { - Assert.True(hooks.IsReadOnly); - Assert.Throws(() => hooks.Add(new DelegateTestHook())); - Assert.Throws(() => hooks.Insert(0, new DelegateTestHook())); - Assert.Throws(() => hooks[0] = new DelegateTestHook()); - Assert.Throws(() => hooks.Remove(hook)); - Assert.Throws(() => hooks.Clear()); await WaitFor(entered.Task); - Assert.False(initialization.IsCompleted); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); - Assert.Throws(() => hooks.Clear()); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); + Assert.Empty(manager.Hooks); + var hook = new DelegateTestHook(); + Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + Assert.Equal(1, reads); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); - Assert.Throws(() => hooks.Clear()); - Assert.Same(hook, Assert.Single(hooks)); + Assert.Equal(2, reads); + Assert.Same(hook, Assert.Single(manager.Hooks)); } finally { @@ -160,17 +192,275 @@ public async Task Hooks_RegistrationClosesBeforeRecoveryAndRemainsClosedAfterFai async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken token) { + reads++; entered.TrySetResult(); await release.Task.WaitAsync(token); if (fail) { throw failure; } - consumer.Complete(metadata: null); } } + [Theory] + [InlineData("before")] + [InlineData("storage")] + [InlineData("after")] + public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCancellation(string phase) + { + var storage = new CapturingStorage { BlockNextAppend = phase == "storage" }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableValue("state", manager, CreateValueCodec()); + var entered = NewSignal(); + var release = NewSignal(); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + var original = new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + oldBefore++; + if (phase == "before") + { + entered.TrySetResult(); + await release.Task; + } + }, + AfterOperationAsync = async (_, _) => + { + oldAfter++; + if (phase == "after") + { + entered.TrySetResult(); + await release.Task; + } + } + }; + var replacement = new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + state.Value = 1; + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(phase == "storage" ? storage.BlockedAppendStarted.Task : entered.Task); + using var caller = new CancellationTokenSource(); + var added = manager.AddHookAsync(replacement, index: 0, caller.Token).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + state.Value = 2; + var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(added.IsCompleted); + Assert.False(removed.IsCompleted); + Assert.Same(original, Assert.Single(manager.Hooks)); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => added); + Assert.Same(original, Assert.Single(manager.Hooks)); + + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, removed, next)); + Assert.Same(replacement, Assert.Single(manager.Hooks)); + Assert.Equal(1, oldBefore); + Assert.Equal(1, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + Assert.Equal(2, state.Value); + } + finally + { + release.TrySetResult(); + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_QueuedMutationSeparatesWriteCoalescingGroups() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var oldBefore = 0; + var oldAfter = 0; + var newBefore = 0; + var newAfter = 0; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => oldBefore++, + AfterOperation = (_, _) => oldAfter++ + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var beforeChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var change = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var afterChange = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var coalesced = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await WaitFor(Task.WhenAll(first, beforeChange, change, afterChange, coalesced)); + Assert.Equal(3, state.CaptureCount); + Assert.Equal(3, storage.Appends.Count); + Assert.Equal(3, oldBefore); + Assert.Equal(3, oldAfter); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ResetCanQueueRegistrationWithoutChangingCurrentAfterHooks() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + Task? registration = null; + var newBefore = 0; + var newAfter = 0; + state.ResetAction = () => + { + if (state.ResetCount == 2) + { + registration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => newBefore++, + AfterOperation = (_, _) => newAfter++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + } + }; + manager.RegisterStateMachine("state", state); + await manager.AddHookAsync(new DelegateTestHook + { + AfterOperation = (operation, _) => + { + if (operation == JournaledStateOperation.Delete) + { + Assert.NotNull(registration); + Assert.False(registration.IsCompleted); + Assert.Equal(0, newBefore); + Assert.Equal(0, newAfter); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.NotNull(registration); + await WaitFor(registration); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, newBefore); + Assert.Equal(1, newAfter); + } + + [Fact] + public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var valid = new DelegateTestHook(); + var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + await Assert.ThrowsAsync(() => WaitFor(invalid)); + await WaitFor(Task.WhenAll(first, accepted)); + Assert.Same(valid, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause() + { + var failure = new IOException("Storage failed."); + var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = failure }; + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(storage.BlockedAppendStarted.Task); + var registration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.TrySetResult(); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); + Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); + Assert.Empty(manager.Hooks); + var fenced = await Assert.ThrowsAsync(() => + manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); + Assert.Same(failure, fenced.InnerException); + } + finally + { + storage.ReleaseAppend.TrySetResult(); + } + } + + [Fact] + public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOperation() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + var entered = NewSignal(); + var release = NewSignal(); + var original = new DelegateTestHook + { + AfterOperationAsync = async (_, _) => + { + entered.TrySetResult(); + await release.Task; + } + }; + await manager.AddHookAsync(original, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + try + { + await WaitFor(entered.Task); + var added = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var removed = manager.RemoveHookAsync(original, cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var shutdown = manager.DisposeAsync().AsTask(); + Assert.False(shutdown.IsCompleted); + release.TrySetResult(); + await WaitFor(Task.WhenAll(write, shutdown)); + await Assert.ThrowsAnyAsync(() => WaitFor(added)); + await Assert.ThrowsAnyAsync(() => WaitFor(removed)); + Assert.Same(original, Assert.Single(manager.Hooks)); + Assert.Single(storage.Appends); + } + finally + { + release.TrySetResult(); + } + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -183,7 +473,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var expected = snapshot ? JournaledStateOperation.Snapshot : JournaledStateOperation.Write; CancellationToken ownedToken = default; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, token) => { @@ -213,7 +503,7 @@ public async Task Hooks_DelegateOrderSurroundsCaptureAndActualAcknowledgement(bo await Task.Yield(); events.Add("after-async"); } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -254,7 +544,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta var failure = new IOException("Prerequisite failed."); var fail = true; var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => { @@ -264,7 +554,7 @@ public async Task Hooks_BeforeFailurePreservesStateForExplicitRetry(JournaledSta } }, AfterOperation = (_, _) => after++ - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var pendingBytes = manager.PendingWriteByteCount; @@ -307,7 +597,7 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); var fail = true; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => { @@ -316,8 +606,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw first; } } - }); - manager.Hooks.Add(new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -328,8 +618,8 @@ public async Task Hooks_AfterFailureReportsCompletedOperationAndRunsRemainingHoo throw second; } } - }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var caught = await Assert.ThrowsAsync(() => (operation == JournaledStateOperation.Delete @@ -366,7 +656,7 @@ public async Task Hooks_StorageFailureSkipsAfterHooksAndPreservesTerminalCause() await using var manager = CreateTestSystem(storage).Manager; var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var after = 0; - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; Assert.Same(failure, await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); @@ -387,7 +677,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - manager.Hooks.Add(new DelegateTestHook( + await manager.AddHookAsync(new DelegateTestHook( (_, _) => { Assert.Equal(3, state["late"]); @@ -395,8 +685,8 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar events.Add("capture-prerequisite"); return default; }, - (_, _) => events.Add("capture-after"))); - manager.Hooks.Add(new DelegateTestHook + (_, _) => events.Add("capture-after")), cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -406,7 +696,7 @@ public async Task Hooks_FinalPrerequisiteCoversChangesArrivingDuringLaterOrdinar events.Add("ordinary-complete"); }, AfterOperation = (_, _) => events.Add("ordinary-after") - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["business"] = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -459,7 +749,7 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() Assert.Single(storage.Appends); completed.TrySetResult(); }; - manager.Hooks.Add(new DelegateTestHook(before, after)); + await manager.AddHookAsync(new DelegateTestHook(before, after), cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); state["business"] = 1; @@ -497,21 +787,21 @@ public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrd await using var manager = CreateTestSystem().Manager; List events = []; manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("second-before"), AfterOperation = (_, _) => events.Add("second-after") - }); - manager.Hooks.Add(new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("third-before"), AfterOperation = (_, _) => events.Add("third-after") - }); - manager.Hooks.Insert(0, new DelegateTestHook + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => events.Add("first-before"), AfterOperation = (_, _) => events.Add("first-after") - }); + }, index: 0, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal( @@ -529,7 +819,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var after = 0; - manager.Hooks.Add(new DelegateTestHook( + await manager.AddHookAsync(new DelegateTestHook( async (operation, token) => { if (!deleting) @@ -554,7 +844,7 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() Assert.Equal(0, state.Value); Assert.Equal(1, storage.DeleteCount); after++; - })); + }), cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; await manager.WriteStateAsync(TestContext.Current.CancellationToken); @@ -586,7 +876,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -595,7 +885,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await release.Task; drained = true; } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -615,7 +905,8 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() await Assert.ThrowsAnyAsync(() => queued); Assert.True(drained); Assert.Empty(storage.Appends); - Assert.Throws(() => manager.Hooks.Clear()); + await Assert.ThrowsAsync(() => + manager.RemoveHookAsync(Assert.Single(manager.Hooks), cancellationToken: TestContext.Current.CancellationToken).AsTask()); } finally { @@ -640,11 +931,11 @@ public async Task Hooks_ShutdownCancelsQueuedInitializationAfterOwnedCallbackDra entered.TrySetResult(); await release.Task; }; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = after ? null : callback, AfterOperationAsync = after ? callback : null - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -711,11 +1002,11 @@ public async Task Hooks_ShutdownCallbackFailureDrainsActualStorageAndPreservesIt manager.RegisterStateMachine("state", state); CancellationTokenRegistration registration = default; var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, token) => registration = token.Register(() => throw cancellationFailure), AfterOperation = (_, _) => after++ - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -778,7 +1069,7 @@ public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() var release = NewSignal(); Task? reenteredDisposal = null; var callbackCount = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -790,7 +1081,7 @@ public async Task Hooks_ShutdownCallbackReentrySharesDisposalCompletion() entered.TrySetResult(); await release.Task; } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -824,7 +1115,7 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() var first = new IOException("First cleanup failed."); var second = new IOException("Second cleanup failed."); List events = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, token) => { @@ -835,9 +1126,9 @@ public async Task Hooks_ShutdownDrainsEveryAfterHookDespiteCleanupFailures() events.Add("first"); throw first; } - }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => throw second }); - manager.Hooks.Add(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }); + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => throw second }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook { AfterOperation = (_, _) => events.Add("last") }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -874,11 +1165,11 @@ await context.Run(async () => var state = new DurableValue("state", manager, CreateValueCodec()); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, _) => before.Add(operation), AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var append = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -916,7 +1207,7 @@ public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() var failure = new IOException("Snapshot prerequisite failed."); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperationAsync = async (operation, _) => { @@ -929,7 +1220,7 @@ public async Task Hooks_BeforeFailureRestoresSnapshotForAlreadyQueuedWrite() } }, AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -968,11 +1259,11 @@ public async Task Hooks_SnapshotAcknowledgementPreservesLaterQueuedSnapshot() var state = new DurableValue("state", manager, CreateValueCodec()); List before = []; List after = []; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (operation, _) => before.Add(operation), AfterOperation = (operation, _) => after.Add(operation) - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1010,7 +1301,7 @@ public async Task Hooks_CoalescedCallersShareActualOperationCallbacks() var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); var before = 0; var after = 0; - manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1044,7 +1335,7 @@ public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() var entered = NewSignal(); var release = NewSignal(); var after = 0; - manager.Hooks.Add(new DelegateTestHook + await manager.AddHookAsync(new DelegateTestHook { AfterOperationAsync = async (_, _) => { @@ -1054,7 +1345,7 @@ public async Task Hooks_StateChangesDuringAfterCallbackPersistInNextOperation() await release.Task; } } - }); + }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state["first"] = 1; var first = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -1099,7 +1390,7 @@ public async Task Hooks_NoByteWriteStillCompletesLogicalOperation() var state = new DurableValue("state", manager, CreateValueCodec()); var before = 0; var after = 0; - manager.Hooks.Add(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }); + await manager.AddHookAsync(new DelegateTestHook { BeforeOperation = (_, _) => before++, AfterOperation = (_, _) => after++ }, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); state.Value = 1; await manager.WriteStateAsync(TestContext.Current.CancellationToken); From 5ab481ca2658b660d88d65da463ea1b771454ff5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 17:25:32 -0700 Subject: [PATCH 18/22] refactor(journaling)!: keep operation hooks private --- .../grains/journaling/runtime-behavior.md | 15 ++--- .../CompatibilitySuppressions.xml | 14 ----- .../IJournaledStateManager.cs | 24 +++----- .../JournaledStateManager.Hooks.cs | 41 ++----------- .../JournaledStateManager.cs | 25 +++----- src/Orleans.Journaling/README.md | 15 +++-- .../Orleans.Journaling/Orleans.Journaling.cs | 2 - .../DurableListJournalBenchmarks.cs | 2 - .../JournaledJobShardDiscoveryTests.cs | 1 - .../DurableCollectionDirectWriteTests.cs | 2 - .../DurableListDirectWriteTests.cs | 2 - .../KeyedJournalingRegistrationTests.cs | 1 - .../StateManagerHookTests.cs | 60 +++++++++---------- 13 files changed, 68 insertions(+), 136 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 7230ffd7315..70d297fcb49 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -75,18 +75,19 @@ staged mutations from other calls. ## Journal operation hooks -Activation-scoped features coordinate prerequisites and completion through -. The owner exposes a stable, lazily -allocated read-only view of registrations. -Use and - to change the set. Idle changes -apply synchronously; busy changes queue between complete operations, after current after callbacks. +Features coordinate prerequisites and completion by registering + callbacks through + and +. +Registration completes synchronously before initialization, supporting constructors. +After initialization begins, idle changes apply synchronously; busy changes queue between +complete operations, after current after callbacks. The hook set stays stable throughout each operation. Queued changes also separate coalesced writes, so writes admitted after a change observe that change. Cancellation ends only the caller's wait while the manager owns admitted changes. Every owner implements the registration methods and callback boundaries; delegating owners forward them to their inner owner. Synchronous state callbacks can request a queued change and observe its -completion outside the callback. Standalone callers coordinate view inspection with changes. +completion outside the callback. Each actual append, snapshot, or deletion runs before callbacks in reverse list order, outside the manager lock. Add the final prerequisite at index zero so its callback runs last before diff --git a/src/Orleans.Journaling/CompatibilitySuppressions.xml b/src/Orleans.Journaling/CompatibilitySuppressions.xml index 3b379c6860f..5e86005fc7b 100644 --- a/src/Orleans.Journaling/CompatibilitySuppressions.xml +++ b/src/Orleans.Journaling/CompatibilitySuppressions.xml @@ -281,13 +281,6 @@ lib/net10.0/Orleans.Journaling.dll true - - CP0006 - P:Orleans.Journaling.IJournaledStateManager.Hooks - lib/net10.0/Orleans.Journaling.dll - lib/net10.0/Orleans.Journaling.dll - true - CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey @@ -337,13 +330,6 @@ lib/net8.0/Orleans.Journaling.dll true - - CP0006 - P:Orleans.Journaling.IJournaledStateManager.Hooks - lib/net8.0/Orleans.Journaling.dll - lib/net8.0/Orleans.Journaling.dll - true - CP0006 P:Orleans.Journaling.IJournalMetadata.FormatKey diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 761eb1d50cd..5c18162570f 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -12,20 +12,6 @@ namespace Orleans.Journaling; /// public interface IJournaledStateManager : IAsyncDisposable { - /// - /// Gets the read-only view of registered journal operation hooks. - /// - /// - /// Inspect hooks on the owner's logical execution context, coordinating standalone inspection - /// with registration changes. Use and - /// to change registrations. Before hooks execute in - /// reverse list order; after hooks execute in forward list order. Register the final capture - /// prerequisite at index zero so it runs last before capture or deletion. - /// Registration is independent of state-machine registration and persists through recovery and deletion. - /// Every implementation provides this list and invokes its hooks at the documented operation boundaries. - /// - IReadOnlyList Hooks { get; } - /// /// Adds a hook before the next persistence operation which follows this registration. /// @@ -34,8 +20,14 @@ public interface IJournaledStateManager : IAsyncDisposable /// The token used to cancel the caller's wait. /// A completion representing application of the registration. /// - /// Idle registration applies synchronously. Otherwise it is queued between owned operations, - /// after the current operation's after hooks finish. Queued registration is a write-coalescing barrier. + /// Registration completes synchronously before initialization begins, supporting constructor registration. + /// After initialization begins, idle registration applies synchronously; otherwise it is queued + /// between owned operations, after the current operation's after hooks finish. + /// Before hooks execute in reverse registration order; after hooks execute in forward order. + /// Register the final capture prerequisite at index zero so it runs last before capture or deletion. + /// Registration is independent of state-machine registration and persists through recovery and deletion. + /// Every implementation provides registration and invokes hooks at the documented operation boundaries. + /// Queued registration is a write-coalescing barrier. /// Cancellation ends the caller's wait while admitted registration remains owned by the manager. /// Synchronous state callbacks can request registration and observe its completion outside the callback. /// Await registration before staging changes whose safety depends on the added hook. diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 8cb1fe842cf..751ece9e5a3 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -1,22 +1,10 @@ -using System.Collections; using System.Runtime.ExceptionServices; namespace Orleans.Journaling; internal partial class JournaledStateManager { - private HookCollection? _hooks; - - public IReadOnlyList Hooks - { - get - { - lock (_lock) - { - return _hooks ??= new(); - } - } - } + private List? _hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => MutateHookAsync(hook, index, remove: false, cancellationToken); @@ -54,11 +42,14 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove var hooks = _hooks ??= new(); if (remove) { - hooks.Remove(hook); + if (!hooks.Remove(hook)) + { + throw new InvalidOperationException("The journal operation hook is not registered."); + } } else { - hooks.Insert(index, hook); + hooks.Insert(index ?? hooks.Count, hook); } } @@ -99,26 +90,6 @@ private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, } } - private sealed class HookCollection : IReadOnlyList - { - private readonly List _items = []; - - public int Count => _items.Count; - public IJournaledStateHook this[int index] => _items[index]; - public IEnumerator GetEnumerator() => _items.GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - - public void Insert(int? index, IJournaledStateHook hook) => _items.Insert(index ?? _items.Count, hook); - - public void Remove(IJournaledStateHook hook) - { - if (!_items.Remove(hook)) - { - throw new InvalidOperationException("The journal operation hook is not registered."); - } - } - } - private sealed class HookMutationWorkItem(IJournaledStateHook hook, int? index, bool remove) : WorkItem { public IJournaledStateHook Hook { get; } = hook; diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 1f3e6c18d62..fb634b4f169 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -202,6 +202,7 @@ private async Task WorkLoop() _workItemRunning = true; } + Exception? recoveryFailure = null; try { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -214,24 +215,18 @@ private async Task WorkLoop() } catch (Exception exception) { - try - { - LogErrorProcessingWorkItems(_shared.Logger, exception); - } - finally - { - lock (_lock) - { - FaultQueuedWorkItemsUnderLock(exception); - _workItemRunning = false; - } - } + recoveryFailure = exception; + LogErrorProcessingWorkItems(_shared.Logger, exception); } finally { lock (_lock) { _workItemRunning = false; + if (recoveryFailure is not null) + { + FaultQueuedWorkItemsUnderLock(recoveryFailure); + } } } @@ -345,10 +340,8 @@ private async Task WorkLoop() { case HookMutationWorkItem mutation: { - lock (_lock) - { - ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); - } + // The work loop owns the hook set while _workItemRunning is true. + ApplyHookMutation(mutation.Hook, mutation.Index, mutation.Remove); break; } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 4e43c8c8334..ebba60fad84 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -266,18 +266,17 @@ of the ambient grain context, including when the caller subsequently enrolls the ## Journal operation hooks -`IJournaledStateManager.Hooks` is a lazily allocated, stable read-only view of `IJournaledStateHook` -registrations. Use `AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to change registrations. -Changes apply synchronously when idle, or queue between complete journal operations while busy. +Use `IJournaledStateManager.AddHookAsync(hook, index)` and `RemoveHookAsync(hook)` to register +`IJournaledStateHook` callbacks. Registration completes synchronously before initialization, +supporting constructors. After initialization begins, changes apply synchronously when idle, +or queue between complete journal operations while busy. Each operation keeps its hook set through preparation, capture, storage ACK and after callbacks. Queued changes are write-coalescing barriers: writes admitted afterward observe the changed set. Caller cancellation ends only its wait for an admitted change; the manager retains ownership. Await registration before staging changes whose safety depends on the new hook. -Every journal owner implementation provides the hook list and invokes registered callbacks at -the operation boundaries described below. Delegating owners forward registration methods and the -read-only view to their inner owner. Synchronous state callbacks can request a queued registration -and observe completion outside the callback. Standalone callers coordinate view inspection with -registration changes. +Every journal owner implementation provides registration and invokes callbacks at the operation +boundaries described below. Delegating owners forward registration methods to their inner owner. +Synchronous state callbacks can request a queued registration and observe completion outside the callback. For each actual write, snapshot, or deletion, before hooks run in reverse list order outside the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 969a8c672d1..cf2ecdf72e7 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -227,8 +227,6 @@ public partial interface IJournaledStateHook public partial interface IJournaledStateManager : System.IAsyncDisposable { - System.Collections.Generic.IReadOnlyList Hooks { get; } - long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, System.Threading.CancellationToken cancellationToken = default); diff --git a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs index fa2f4658687..5cb36fbffcf 100644 --- a/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs +++ b/test/Benchmarks/Journaling/DurableListJournalBenchmarks.cs @@ -128,8 +128,6 @@ private sealed class BenchmarkJournalManager(OrleansBinaryJournalBufferWriter bu { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index c06a4576ec7..39e4213c5c1 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -727,7 +727,6 @@ public IJournaledStateManager CreateStandalone(JournalId journalId) private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IReadOnlyList Hooks => inner.Hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => inner.AddHookAsync(hook, index, cancellationToken); public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => diff --git a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs index 94fc6b4ff09..ede9edb2f56 100644 --- a/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableCollectionDirectWriteTests.cs @@ -195,8 +195,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs index bd4176f6403..245561f8c50 100644 --- a/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs +++ b/test/Orleans.Journaling.Tests/DurableListDirectWriteTests.cs @@ -94,8 +94,6 @@ private sealed class TestJournalManager(TestJournalStreamWriter writer) : IJourn { private readonly List _hooks = []; - public IReadOnlyList Hooks => _hooks; - public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) { _hooks.Insert(index ?? _hooks.Count, hook); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 5d91cafdd8b..dd7945ec7cb 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -746,7 +746,6 @@ public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public IReadOnlyList Hooks => inner.Hooks; public ValueTask AddHookAsync(IJournaledStateHook hook, int? index = null, CancellationToken cancellationToken = default) => inner.AddHookAsync(hook, index, cancellationToken); public ValueTask RemoveHookAsync(IJournaledStateHook hook, CancellationToken cancellationToken = default) => diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 4166f54ab9a..3918d093fed 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -65,30 +65,28 @@ public async Task Hooks_QueueActivityUsesSelectedOperationAndCompletesWithStorag } [Fact] - public async Task Hooks_IdleRegistrationAppliesSynchronouslyAndExposesStableView() + public async Task Hooks_IdleRegistrationAppliesSynchronously() { await using var manager = CreateTestSystem().Manager; _ = new DurableValue("state", manager, CreateValueCodec()); var field = typeof(JournaledStateManager).GetField("_hooks", BindingFlags.Instance | BindingFlags.NonPublic)!; Assert.Null(field.GetValue(manager)); - var hook = new DelegateTestHook(); + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - var hooks = manager.Hooks; - Assert.False(hooks is IList); - Assert.Contains(hook, hooks); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); - Assert.Empty(manager.Hooks); await Assert.ThrowsAsync(() => manager.AddHookAsync(null!, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, before); Assert.True(manager.AddHookAsync(hook, index: 0, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); + Assert.Equal(1, before); Assert.True(manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); await using var withoutHooks = CreateTestSystem().Manager; await withoutHooks.InitializeAsync(TestContext.Current.CancellationToken); @@ -102,12 +100,9 @@ public async Task Hooks_ShutdownRejectsRegistrationAndRemoval() var manager = CreateTestSystem().Manager; var hook = new DelegateTestHook(); await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); - var hooks = manager.Hooks; await manager.DisposeAsync(); - Assert.Same(hooks, manager.Hooks); - Assert.Same(hook, Assert.Single(hooks)); await Assert.ThrowsAsync(() => manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); await Assert.ThrowsAsync(() => manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); } @@ -135,10 +130,8 @@ public async Task Hooks_RegistrationDuringRecoveryAppliesBeforeLaterPersistence( await WaitFor(entered.Task); Assert.False(initialization.IsCompleted); Assert.False(registration.IsCompleted); - Assert.Empty(manager.Hooks); release.TrySetResult(); await WaitFor(Task.WhenAll(initialization, registration)); - Assert.Same(hook, Assert.Single(manager.Hooks)); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, before); Assert.Equal(1, after); @@ -169,21 +162,27 @@ public async Task Hooks_QueuedRegistrationReportsRecoveryFailureAndIdleRegistrat .Returns(call => ReadAsync(call.Arg(), call.Arg())); await using var manager = CreateTestSystem(storage).Manager; var initialization = manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask(); - var failedRegistration = manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask(); + var failedBefore = 0; + var failedRegistration = manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => failedBefore++ + }, cancellationToken: TestContext.Current.CancellationToken).AsTask(); try { await WaitFor(entered.Task); release.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(initialization))); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(failedRegistration))); - Assert.Empty(manager.Hooks); - var hook = new DelegateTestHook(); + var before = 0; + var hook = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; Assert.True(manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).IsCompletedSuccessfully); Assert.Equal(1, reads); fail = false; await manager.InitializeAsync(TestContext.Current.CancellationToken); Assert.Equal(2, reads); - Assert.Same(hook, Assert.Single(manager.Hooks)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, failedBefore); + Assert.Equal(1, before); } finally { @@ -258,17 +257,14 @@ public async Task Hooks_BusyMutationWaitsForCompleteOperationAndSurvivesCallerCa var next = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); Assert.False(added.IsCompleted); Assert.False(removed.IsCompleted); - Assert.Same(original, Assert.Single(manager.Hooks)); Assert.Equal(0, newBefore); Assert.Equal(0, newAfter); caller.Cancel(); await Assert.ThrowsAnyAsync(() => added); - Assert.Same(original, Assert.Single(manager.Hooks)); release.TrySetResult(); storage.ReleaseAppend.TrySetResult(); await WaitFor(Task.WhenAll(first, removed, next)); - Assert.Same(replacement, Assert.Single(manager.Hooks)); Assert.Equal(1, oldBefore); Assert.Equal(1, oldAfter); Assert.Equal(1, newBefore); @@ -381,13 +377,14 @@ public async Task Hooks_FailedQueuedMutationReportsErrorAndLeavesOwnerUsable() { await WaitFor(storage.BlockedAppendStarted.Task); var invalid = manager.AddHookAsync(new DelegateTestHook(), index: 1, cancellationToken: TestContext.Current.CancellationToken).AsTask(); - var valid = new DelegateTestHook(); + var before = 0; + var valid = new DelegateTestHook { BeforeOperation = (_, _) => before++ }; var accepted = manager.AddHookAsync(valid, cancellationToken: TestContext.Current.CancellationToken).AsTask(); storage.ReleaseAppend.TrySetResult(); await Assert.ThrowsAsync(() => WaitFor(invalid)); await WaitFor(Task.WhenAll(first, accepted)); - Assert.Same(valid, Assert.Single(manager.Hooks)); await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, before); } finally { @@ -411,7 +408,6 @@ public async Task Hooks_StorageFailureFaultsQueuedRegistrationWithOriginalCause( storage.ReleaseAppend.TrySetResult(); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(write))); Assert.Same(failure, await Assert.ThrowsAsync(() => WaitFor(registration))); - Assert.Empty(manager.Hooks); var fenced = await Assert.ThrowsAsync(() => manager.AddHookAsync(new DelegateTestHook(), cancellationToken: TestContext.Current.CancellationToken).AsTask()); Assert.Same(failure, fenced.InnerException); @@ -452,7 +448,6 @@ public async Task Hooks_ShutdownCancelsQueuedMutationWithoutChangingCommittedOpe await WaitFor(Task.WhenAll(write, shutdown)); await Assert.ThrowsAnyAsync(() => WaitFor(added)); await Assert.ThrowsAnyAsync(() => WaitFor(removed)); - Assert.Same(original, Assert.Single(manager.Hooks)); Assert.Single(storage.Appends); } finally @@ -818,10 +813,12 @@ public async Task Hooks_DeleteWaitsForPrerequisitesAndCompletesAfterReset() var deleting = false; var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var before = 0; var after = 0; await manager.AddHookAsync(new DelegateTestHook( async (operation, token) => { + before++; if (!deleting) { Assert.Equal(JournaledStateOperation.Write, operation); @@ -859,7 +856,9 @@ await manager.AddHookAsync(new DelegateTestHook( release.TrySetResult(); await deletion.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); Assert.Equal(1, after); - Assert.Single(manager.Hooks); + deleting = false; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(3, before); } finally { @@ -876,7 +875,7 @@ public async Task Hooks_ShutdownCallbackFailureStillDrainsOwnedPreparation() var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var drained = false; var failure = new IOException("Cancellation callback failed."); - await manager.AddHookAsync(new DelegateTestHook + var hook = new DelegateTestHook { BeforeOperationAsync = async (_, token) => { @@ -885,7 +884,8 @@ await manager.AddHookAsync(new DelegateTestHook await release.Task; drained = true; } - }, cancellationToken: TestContext.Current.CancellationToken); + }; + await manager.AddHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken); await manager.InitializeAsync(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); try @@ -906,7 +906,7 @@ await manager.AddHookAsync(new DelegateTestHook Assert.True(drained); Assert.Empty(storage.Appends); await Assert.ThrowsAsync(() => - manager.RemoveHookAsync(Assert.Single(manager.Hooks), cancellationToken: TestContext.Current.CancellationToken).AsTask()); + manager.RemoveHookAsync(hook, cancellationToken: TestContext.Current.CancellationToken).AsTask()); } finally { From 8d58d858f3340bbbf1846ea05f73f71c3b358370 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 17:52:04 -0700 Subject: [PATCH 19/22] fix(journaling): capture directly after synchronous final hooks --- .../grains/journaling/runtime-behavior.md | 3 + src/Orleans.Journaling/IJournaledStateHook.cs | 2 + .../JournaledStateManager.Hooks.cs | 11 -- .../JournaledStateManager.cs | 7 +- src/Orleans.Journaling/README.md | 3 + .../StateManagerHookTests.cs | 106 ++++++++++++++++++ 6 files changed, 120 insertions(+), 12 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 70d297fcb49..90d8eb3e3b2 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -94,6 +94,9 @@ the manager lock. Add the final prerequisite at index zero so its callback runs synchronous capture or storage deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. Preserve operation-local bookkeeping for the captured batch separately from changes staged later. +When the final callback completes synchronously, capture or the start of deletion follows in the +same work-loop continuation. Other owner turns can stage changes while an incomplete callback is awaited, +so its prerequisite covers those changes through capture or deletion. Storage acknowledgement and registered-state acknowledgement or reset precede after callbacks. All after callbacks run in list order, including for successful zero-byte writes. Coalesced callers diff --git a/src/Orleans.Journaling/IJournaledStateHook.cs b/src/Orleans.Journaling/IJournaledStateHook.cs index 10c8d5eb9f1..c2e187254ff 100644 --- a/src/Orleans.Journaling/IJournaledStateHook.cs +++ b/src/Orleans.Journaling/IJournaledStateHook.cs @@ -21,6 +21,8 @@ public enum JournaledStateOperation /// /// Before hooks run in reverse list order on the owner's logical execution context, outside its lock. /// The hook at index zero supplies the final prerequisite before capture or deletion. +/// When the final callback completes synchronously, capture or the start of deletion follows in the +/// same work-loop continuation. /// Prerequisites remain valid until capture or deletion, covering changes staged during asynchronous /// preparation and while the journal work loop resumes. After hooks run in forward list order after /// storage acknowledgement and state acknowledgement or reset, including successful writes which diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 751ece9e5a3..e02c5f583b4 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -53,17 +53,6 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove } } - private async ValueTask InvokeBeforeHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) - { - for (var i = _hooks!.Count - 1; i >= 0; i--) - { - cancellationToken.ThrowIfCancellationRequested(); - await _hooks[i].BeforeOperationAsync(operation, cancellationToken).ConfigureAwait(true); - } - - cancellationToken.ThrowIfCancellationRequested(); - } - private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { List? failures = null; diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index fb634b4f169..e0a8b676ba2 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -328,7 +328,12 @@ private async Task WorkLoop() if (operationType is { } operation && _hooks is { Count: > 0 }) { beforeHookRunning = true; - await InvokeBeforeHooksAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + // Keep synchronous final completion and capture in the same work-loop continuation. + for (var i = _hooks.Count - 1; i >= 0; i--) + { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); + await _hooks[i].BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + } _shutdownCancellation.Token.ThrowIfCancellationRequested(); beforeHookRunning = false; diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index ebba60fad84..d50c3b34275 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -282,6 +282,9 @@ For each actual write, snapshot, or deletion, before hooks run in reverse list o the manager lock. Add the final prerequisite with `index: 0` so it runs last before the work loop captures the registered states or starts deletion. Prerequisites remain valid through capture or deletion, covering changes staged during asynchronous preparation and while the work loop resumes. +When the final callback completes synchronously, capture or the start of deletion follows in the +same work-loop continuation. Other owner turns can stage changes while an incomplete callback is awaited, +so its prerequisite covers those changes through capture or deletion. Keep operation-local bookkeeping for the captured batch separate from later pending changes. After hooks run in list order after storage acknowledgement and state acknowledgement or reset. diff --git a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs index 3918d093fed..8833f87cb85 100644 --- a/test/Orleans.Journaling.Tests/StateManagerHookTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerHookTests.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Reflection; +using System.Threading.Channels; using Microsoft.Extensions.DependencyInjection; using NSubstitute; using Orleans.Diagnostics; @@ -776,6 +777,77 @@ public async Task Hooks_CanceledCallerRetainsFinalPreparationAndPostCompletion() } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Hooks_SynchronousFinalPrerequisiteAndCaptureShareContinuationAfterAsyncEarlierHook(bool snapshot) + { + var scheduler = new QueuedHookTaskScheduler(); + await scheduler.Run(async () => + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new DurableDictionary("state", manager, CreateDictionaryCodec()); + List events = []; + List prepared = []; + manager.RegisterStateMachine("ack", new HookAcknowledgementState(events)); + var queueMutation = true; + Task? mutation = null; + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperation = (_, _) => + { + events.Add("final-before"); + prepared.AddRange(state.Keys); + if (queueMutation) + { + Assert.Empty(state); + queueMutation = false; + mutation = scheduler.Queue(() => + { + state["intent"] = 1; + events.Add("queued-mutation"); + }); + } + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.AddHookAsync(new DelegateTestHook + { + BeforeOperationAsync = async (_, _) => + { + events.Add("ordinary-before"); + await Task.Yield(); + events.Add("ordinary-complete"); + } + }, cancellationToken: TestContext.Current.CancellationToken); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(mutation); + await mutation; + Assert.Empty(prepared); + Assert.Equal(1, state["intent"]); + await using (var recoveredManager = CreateTestSystem(storage).Manager) + { + var recovered = new DurableDictionary("state", recoveredManager, CreateDictionaryCodec()); + await recoveredManager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Empty(recovered); + } + + Assert.Equal( + ["ordinary-before", "ordinary-complete", "final-before", "capture", "ack", "queued-mutation"], + events); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(["intent"], prepared); + await using var replayManager = CreateTestSystem(storage).Manager; + var replayed = new DurableDictionary("state", replayManager, CreateDictionaryCodec()); + await replayManager.InitializeAsync(TestContext.Current.CancellationToken); + var intent = Assert.Single(replayed); + Assert.Equal("intent", intent.Key); + Assert.Equal(1, intent.Value); + }); + } + [Fact] public async Task Hooks_BeforeCallbacksRunInReverseAndAfterCallbacksRunInListOrder() { @@ -1427,6 +1499,40 @@ public void Hooks_FailureOutcomesRoundTripWithTypeAndOriginalCause(bool after) Assert.Equal(original.Message, copy.Message); } + private sealed class QueuedHookTaskScheduler : TaskScheduler + { + private readonly Channel _queue = Channel.CreateUnbounded(); + + public override int MaximumConcurrencyLevel => 1; + protected override IEnumerable? GetScheduledTasks() => null; + protected override void QueueTask(Task task) => Assert.True(_queue.Writer.TryWrite(task)); + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; + + public Task Queue(Action action) => + Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this); + + public async Task Run(Func action) + { + var task = Task.Factory.StartNew(action, TestContext.Current.CancellationToken, TaskCreationOptions.DenyChildAttach, this).Unwrap(); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(null); + try + { + Assert.True(TryExecuteTask(next)); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + } + } + + await task; + } + } + private sealed class DelegateTestHook( Func? before = null, Action? after = null) : IJournaledStateHook From 2da269ade0008ce9f356f4a2c838a3795711ace8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 18:55:34 -0700 Subject: [PATCH 20/22] refactor(journaling): extract capture-safe operation helpers --- .../JournaledStateManager.Hooks.cs | 6 ++++++ .../JournaledStateManager.cs | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index e02c5f583b4..8be5e2e9d00 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -53,6 +53,12 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove } } + private ValueTask InvokeBeforeHookAsync(int index, JournaledStateOperation operation, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return _hooks![index].BeforeOperationAsync(operation, cancellationToken); + } + private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) { List? failures = null; diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index e0a8b676ba2..fb54d88a18a 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -307,13 +307,7 @@ private async Task WorkLoop() "The journaled state operation was queued before deletion reset its state."); } - operationType = workItem switch - { - WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, - WriteStateWorkItem => JournaledStateOperation.Write, - DeleteStateWorkItem => JournaledStateOperation.Delete, - _ => null - }; + operationType = GetOperationType(workItem); if (operationType == JournaledStateOperation.Snapshot) { compactionReason = _snapshotRequired @@ -331,8 +325,7 @@ private async Task WorkLoop() // Keep synchronous final completion and capture in the same work-loop continuation. for (var i = _hooks.Count - 1; i >= 0; i--) { - _shutdownCancellation.Token.ThrowIfCancellationRequested(); - await _hooks[i].BeforeOperationAsync(operation, _shutdownCancellation.Token).ConfigureAwait(true); + await InvokeBeforeHookAsync(i, operation, _shutdownCancellation.Token).ConfigureAwait(true); } _shutdownCancellation.Token.ThrowIfCancellationRequested(); @@ -698,6 +691,14 @@ private async Task WorkLoop() } } + private JournaledStateOperation? GetOperationType(WorkItem workItem) => workItem switch + { + WriteStateWorkItem when IsSnapshotRequired => JournaledStateOperation.Snapshot, + WriteStateWorkItem => JournaledStateOperation.Write, + DeleteStateWorkItem => JournaledStateOperation.Delete, + _ => null + }; + private void Fence(Exception exception) { lock (_lock) From ded01ef7454fd852887ee8d6a2b2b5f94c7c0074 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 19:38:59 -0700 Subject: [PATCH 21/22] refactor(journaling): simplify recovery and before-hook traversal --- .../JournaledStateManager.Hooks.cs | 17 ++++++++++++++--- src/Orleans.Journaling/JournaledStateManager.cs | 16 +++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 8be5e2e9d00..5b5c68fd943 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -53,10 +53,21 @@ private void ApplyHookMutation(IJournaledStateHook hook, int? index, bool remove } } - private ValueTask InvokeBeforeHookAsync(int index, JournaledStateOperation operation, CancellationToken cancellationToken) + private ValueTask InvokeBeforeHooksAsync(ref int index, JournaledStateOperation operation, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - return _hooks![index].BeforeOperationAsync(operation, cancellationToken); + while (index >= 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var pending = _hooks![index--].BeforeOperationAsync(operation, cancellationToken); + if (!pending.IsCompleted) + { + return pending; + } + + pending.GetAwaiter().GetResult(); + } + + return default; } private async ValueTask InvokeAfterHooksAsync(JournaledStateOperation operation, CancellationToken cancellationToken) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index fb54d88a18a..4ef2cfd34c5 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -241,17 +241,7 @@ private async Task WorkLoop() lock (_lock) { - var retryRequested = false; - foreach (var workItem in _workQueue) - { - if (workItem is InitializeWorkItem) - { - retryRequested = true; - break; - } - } - - if (retryRequested) + if (_workQueue.Count > 0) { break; } @@ -323,9 +313,9 @@ private async Task WorkLoop() { beforeHookRunning = true; // Keep synchronous final completion and capture in the same work-loop continuation. - for (var i = _hooks.Count - 1; i >= 0; i--) + for (var i = _hooks.Count - 1; i >= 0;) { - await InvokeBeforeHookAsync(i, operation, _shutdownCancellation.Token).ConfigureAwait(true); + await InvokeBeforeHooksAsync(ref i, operation, _shutdownCancellation.Token).ConfigureAwait(true); } _shutdownCancellation.Token.ThrowIfCancellationRequested(); From d39127370199503f7a6d916776b07a757aa4b096 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 11 Oct 2026 20:02:15 -0700 Subject: [PATCH 22/22] refactor(journaling): derive disposed state from shared completion --- src/Orleans.Journaling/JournaledStateManager.Hooks.cs | 2 +- src/Orleans.Journaling/JournaledStateManager.cs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs index 5b5c68fd943..115729e6ac5 100644 --- a/src/Orleans.Journaling/JournaledStateManager.Hooks.cs +++ b/src/Orleans.Journaling/JournaledStateManager.Hooks.cs @@ -20,7 +20,7 @@ private ValueTask MutateHookAsync(IJournaledStateHook hook, int? index, bool rem lock (_lock) { ThrowIfFenced(); - ObjectDisposedException.ThrowIf(_disposed != 0, this); + ObjectDisposedException.ThrowIf(_disposeTask is not null, this); _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is null || (!_workItemRunning && _workQueue.Count == 0)) { diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 4ef2cfd34c5..bd0b1ae47bb 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -36,7 +36,6 @@ internal partial class JournaledStateManager : IJournaledStateManager, IJournalS private Exception? _failure; // A snapshot requirement not yet claimed by a write. Consume under the lock before preparation awaits. private bool _snapshotRequired; - private int _disposed; private Task? _disposeTask; public JournaledStateManager(JournaledStateManagerShared shared, IJournalStorageProvider storageProvider, IGrainContext grainContext) @@ -113,7 +112,7 @@ public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachin lock (_lock) { ThrowIfFenced(); - ObjectDisposedException.ThrowIf(_disposed != 0, this); + ObjectDisposedException.ThrowIf(_disposeTask is not null, this); if (_states.TryGetValue(name, out var existing) && existing is not RetiredStateMachine) { stateMachine = existing; @@ -130,7 +129,7 @@ protected void EnsureRegistrationAllowed() lock (_lock) { ThrowIfFenced(); - ObjectDisposedException.ThrowIf(_disposed != 0, this); + ObjectDisposedException.ThrowIf(_disposeTask is not null, this); _shutdownCancellation.Token.ThrowIfCancellationRequested(); if (_workLoop is not null) { @@ -1212,6 +1211,7 @@ public ValueTask DisposeAsync() if (_disposeTask is null) { var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + // Publish before shutdown cancellation can reenter disposal. _disposeTask = completion.Task; _ = DisposeCoreAsync(completion); } @@ -1222,7 +1222,6 @@ public ValueTask DisposeAsync() private async Task DisposeCoreAsync(TaskCompletionSource completion) { - _disposed = 1; try { try