diff --git a/docs/site/src/content/docs/grains/journaling/configuration.md b/docs/site/src/content/docs/grains/journaling/configuration.md index 1a857ae7fb8..70c53059e5c 100644 --- a/docs/site/src/content/docs/grains/journaling/configuration.md +++ b/docs/site/src/content/docs/grains/journaling/configuration.md @@ -178,4 +178,25 @@ The default minimum is seven days. Removal is persisted by a compaction after th registers the core services, formats, durable-state factories, and lifecycle integration. Register storage through a provider-specific method or the generic method. Runtime tests and disposable development hosts can use with a provider name. Its contents live in process memory, so use persistent emulator storage to validate restart recovery and provider migration. + controls the retained append +history. The defaults request a snapshot after **100 successful appends or 1 MiB +(1,048,576 bytes)** of appended data, whichever limit is reached first: + +:::code language="csharp" source="./snippets/journaling/JournalingConfiguration.cs" id="volatile_journal_thresholds"::: + +The state manager observes that request before its next write. That write replaces +the history with a snapshot of current state; successful replacement resets both +counters. Snapshot bytes are excluded from the appended-byte count. Deletion and +recreation begin with empty history. Each limit must be positive; a single append +can exceed the byte threshold. + +Use the overload accepting a provider name and options delegate to configure +independent named thresholds. Providers capture the configured limits when +constructed. Counters belong to each journal and are shared by +handles for the same journal within one provider. Larger limits amortize +full-state snapshot copies over more updates; smaller limits reduce retained +append history and replay work. Size both limits alongside the current snapshot. +Retained memory includes the latest snapshot, appended encoded bytes up to and +including the crossing append, and the storage provider's bookkeeping. + Use the same durable provider category in staging that production uses so recovery, compaction, concurrency, and backup procedures receive realistic validation. diff --git a/docs/site/src/content/docs/grains/journaling/snippets/journaling/JournalingConfiguration.cs b/docs/site/src/content/docs/grains/journaling/snippets/journaling/JournalingConfiguration.cs index 3e67c4ac483..93691c9a353 100644 --- a/docs/site/src/content/docs/grains/journaling/snippets/journaling/JournalingConfiguration.cs +++ b/docs/site/src/content/docs/grains/journaling/snippets/journaling/JournalingConfiguration.cs @@ -17,6 +17,18 @@ internal partial class JournalJsonContext : JsonSerializerContext; public static class JournalingConfiguration { + public static ISiloBuilder ConfigureVolatile(ISiloBuilder siloBuilder) + { + // + siloBuilder.AddVolatileJournalStorage(options => + { + options.MaxAppendsBeforeSnapshot = 100; + options.MaxBytesBeforeSnapshot = 1024 * 1024; + }); + // + return siloBuilder; + } + public static IHost ConfigureJson() { // diff --git a/src/Orleans.Journaling/JournalingHostingExtensions.cs b/src/Orleans.Journaling/JournalingHostingExtensions.cs index 140dc2762a4..2e3e5eec659 100644 --- a/src/Orleans.Journaling/JournalingHostingExtensions.cs +++ b/src/Orleans.Journaling/JournalingHostingExtensions.cs @@ -257,7 +257,44 @@ public static ISiloBuilder AddVolatileJournalStorage(this ISiloBuilder builder) /// The provider name. /// The silo builder. public static ISiloBuilder AddVolatileJournalStorage(this ISiloBuilder builder, string name) - => builder.AddJournalStorage(name, static services => ActivatorUtilities.CreateInstance(services)); + => builder.AddVolatileJournalStorage(name, configureOptions: null); + + /// + /// Registers the default volatile journal provider with configurable snapshot thresholds. + /// + /// The silo builder. + /// Configures the provider's snapshot thresholds. + /// The silo builder. + public static ISiloBuilder AddVolatileJournalStorage(this ISiloBuilder builder, Action configureOptions) + => builder.AddVolatileJournalStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions); + + /// + /// Registers a named volatile journal provider with configurable snapshot thresholds. + /// + /// The silo builder. + /// The provider name. + /// Optionally configures the provider's snapshot thresholds. + /// The silo builder. + public static ISiloBuilder AddVolatileJournalStorage( + this ISiloBuilder builder, string name, Action? configureOptions) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var options = builder.Services.AddJournalStorageOptions(name) + .Validate(static configuration => configuration.MaxAppendsBeforeSnapshot > 0, + "MaxAppendsBeforeSnapshot must be positive.") + .Validate(static configuration => configuration.MaxBytesBeforeSnapshot > 0, + "MaxBytesBeforeSnapshot must be positive."); + if (configureOptions is not null) + { + options.Configure(configureOptions); + } + + return builder.AddJournalStorage(name, services => new VolatileJournalStorageProvider( + services.GetRequiredService>(), + services.GetJournalStorageOptions(name), + services.GetService())); + } private sealed record JournalStorageRegistration(string Name, Type ProviderType); diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index bd3c40be155..2f5046a30be 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -9,6 +9,24 @@ Application code uses `IDurableStateManager` to manage the grain's durable state `IStateMachine` defines the low-level replay and snapshot protocol for state implementations. `IJournaledStateManager` is an independent contract for journal ownership and lifetime operations. +## Volatile storage snapshot thresholds + +`AddVolatileJournalStorage(options => ...)` configures +`VolatileJournalStorageOptions.MaxAppendsBeforeSnapshot` (default 100) and +`MaxBytesBeforeSnapshot` (default 1,048,576 bytes). The named-provider overload +configures each provider independently. Both values must be positive. +Providers capture the configured limits when constructed. + +Reaching either threshold requests a snapshot before the next journal write. +Successful replacement resets both counters; the snapshot itself is excluded +from append history. Deletion and recreation reset the history too. A single +append can exceed the byte threshold. Larger limits amortize snapshot copies +over more updates; smaller limits reduce retained history and replay work. +Counters belong to each journal and are shared by handles for the same journal +within one provider. +Retained memory includes the latest snapshot, appended encoded bytes up to and +including the crossing append, and provider bookkeeping. + ## Getting Started To use this package, install it via NuGet: diff --git a/src/Orleans.Journaling/VolatileJournalStorage.cs b/src/Orleans.Journaling/VolatileJournalStorage.cs index c5d2cc98162..a85e56b2029 100644 --- a/src/Orleans.Journaling/VolatileJournalStorage.cs +++ b/src/Orleans.Journaling/VolatileJournalStorage.cs @@ -18,6 +18,7 @@ public sealed class VolatileJournalStorageProvider : IJournalStorageProvider, IJ private readonly SortedSet _storageKeys = new(StringComparer.Ordinal); private readonly object _catalogLock = new(); private readonly JournalStorageTelemetry _telemetry; + private readonly VolatileJournalStorageOptions _storageOptions = new(); /// /// Initializes a new instance of the class using the default journal format. @@ -51,6 +52,31 @@ public VolatileJournalStorageProvider(IOptions opt _telemetry = new JournalStorageTelemetry(instruments); } + /// + /// Initializes an in-memory provider with configurable snapshot thresholds. + /// + /// The journaled state manager options. + /// The provider's snapshot thresholds. + /// The optional Orleans runtime metrics meter. + public VolatileJournalStorageProvider( + IOptions options, + IOptions storageOptions, + OrleansInstruments? instruments) : this(options) + { + ArgumentNullException.ThrowIfNull(storageOptions); + var configuration = storageOptions.Value; + configuration.Validate(); + _storageOptions = new() + { + MaxAppendsBeforeSnapshot = configuration.MaxAppendsBeforeSnapshot, + MaxBytesBeforeSnapshot = configuration.MaxBytesBeforeSnapshot + }; + if (instruments is not null) + { + _telemetry = new JournalStorageTelemetry(instruments); + } + } + /// public IJournalStorage CreateStorage(JournalId journalId) { @@ -73,7 +99,7 @@ public IJournalStorage CreateStorage(JournalId journalId) } } - return new VolatileJournalStorage(store, journalFormatKey); + return new VolatileJournalStorage(store, journalFormatKey, _storageOptions); } /// @@ -147,6 +173,8 @@ private string GetJournalFormatKey() public sealed class VolatileJournalStorage : IJournalStorage { private readonly Store _store; + private readonly int _maxAppendsBeforeSnapshot; + private readonly long _maxBytesBeforeSnapshot; private string? _configuredJournalFormatKey; /// @@ -164,10 +192,24 @@ public VolatileJournalStorage(string? journalFormatKey) : this(new Store(CreateV { } - internal VolatileJournalStorage(Store store, string? journalFormatKey) + /// + /// Initializes an isolated in-memory journal with configurable snapshot thresholds. + /// + /// The journal format key to stamp on writes. + /// The snapshot thresholds. + public VolatileJournalStorage(string? journalFormatKey, VolatileJournalStorageOptions options) + : this(new Store(CreateVolatileStorageId()), journalFormatKey, options ?? throw new ArgumentNullException(nameof(options))) + { + } + + internal VolatileJournalStorage(Store store, string? journalFormatKey, VolatileJournalStorageOptions? options = null) { ArgumentNullException.ThrowIfNull(store); + options ??= new(); + options.Validate(); _store = store; + _maxAppendsBeforeSnapshot = options.MaxAppendsBeforeSnapshot; + _maxBytesBeforeSnapshot = options.MaxBytesBeforeSnapshot; SetConfiguredJournalFormatKey(journalFormatKey); } @@ -178,7 +220,7 @@ public bool IsCompactionRequested { lock (_store.SyncRoot) { - return _store.Segments.Count > 10; + return _store.AppendCount >= _maxAppendsBeforeSnapshot || _store.AppendedBytes >= _maxBytesBeforeSnapshot; } } } @@ -295,6 +337,8 @@ public ValueTask AppendAsync(ReadOnlySequence segment, CancellationToken c _store.Exists = true; _store.StoredJournalFormatKey = _configuredJournalFormatKey; _store.Segments.Add(segment.ToArray()); + _store.AppendedBytes += segment.Length; + _store.AppendCount++; _store.RefreshETag(); } @@ -305,12 +349,15 @@ public ValueTask AppendAsync(ReadOnlySequence segment, CancellationToken c public ValueTask ReplaceAsync(ReadOnlySequence snapshot, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + var data = snapshot.ToArray(); lock (_store.SyncRoot) { _store.Exists = true; _store.StoredJournalFormatKey = _configuredJournalFormatKey; _store.Segments.Clear(); - _store.Segments.Add(snapshot.ToArray()); + _store.Segments.Add(data); + _store.AppendedBytes = 0; + _store.AppendCount = 0; _store.RefreshETag(); } @@ -337,6 +384,11 @@ internal sealed class Store(string storageId) public List Segments { get; } = []; + public long AppendCount { get; set; } + + // List count and each stored array length fit in int, so their total fits in long. + public long AppendedBytes { get; set; } + public Dictionary Properties { get; } = new(StringComparer.Ordinal); public string? StoredJournalFormatKey { get; set; } @@ -351,6 +403,8 @@ public void Create(IReadOnlyDictionary? properties) { Exists = true; Segments.Clear(); + AppendCount = 0; + AppendedBytes = 0; Properties.Clear(); StoredJournalFormatKey = null; if (properties is not null) @@ -368,6 +422,8 @@ public void Delete() { Exists = false; Segments.Clear(); + AppendCount = 0; + AppendedBytes = 0; Properties.Clear(); StoredJournalFormatKey = null; ETag = null; diff --git a/src/Orleans.Journaling/VolatileJournalStorageOptions.cs b/src/Orleans.Journaling/VolatileJournalStorageOptions.cs new file mode 100644 index 00000000000..d60cd82415e --- /dev/null +++ b/src/Orleans.Journaling/VolatileJournalStorageOptions.cs @@ -0,0 +1,34 @@ +namespace Orleans.Journaling; + +/// +/// Configures the journal history retained by volatile storage between snapshots. +/// +/// +/// Providers and isolated storage instances capture these values during construction. +/// Each journal counts successful storage append calls, including empty appends. +/// +public sealed class VolatileJournalStorageOptions +{ + /// + /// Gets or sets the number of successful appends after which storage requests a snapshot. + /// The default is 100. The value must be positive. + /// + public int MaxAppendsBeforeSnapshot { get; set; } = 100; + + /// + /// Gets or sets the appended byte count after which storage requests a snapshot. + /// The default is 1 MiB (1,048,576 bytes). The value must be positive. + /// + /// + /// Storage requests a snapshot when either limit is reached. The appended byte count measures + /// encoded journal bytes, with snapshot bytes excluded. A successful snapshot or deletion resets both counters. + /// The state manager checks the request before its next write, so an individual append can exceed this limit. + /// + public long MaxBytesBeforeSnapshot { get; set; } = 1024 * 1024; + + internal void Validate() + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(MaxAppendsBeforeSnapshot); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(MaxBytesBeforeSnapshot); + } +} diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 3fba3cb84e4..396a1692cad 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -492,6 +492,10 @@ public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddSta public static Microsoft.Extensions.DependencyInjection.IServiceCollection AddStateMachine(this Microsoft.Extensions.DependencyInjection.IServiceCollection services) where TState : class where TImplementation : class, TState, IStateMachine { throw null; } + public static Hosting.ISiloBuilder AddVolatileJournalStorage(this Hosting.ISiloBuilder builder, System.Action configureOptions) { throw null; } + + public static Hosting.ISiloBuilder AddVolatileJournalStorage(this Hosting.ISiloBuilder builder, string name, System.Action? configureOptions) { throw null; } + public static Hosting.ISiloBuilder AddVolatileJournalStorage(this Hosting.ISiloBuilder builder, string name) { throw null; } public static Hosting.ISiloBuilder AddVolatileJournalStorage(this Hosting.ISiloBuilder builder) { throw null; } @@ -593,6 +597,8 @@ public sealed partial class VolatileJournalStorage : IJournalStorage { public VolatileJournalStorage() { } + public VolatileJournalStorage(string? journalFormatKey, VolatileJournalStorageOptions options) { } + public VolatileJournalStorage(string? journalFormatKey) { } public bool IsCompactionRequested { get { throw null; } } @@ -612,10 +618,19 @@ public VolatileJournalStorage(string? journalFormatKey) { } public System.Threading.Tasks.ValueTask UpdateMetadataAsync(System.Collections.Generic.IReadOnlyDictionary? set = null, System.Collections.Generic.IEnumerable? remove = null, string? expectedETag = null, System.Threading.CancellationToken cancellationToken = default) { throw null; } } + public sealed partial class VolatileJournalStorageOptions + { + public int MaxAppendsBeforeSnapshot { get { throw null; } set { } } + + public long MaxBytesBeforeSnapshot { get { throw null; } set { } } + } + public sealed partial class VolatileJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog { public VolatileJournalStorageProvider() { } + public VolatileJournalStorageProvider(Microsoft.Extensions.Options.IOptions options, Microsoft.Extensions.Options.IOptions storageOptions, Runtime.OrleansInstruments? instruments) { } + public VolatileJournalStorageProvider(Microsoft.Extensions.Options.IOptions options, Runtime.OrleansInstruments instruments) { } public VolatileJournalStorageProvider(Microsoft.Extensions.Options.IOptions options) { } diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 2e211b87e20..e9822dd9557 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -26,6 +26,188 @@ public sealed class KeyedJournalingRegistrationTests : JournalingTestBase { private const string CustomFormatKey = "custom-test-format"; + [Fact] + public async Task VolatileProviders_UseIndependentNamedSnapshotThresholds() + { + var builder = CreateNamedProviderBuilder(); + builder.AddVolatileJournalStorage(options => + { + options.MaxAppendsBeforeSnapshot = 2; + options.MaxBytesBeforeSnapshot = 100; + }); + builder.AddVolatileJournalStorage("small-bytes", options => + { + options.MaxAppendsBeforeSnapshot = 100; + options.MaxBytesBeforeSnapshot = 3; + }); + builder.AddVolatileJournalStorage("large", options => + { + options.MaxAppendsBeforeSnapshot = int.MaxValue; + options.MaxBytesBeforeSnapshot = long.MaxValue; + }); + await using var services = builder.Services.BuildServiceProvider(); + var defaults = services.GetRequiredService().CreateStorage(new("threshold")); + var named = services.GetRequiredKeyedService("small-bytes").CreateStorage(new("threshold")); + var large = services.GetRequiredKeyedService("large").CreateStorage(new("threshold")); + var token = TestContext.Current.CancellationToken; + await defaults.AppendAsync(new ReadOnlySequence([1]), token); + await named.AppendAsync(new ReadOnlySequence([1, 2]), token); + await large.AppendAsync(new ReadOnlySequence([1, 2, 3]), token); + Assert.False(defaults.IsCompactionRequested); + Assert.False(named.IsCompactionRequested); + Assert.False(large.IsCompactionRequested); + await defaults.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(defaults.IsCompactionRequested); + Assert.False(named.IsCompactionRequested); + await named.AppendAsync(new ReadOnlySequence([3]), token); + Assert.True(named.IsCompactionRequested); + await defaults.ReplaceAsync(new ReadOnlySequence([4]), token); + Assert.False(defaults.IsCompactionRequested); + Assert.True(named.IsCompactionRequested); + Assert.False(large.IsCompactionRequested); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(-1, 1)] + [InlineData(1, 0)] + [InlineData(1, -1)] + public void VolatileProviders_InvalidSnapshotThresholdsFailDefaultAndNamedResolution(int appends, long bytes) + { + var builder = CreateNamedProviderBuilder(); + builder.AddVolatileJournalStorage(Configure); + builder.AddVolatileJournalStorage("invalid", Configure); + using var services = builder.Services.BuildServiceProvider(); + foreach (var name in new[] { ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, "invalid" }) + { + var exception = Assert.Throws(() => + services.GetRequiredKeyedService(name)); + Assert.Equal(typeof(VolatileJournalStorageOptions), exception.OptionsType); + Assert.Equal(name == ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME ? Options.DefaultName : name, exception.OptionsName); + Assert.Equal([appends <= 0 ? "MaxAppendsBeforeSnapshot must be positive." : "MaxBytesBeforeSnapshot must be positive."], + exception.Failures); + } + + void Configure(VolatileJournalStorageOptions options) + { + options.MaxAppendsBeforeSnapshot = appends; + options.MaxBytesBeforeSnapshot = bytes; + } + } + + [Theory] + [InlineData(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME)] + [InlineData("configured")] + public async Task VolatileProviders_SeparateOptionsPipelineAppliesOnceAndIsCapturedAtConstruction(string name) + { + var builder = CreateNamedProviderBuilder(); + var optionsName = name == ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME ? Options.DefaultName : name; + builder.Services.Configure(optionsName, options => + { + options.MaxAppendsBeforeSnapshot = 2; + options.MaxBytesBeforeSnapshot = long.MaxValue; + }); + builder.AddVolatileJournalStorage(name, configureOptions: null); + builder.Services.ConfigureAll(options => options.MaxAppendsBeforeSnapshot++); + builder.Services.PostConfigure(optionsName, options => options.MaxAppendsBeforeSnapshot++); + await using var services = builder.Services.BuildServiceProvider(); + var configured = name == ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME + ? services.GetRequiredService>().Value + : services.GetRequiredService>().Get(optionsName); + Assert.Equal(4, configured.MaxAppendsBeforeSnapshot); + Assert.Equal(long.MaxValue, configured.MaxBytesBeforeSnapshot); + var provider = services.GetRequiredKeyedService(name); + var storage = provider.CreateStorage(new("options-pipeline")); + if (name == ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME) + { + Assert.Same(provider, services.GetRequiredService()); + } + + configured.MaxAppendsBeforeSnapshot = 1; + var token = TestContext.Current.CancellationToken; + for (var index = 0; index < 3; index++) + { + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.False(storage.IsCompactionRequested); + } + + await storage.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(storage.IsCompactionRequested); + var second = provider.CreateStorage(new("options-pipeline")); + Assert.True(second.IsCompactionRequested); + await second.ReplaceAsync(new ReadOnlySequence([3]), token); + Assert.False(storage.IsCompactionRequested); + await second.AppendAsync(new ReadOnlySequence([4]), token); + Assert.False(storage.IsCompactionRequested); + } + + [Theory] + [InlineData(OrleansBinaryJournalFormat.JournalFormatKey)] + [InlineData(JsonLinesJournalFormat.JournalFormatKey)] + public async Task VolatileProviders_ThresholdSnapshotIsAcknowledgedAndReplaysCurrentFormats(string formatKey) + { + var builder = CreateNamedProviderBuilder(); + builder.AddVolatileJournalStorage(options => + { + options.MaxAppendsBeforeSnapshot = 2; + options.MaxBytesBeforeSnapshot = long.MaxValue; + }); + builder.Services.Configure(options => options.JournalFormatKey = formatKey); + await using var services = builder.Services.BuildServiceProvider(); + var journalId = JournalId.FromGrainId(GrainId.Create("test-grain", "snapshot-recovery")); + var storage = Assert.IsType( + services.GetRequiredService().CreateStorage(journalId)); + var factory = services.GetRequiredService(); + var codec = services.GetRequiredKeyedService>(formatKey); + var token = TestContext.Current.CancellationToken; + await using (var manager = factory.CreateStandalone(journalId)) + { + var value = new DurableValue("value", manager, codec); + await manager.InitializeAsync(token); + value.Value = 1; + await manager.WriteStateAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Single(storage.Segments); + value.Value = 2; + await manager.WriteStateAsync(token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(2, storage.Segments.Count); + + value.Value = 3; + await manager.WriteStateAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Single(storage.Segments); + Assert.Equal(formatKey, (await storage.GetMetadataAsync(token))!.FormatKey); + value.Value = 4; + await manager.WriteStateAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Equal(2, storage.Segments.Count); + value.Value = 5; + await manager.WriteStateAsync(token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(3, storage.Segments.Count); + } + + await using (var manager = factory.CreateStandalone(journalId)) + { + var value = new DurableValue("value", manager, codec); + await manager.InitializeAsync(token); + Assert.Equal(5, value.Value); + Assert.True(storage.IsCompactionRequested); + value.Value = 6; + await manager.WriteStateAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Single(storage.Segments); + } + + await using (var manager = factory.CreateStandalone(journalId)) + { + var value = new DurableValue("value", manager, codec); + await manager.InitializeAsync(token); + Assert.Equal(6, value.Value); + } + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 2c9abd55015..fc1738a6c27 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -1367,7 +1367,10 @@ public async Task StateManager_AutoRetiringStates() var period = ManagerOptions.RetirementGracePeriod; var timeProvider = new FakeTimeProvider(DateTime.UtcNow); - var storage = CreateStorage(); + var storage = new VolatileJournalStorage(null, new() + { + MaxAppendsBeforeSnapshot = 10 + }); // -------------- STEP 1 -------------- diff --git a/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs b/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs index 44dd651fc09..ad21c5faf90 100644 --- a/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs +++ b/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs @@ -1,4 +1,6 @@ using System.Buffers; +using Microsoft.Extensions.Options; +using Orleans.Runtime; using Xunit; namespace Orleans.Journaling.Tests; @@ -8,6 +10,324 @@ namespace Orleans.Journaling.Tests; [TestCategory("BVT")] public sealed class VolatileJournalStorageProviderTests { + [Fact] + public async Task Compaction_ByteThresholdExcludesSnapshotAndResetsAfterReplacement() + { + var storage = new VolatileJournalStorage(null, new() + { + MaxAppendsBeforeSnapshot = int.MaxValue, + MaxBytesBeforeSnapshot = 1000 + }); + var token = TestContext.Current.CancellationToken; + await storage.ReplaceAsync(new ReadOnlySequence(new byte[1000]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence(new byte[999]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(2000, storage.Segments.Sum(static segment => segment.Length)); + + await storage.ReplaceAsync(new ReadOnlySequence(new byte[2000]), token); + Assert.False(storage.IsCompactionRequested); + Assert.Equal(2000, Assert.Single(storage.Segments).Length); + await storage.AppendAsync(new ReadOnlySequence(new byte[999]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(storage.IsCompactionRequested); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Compaction_DefaultAppendThresholdAndResetExcludeSnapshot(bool recreate) + { + var storage = new VolatileJournalStorage(); + var token = TestContext.Current.CancellationToken; + await storage.ReplaceAsync(new ReadOnlySequence(new byte[1000]), token); + Assert.False(storage.IsCompactionRequested); + for (var index = 0; index < 99; index++) + { + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.False(storage.IsCompactionRequested); + } + + await storage.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(101, storage.Segments.Count); + await storage.ReplaceAsync(new ReadOnlySequence([3]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([4]), token); + await storage.DeleteAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Empty(storage.Segments); + if (recreate) + { + Assert.True(await storage.CreateIfNotExistsAsync(cancellationToken: token)); + } + + for (var index = 0; index < 99; index++) + { + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.False(storage.IsCompactionRequested); + } + + await storage.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(100, storage.Segments.Count); + Assert.Equal([2], storage.Segments[^1]); + } + + [Fact] + public async Task Compaction_DefaultByteThresholdRequestsAtOneMiBIndependentlyOfAppendCount() + { + var storage = new VolatileJournalStorage(); + var token = TestContext.Current.CancellationToken; + await storage.AppendAsync(new ReadOnlySequence(new byte[1024 * 1024 - 1]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(2, storage.Segments.Count); + Assert.Equal(1024 * 1024, storage.Segments.Sum(static segment => segment.Length)); + } + + [Theory] + [InlineData(2, 100, false)] + [InlineData(2, 100, true)] + [InlineData(100, 2, false)] + [InlineData(100, 2, true)] + public async Task Compaction_DeletionResetsBothCountersAndExistingCreationPreservesThem(int appends, long bytes, bool recreate) + { + var storage = new VolatileJournalStorage(null, new() + { + MaxAppendsBeforeSnapshot = appends, + MaxBytesBeforeSnapshot = bytes + }); + var token = TestContext.Current.CancellationToken; + await storage.AppendAsync(new ReadOnlySequence([1]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(storage.IsCompactionRequested); + Assert.False(await storage.CreateIfNotExistsAsync(cancellationToken: token)); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(2, storage.Segments.Count); + + await storage.DeleteAsync(token); + Assert.False(storage.IsCompactionRequested); + Assert.Empty(storage.Segments); + if (recreate) + { + Assert.True(await storage.CreateIfNotExistsAsync(cancellationToken: token)); + } + + await storage.AppendAsync(new ReadOnlySequence([3]), token); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(new ReadOnlySequence([4]), token); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(2, storage.Segments.Count); + } + + [Theory] + [InlineData(3, 100)] + [InlineData(100, 3)] + public async Task Compaction_ConfiguredCountersAreSharedAcrossHandlesAndIsolatedAcrossGrains(int appends, long bytes) + { + var provider = new VolatileJournalStorageProvider( + Options.Create(new JournaledStateManagerOptions()), + Options.Create(new VolatileJournalStorageOptions + { + MaxAppendsBeforeSnapshot = appends, + MaxBytesBeforeSnapshot = bytes + }), null); + var firstId = JournalId.FromGrainId(GrainId.Create("test-grain", "first")); + var first = provider.CreateStorage(firstId); + var second = provider.CreateStorage(firstId); + var other = provider.CreateStorage(JournalId.FromGrainId(GrainId.Create("test-grain", "other"))); + var token = TestContext.Current.CancellationToken; + for (var index = 0; index < 2; index++) + { + await first.AppendAsync(new ReadOnlySequence([1]), token); + Assert.False(second.IsCompactionRequested); + } + + await second.AppendAsync(new ReadOnlySequence([2]), token); + Assert.True(first.IsCompactionRequested); + Assert.False(other.IsCompactionRequested); + Assert.Null(await other.GetMetadataAsync(token)); + await other.AppendAsync(new ReadOnlySequence([3]), token); + await second.ReplaceAsync(new ReadOnlySequence(new byte[100]), token); + Assert.False(first.IsCompactionRequested); + Assert.False(other.IsCompactionRequested); + await first.AppendAsync(new ReadOnlySequence([4]), token); + Assert.False(second.IsCompactionRequested); + await other.AppendAsync(new ReadOnlySequence([5]), token); + Assert.False(other.IsCompactionRequested); + await other.AppendAsync(new ReadOnlySequence([6]), token); + Assert.True(other.IsCompactionRequested); + Assert.False(first.IsCompactionRequested); + } + + [Theory] + [InlineData(3, 3, true)] + [InlineData(4, 3, true)] + [InlineData(3, long.MaxValue, false)] + public async Task Compaction_MultiSegmentAppendCountsOnceAndCopiesBorrowedInput(int length, long maxBytes, bool thresholdReached) + { + var storage = new VolatileJournalStorage(null, new() + { + MaxAppendsBeforeSnapshot = 2, + MaxBytesBeforeSnapshot = maxBytes + }); + byte[] prefix = [1]; + var suffix = Enumerable.Range(2, length - 1).Select(static value => (byte)value).ToArray(); + var first = new BufferSegment(prefix); + var last = first.Append(suffix); + var segment = new ReadOnlySequence(first, 0, last, suffix.Length); + await storage.AppendAsync(segment, TestContext.Current.CancellationToken); + Assert.Equal(thresholdReached, storage.IsCompactionRequested); + prefix[0] = 99; + suffix[0] = 99; + Assert.Equal(Enumerable.Range(1, length).Select(static value => (byte)value), Assert.Single(storage.Segments)); + + byte[] snapshot = [5, 6, 7, 8]; + await storage.ReplaceAsync(new ReadOnlySequence(snapshot), TestContext.Current.CancellationToken); + snapshot[0] = 99; + Assert.Equal([5, 6, 7, 8], Assert.Single(storage.Segments)); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(ReadOnlySequence.Empty, TestContext.Current.CancellationToken); + Assert.False(storage.IsCompactionRequested); + await storage.AppendAsync(ReadOnlySequence.Empty, TestContext.Current.CancellationToken); + Assert.True(storage.IsCompactionRequested); + Assert.Equal(3, storage.Segments.Count); + } + + [Theory] + [InlineData(8, long.MaxValue)] + [InlineData(int.MaxValue, 8)] + public async Task Compaction_ConcurrentHandlesPreserveEveryAppendAndResetTogether(int appends, long bytes) + { + var provider = new VolatileJournalStorageProvider( + Options.Create(new JournaledStateManagerOptions()), + Options.Create(new VolatileJournalStorageOptions + { + MaxAppendsBeforeSnapshot = appends, + MaxBytesBeforeSnapshot = bytes + }), null); + var id = new JournalId("concurrent-thresholds"); + var handles = Enumerable.Range(0, 8).Select(_ => provider.CreateStorage(id)).ToArray(); + var other = provider.CreateStorage(new("isolated-thresholds")); + var storage = Assert.IsType(handles[0]); + var token = TestContext.Current.CancellationToken; + await AppendConcurrently(8); + Assert.Equal(Enumerable.Range(0, 8).Select(static index => (byte)index), + storage.Segments.SelectMany(static segment => segment).Order()); + Assert.All(handles, static handle => Assert.True(handle.IsCompactionRequested)); + Assert.False(other.IsCompactionRequested); + + await handles[^1].ReplaceAsync(new ReadOnlySequence([99]), token); + Assert.Equal([99], Assert.Single(storage.Segments)); + Assert.All(handles, static handle => Assert.False(handle.IsCompactionRequested)); + await AppendConcurrently(7); + Assert.Equal(8, storage.Segments.Count); + Assert.Equal(Enumerable.Range(0, 7).Select(static index => (byte)index), + storage.Segments.Skip(1).SelectMany(static segment => segment).Order()); + Assert.All(handles, static handle => Assert.False(handle.IsCompactionRequested)); + await handles[^1].AppendAsync(new ReadOnlySequence([7]), token); + Assert.All(handles, static handle => Assert.True(handle.IsCompactionRequested)); + Assert.False(other.IsCompactionRequested); + + Task AppendConcurrently(int count) => Task.WhenAll( + Enumerable.Range(0, count).Select(index => Task.Run( + async () => await handles[index].AppendAsync(new ReadOnlySequence([(byte)index]), token), token))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Compaction_CanceledOperationsPreserveHistoryAndCounters(bool thresholdReached) + { + var storage = new VolatileJournalStorage(null, new() + { + MaxAppendsBeforeSnapshot = 2, + MaxBytesBeforeSnapshot = 3 + }); + var token = TestContext.Current.CancellationToken; + await storage.AppendAsync(new ReadOnlySequence([1, 2]), token); + if (thresholdReached) + { + await storage.AppendAsync(new ReadOnlySequence([3]), token); + } + + var original = storage.Segments.ToArray(); + var metadata = await storage.GetMetadataAsync(token); + var canceled = new CancellationToken(canceled: true); + await Assert.ThrowsAsync(() => storage.AppendAsync(new ReadOnlySequence([4]), canceled).AsTask()); + await Assert.ThrowsAsync(() => storage.ReplaceAsync(new ReadOnlySequence([5]), canceled).AsTask()); + await Assert.ThrowsAsync(() => storage.DeleteAsync(canceled).AsTask()); + Assert.Equal(thresholdReached, storage.IsCompactionRequested); + Assert.Equal(original, storage.Segments); + Assert.Equal(metadata!.ETag, (await storage.GetMetadataAsync(token))!.ETag); + await storage.AppendAsync(new ReadOnlySequence([6]), token); + Assert.True(storage.IsCompactionRequested); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(-1, 1)] + [InlineData(int.MinValue, 1)] + [InlineData(1, 0)] + [InlineData(1, -1)] + [InlineData(1, long.MinValue)] + public void Compaction_InvalidLimitsFailAtConstruction(int appends, long bytes) + { + var options = new VolatileJournalStorageOptions { MaxAppendsBeforeSnapshot = appends, MaxBytesBeforeSnapshot = bytes }; + var parameter = appends <= 0 ? nameof(options.MaxAppendsBeforeSnapshot) : nameof(options.MaxBytesBeforeSnapshot); + Assert.Throws(parameter, () => new VolatileJournalStorage(null, options)); + Assert.Throws(parameter, () => new VolatileJournalStorageProvider( + Options.Create(new JournaledStateManagerOptions()), Options.Create(options), null)); + } + + [Fact] + public void Compaction_NullOptionsFailAtConstruction() + { + Assert.Throws("options", () => new VolatileJournalStorage(journalFormatKey: null, options: null!)); + Assert.Throws("options", () => new VolatileJournalStorageProvider( + null!, Options.Create(new VolatileJournalStorageOptions()), null)); + Assert.Throws("storageOptions", () => new VolatileJournalStorageProvider( + Options.Create(new JournaledStateManagerOptions()), null!, null)); + } + + [Fact] + public async Task Compaction_ConstructionSnapshotsOptionsAndAcceptsMaximumLimits() + { + var options = new VolatileJournalStorageOptions + { + MaxAppendsBeforeSnapshot = int.MaxValue, + MaxBytesBeforeSnapshot = long.MaxValue + }; + var storage = new VolatileJournalStorage(null, options); + var provider = new VolatileJournalStorageProvider( + Options.Create(new JournaledStateManagerOptions()), Options.Create(options), null); + options.MaxAppendsBeforeSnapshot = 1; + options.MaxBytesBeforeSnapshot = 1; + var fromProvider = provider.CreateStorage(new("maximum-limits")); + await storage.AppendAsync(new ReadOnlySequence([1]), TestContext.Current.CancellationToken); + await fromProvider.AppendAsync(new ReadOnlySequence([2]), TestContext.Current.CancellationToken); + Assert.False(storage.IsCompactionRequested); + Assert.False(fromProvider.IsCompactionRequested); + } + + private sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(ReadOnlyMemory memory) => Memory = memory; + + public BufferSegment Append(ReadOnlyMemory memory) + { + var next = new BufferSegment(memory) { RunningIndex = RunningIndex + Memory.Length }; + Next = next; + return next; + } + } + [Theory] [InlineData(false)] [InlineData(true)]