From 48ac114be30bed945ab33227f8af3e37384d2bb5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 22:42:48 -0700 Subject: [PATCH 1/9] feat(journaling): optimize catalog queries and storage layouts --- .../docs/grains/journaling/azure-storage.md | 14 +- src/AWS/Orleans.Journaling.S3/README.md | 36 +- .../S3JournalStorageOptions.cs | 68 +- .../S3JournalStorageProvider.cs | 66 +- .../AzureBlobJournalStorage.cs | 2 +- .../AzureBlobJournalStorageLayout.cs | 25 + .../AzureBlobJournalStorageOptions.cs | 11 +- .../AzureBlobJournalStorageProvider.cs | 58 +- .../AzureTableJournalStorage.cs | 7 +- .../AzureTableJournalStorageOptions.cs | 43 +- .../AzureTableJournalStorageProvider.cs | 118 +- .../Orleans.Journaling.AzureStorage/README.md | 10 +- .../JournaledJobShardManager.cs | 4 +- .../IJournalStorageCatalog.cs | 19 +- src/Orleans.Journaling/JournalCatalogEntry.cs | 16 + src/Orleans.Journaling/JournalCatalogRange.cs | 113 ++ src/Orleans.Journaling/ListOptions.cs | 31 +- .../Orleans.Journaling.csproj | 3 + src/Orleans.Journaling/README.md | 58 +- .../VolatileJournalStorage.cs | 56 +- src/Redis/Orleans.Journaling.Redis/README.md | 24 +- .../RedisJournalStorage.cs | 41 +- .../RedisJournalStorageOptions.cs | 10 +- .../RedisJournalStorageProvider.cs | 39 +- .../Orleans.Journaling.S3.cs | 4 + .../Orleans.Journaling/Orleans.Journaling.cs | 42 +- .../RedisJournalStorageCatalogTests.cs | 487 +++++++- .../Journaling/RedisJournalStorageTests.cs | 8 +- .../JournaledJobShardManagerTests.cs | 2 +- .../AzureBlobJournalStorageTests.cs | 52 +- .../AzureTableJournalStorageOptionsTests.cs | 97 +- .../AzureTableJournalStorageProviderTests.cs | 582 ++++++++- .../AzureTableJournalStorageTests.cs | 61 +- .../JournalCatalogRangeTests.cs | 159 +++ .../JournalStorageCatalogTests.cs | 1105 +++++++++++++++-- .../S3JournalStorageTests.cs | 482 ++++++- .../VolatileJournalStorageProviderTests.cs | 129 +- 37 files changed, 3734 insertions(+), 348 deletions(-) create mode 100644 src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageLayout.cs create mode 100644 src/Orleans.Journaling/JournalCatalogEntry.cs create mode 100644 src/Orleans.Journaling/JournalCatalogRange.cs create mode 100644 test/Orleans.Journaling.Tests/JournalCatalogRangeTests.cs diff --git a/docs/site/src/content/docs/grains/journaling/azure-storage.md b/docs/site/src/content/docs/grains/journaling/azure-storage.md index edf2747b4df..88e556ee1c8 100644 --- a/docs/site/src/content/docs/grains/journaling/azure-storage.md +++ b/docs/site/src/content/docs/grains/journaling/azure-storage.md @@ -19,8 +19,8 @@ Configure /wal` by default. -- Immutable checkpoint blobs at `/chk.`. +- An append blob at `wal/` by default. +- Immutable checkpoint blobs at `checkpoints//`. - WAL metadata which identifies the current checkpoint, journal format, and optimistic-concurrency state. Recovery reads the published checkpoint followed by the WAL tail. A replacement uploads the new checkpoint and then atomically publishes it through WAL metadata. The provider performs best-effort cleanup of obsolete checkpoints after publication when is `true`, which is the default. @@ -29,7 +29,9 @@ Customize ` for each journal. The separate checkpoint namespace keeps checkpoints out of listing pages. Raw journal-id prefixes and ASCII lower bounds narrow the native listing, and an ASCII upper bound terminates ordered consumption after the boundary page. + +Catalog callers can request a metadata snapshot with each identity. Blob listings project the WAL's format, ETag, and caller-owned metadata in the listing response. The snapshot can replace a separate metadata read; conditional updates use its ETag to detect concurrent changes. ## Azure Table Storage @@ -48,7 +50,11 @@ A single append batch is limited to 2 MiB by the provider's entity group transac Compaction is requested at either (10,000 rows by default) or (32 MiB by default). -Customize when a different partition layout is required. The mapping must remain unique per journal and satisfy Azure Table partition-key constraints. +The default partition mapping accepts printable ASCII journal ids (`0x20` through `0x7E`) and encodes each byte as two uppercase hexadecimal digits. It preserves ordinal ordering and prefixes for indexed catalog queries and supports journal ids up to 512 characters. Storage creation validates the id before accessing Azure. This restriction is specific to the default Table mapping. + +Customize when a different partition layout is required. Custom mappings support other journal-id alphabets, remain unique per journal, and satisfy Azure Table partition-key constraints. Their catalog queries filter the canonical journal-id property and can require a table scan. + +Metadata-enabled catalog queries select the header's format and caller-owned metadata together with its ETag. Identity-only queries retain their smaller projection. Each returned metadata snapshot has the same meaning as a direct metadata read and can become stale after it is listed. ## Optimistic concurrency diff --git a/src/AWS/Orleans.Journaling.S3/README.md b/src/AWS/Orleans.Journaling.S3/README.md index c5da91585bd..bf5d0a78ecc 100644 --- a/src/AWS/Orleans.Journaling.S3/README.md +++ b/src/AWS/Orleans.Journaling.S3/README.md @@ -10,11 +10,43 @@ Buckets should be created ahead of time for AWS S3 Express One Zone. `CreateBuck Metadata updates rewrite the current WAL using a conditional single-object upload. Publish a checkpoint to compact the WAL before updating metadata when the replacement object would exceed S3's 5 GB (5,000,000,000 byte) single-upload limit. Checkpoint snapshots use the same upload limit. +## Object layout + +`GetObjectKey` maps a logical journal id to its base object key (the identity mapping by default). WAL and checkpoint objects use separate namespaces: + +- WAL: `wal/` +- Checkpoint: `checkpoints//` + +For example, journal `jobs/00001234` uses `wal/jobs/00001234` and checkpoint objects under `checkpoints/jobs/00001234/`. Checkpoint names are stored in the WAL metadata. Catalog requests always stay under `wal/`, so checkpoints do not consume listing pages. + +S3 Express directory buckets benefit from slash-delimited prefixes, but this layout does not introduce time partitions or a discovery horizon. Unordered listings scan the selected WAL directory and retain every matching overdue journal, however old. Applications can supply hierarchical base keys, provided their prefix and reverse mappings satisfy the catalog contract. + ## Catalog enumeration -`IJournalStorageCatalog.ListAsync` returns journal identities incrementally in S3 traversal order, including unordered directory-bucket listings. Set `ListOptions.Prefix` to select an exact journal id and its descendants. +`IJournalStorageCatalog.ListAsync` returns `JournalCatalogEntry` values incrementally in S3 traversal order. Each entry's `Id` is the journal identity. S3 entries always have null `Metadata`, including when `ListOptions.IncludeMetadata = true`: `ListObjectsV2` cannot project the complete journal format, ETag, and caller-owned properties together. Enumeration never adds separate per-journal metadata requests. Call `GetMetadataAsync` explicitly when metadata is needed. + +`ListOptions.Prefix` is a raw ordinal string prefix, including partial segments; use a trailing `/` to select only entries inside a namespace. `MinId` and `MaxId` provide inclusive ordinal bounds, unlimited by default. All constraints are snapshotted when enumeration begins and checked before yielding. Empty intersections issue no request. Consumers needing due order must sort selected ids using `StringComparer.Ordinal`. + +The provider handles `ListObjectsV2` continuations internally and requests up to 1000 objects per page. `UseOrderedListing` defaults to `false`, matching S3 Express directory buckets. Every native prefix begins with `wal/`, including unprefixed catalog requests. Directory mode widens a raw native prefix to its nearest slash-terminated directory boundary, retaining at least `wal/`. Directory buckets are unordered and do not support `StartAfter`: the provider scans that selected namespace and filters both bounds without stopping at the first future id. + +Set `UseOrderedListing = true` only for general-purpose buckets or compatible services guaranteeing lexically ordered `ListObjectsV2` results and `StartAfter` support. With the default identity `GetObjectKey` mapping, ordered mode sends `wal/` plus the raw prefix, further narrowed by the common prefix of both bounds when possible. Since `StartAfter` is exclusive, an ASCII lower bound uses a strictly earlier marker: `wal/` plus the lower bound with its final character removed. This marker is sent only when it sorts after the native prefix, and the inclusive minimum is still checked locally. Subsequent requests use the returned opaque `ContinuationToken` alone to resume traversal, including after empty pages. The provider stops after `wal/` for a safe ASCII maximum. Non-ASCII bounds are filtered without unsafe native key seeks or cutoffs. There is no native upper-end parameter, so the crossing page is fetched but later pages are not. `UseS3ExpressAppend` does not imply a listing-order guarantee. + +Custom `GetObjectKey` mappings must also configure `GetObjectKeyPrefix` for explicitly prefixed catalog listings. Arbitrary object-key mapping functions cannot safely be applied to a journal prefix. The explicit prefix mapper must return a non-empty **base object-key prefix** covering every matching journal, including partial segments; do not include `wal/`. The provider prepends `wal/`, then directory mode widens the result to a slash boundary. Missing or empty prefix configuration throws before listing; unprefixed listing, including bounds-only queries, still works without the mapper. Custom mappings never use identity-based native lower or upper key bounds, even in ordered mode. + +```csharp +options.GetObjectKey = id => $"journals/{id.Value}"; +options.GetObjectKeyPrefix = prefix => $"journals/{prefix.Value}"; +options.TryParseJournalId = key => key.StartsWith("journals/", StringComparison.Ordinal) + ? new JournalId(key["journals/".Length..]) : null; +``` + +`TryParseJournalId` receives the base object key after the catalog strips `wal/`. Canonical-WAL validation still applies after parsing. Checkpoints are outside the selected namespace; aliases and unrelated objects under `wal/` still consume space in the native page before filtering. -The provider handles `ListObjectsV2` continuations internally, requests up to 1000 objects per page, and yields canonical WAL identities from that page before fetching more objects. The bucket traversal supports `GetObjectKey` and `TryParseJournalId` mappings; checkpoints, aliases, and unrelated objects consume space in the native page before filtering. +| Listing mode | Native prefix | Lower bound | Upper bound | +| --- | --- | --- | --- | +| Ordered, default identity mapping | `wal/` + raw prefix/common range prefix | Strictly earlier ASCII `StartAfter` marker when it narrows the prefix | Stop after crossing ASCII `wal/` | +| Directory/unordered | Nearest slash-terminated directory under `wal/` | Local filtering | Local filtering; entire selected namespace is traversed | +| Custom mapping | `wal/` + explicit mapped base prefix, widened in directory mode | Local filtering | Local filtering; no identity-order assumption | Client traversal memory is proportional to the current native page. An enumerator advance can cross multiple filtered or empty pages, and the storage service determines scan work, latency, and retries. Enumeration observes the live bucket; concurrent changes follow S3 listing semantics. Use subsequent enumerations to discover later changes and tolerate repeated identities during changes. diff --git a/src/AWS/Orleans.Journaling.S3/S3JournalStorageOptions.cs b/src/AWS/Orleans.Journaling.S3/S3JournalStorageOptions.cs index f291537defb..3ffa4bc64d6 100644 --- a/src/AWS/Orleans.Journaling.S3/S3JournalStorageOptions.cs +++ b/src/AWS/Orleans.Journaling.S3/S3JournalStorageOptions.cs @@ -10,6 +10,8 @@ namespace Orleans.Journaling; /// public sealed class S3JournalStorageOptions { + internal const string WalObjectKeyPrefix = "wal/"; + private IAmazonS3? _s3Client; /// @@ -20,11 +22,45 @@ public sealed class S3JournalStorageOptions /// /// Gets or sets the delegate used to generate the base object key for a journal. /// + /// + /// WAL objects use wal/<base-key> and checkpoint objects use + /// checkpoints/<base-key>/<snapshot-id>. The delegate must not add these namespaces. + /// public Func GetObjectKey { get; set; } = DefaultGetObjectKey; /// - /// Gets or sets the delegate used to parse journal ids from catalog object keys. + /// Gets or sets the delegate mapping a non-default catalog prefix to a base object-key prefix. + /// + /// + /// The result must be non-empty and include the base object key of every journal whose id + /// starts with the supplied raw ordinal prefix, including partial segments. Additional matches + /// are filtered by the catalog. When unset, the default identity + /// mapping uses the raw journal prefix. Custom object-key mappings must configure this delegate + /// to use prefixed listings. Unprefixed listings do not require this delegate. + /// The provider prepends wal/ to the result; the delegate must not add it. + /// When is false, the mapped prefix is widened to its nearest + /// slash-terminated directory boundary to support directory buckets, while retaining wal/. + /// + public Func? GetObjectKeyPrefix { get; set; } + + /// + /// Gets or sets a value indicating that the bucket supports lexically ordered + /// listings. Defaults to false. /// + /// + /// Enable this only for general-purpose buckets or compatible services which guarantee ordered + /// listings and support . + /// Directory buckets are unordered and must leave this disabled. With the default identity + /// object-key mapping, ordered listings can seek to an ASCII lower bound and stop beyond an + /// ASCII upper bound. Custom mappings do not use native key-range bounds. + /// This capability is independent of . + /// + public bool UseOrderedListing { get; set; } + + /// + /// Gets or sets the delegate used to parse journal ids from base object keys. + /// + /// The catalog removes the wal/ namespace before invoking this delegate. public Func TryParseJournalId { get; set; } = DefaultTryParseJournalId; /// @@ -122,6 +158,32 @@ public IAmazonS3? S3Client internal bool IsClientExternallyOwned { get; private set; } + internal bool UsesDefaultObjectKey => GetObjectKey == DefaultGetObjectKey; + + internal string? GetObjectKeyPrefixForCatalog(string? prefix) + { + if (prefix is null) + { + return null; + } + + var mapper = GetObjectKeyPrefix; + if (mapper is null && !UsesDefaultObjectKey) + { + throw new InvalidOperationException( + $"Custom {nameof(GetObjectKey)} mappings require {nameof(GetObjectKeyPrefix)} for prefixed journal listings."); + } + + var result = mapper is null ? prefix : mapper(new JournalId(prefix)); + if (string.IsNullOrEmpty(result)) + { + throw new InvalidOperationException( + $"{nameof(GetObjectKeyPrefix)} must return a non-empty object-key prefix."); + } + + return result; + } + internal string GetObjectKeyForJournal(JournalId journalId) { if (journalId.IsDefault) @@ -147,9 +209,9 @@ internal static string GetCheckpointObjectKeyForJournal(JournalId journalId, str return GetDefaultCheckpointObjectKey(journalObjectKey, snapshotId); } - internal static string GetDefaultWalObjectKey(string journalObjectKey) => $"{journalObjectKey}/wal"; + internal static string GetDefaultWalObjectKey(string journalObjectKey) => $"{WalObjectKeyPrefix}{journalObjectKey}"; - internal static string GetDefaultCheckpointObjectKey(string journalObjectKey, string snapshotId) => $"{journalObjectKey}/chk.{snapshotId}"; + internal static string GetDefaultCheckpointObjectKey(string journalObjectKey, string snapshotId) => $"checkpoints/{journalObjectKey}/{snapshotId}"; internal Func> GetCreateClient() => CreateClient ?? (_ => Task.FromResult(new AmazonS3Client(ClientConfig ?? new AmazonS3Config()))); diff --git a/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs b/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs index a5b8d876c91..9e50144a618 100644 --- a/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs +++ b/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs @@ -45,12 +45,51 @@ public IJournalStorage CreateStorage(JournalId journalId) return new S3JournalStorage(_shared, GetClient(), journalId); } - public async IAsyncEnumerable ListAsync( + public async IAsyncEnumerable ListAsync( ListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - var prefix = options?.Prefix ?? default; + var range = new JournalCatalogRange(options); + if (range.IsEmpty) + { + yield break; + } + + var ordered = _options.UseOrderedListing; + var identityMapping = _options.UsesDefaultObjectKey; + var listingPrefix = range.ListingPrefix; + if (!identityMapping + && (string.IsNullOrWhiteSpace(listingPrefix) + || _options.GetObjectKeyPrefix is null && range.Prefix is null)) + { + // A common prefix inferred from bounds can be whitespace, which cannot be represented + // as a JournalId for a custom mapper. Bounds alone also do not require a prefix mapper. + listingPrefix = null; + } + + var objectKeyPrefix = S3JournalStorageOptions.GetDefaultWalObjectKey( + _options.GetObjectKeyPrefixForCatalog(listingPrefix) ?? string.Empty); + if (!ordered) + { + var directoryEnd = objectKeyPrefix.LastIndexOf('/') + 1; + objectKeyPrefix = objectKeyPrefix[..directoryEnd]; + } + + string? startAfter = null; + if (ordered && identityMapping && range.LowerBound is { Length: > 0 } lowerBound + && System.Text.Ascii.IsValid(lowerBound)) + { + // StartAfter is exclusive: use a strictly earlier key to retain the inclusive lower bound. + var marker = S3JournalStorageOptions.GetDefaultWalObjectKey(lowerBound[..^1]); + if (string.CompareOrdinal(marker, objectKeyPrefix) > 0) + { + startAfter = marker; + } + } + + var maxObjectKey = ordered && identityMapping && range.MaxId is { } maxId && System.Text.Ascii.IsValid(maxId) + ? S3JournalStorageOptions.GetDefaultWalObjectKey(maxId) : null; var client = GetClient(); var bucketName = GetBucketName(); string? continuationToken = null; @@ -61,18 +100,27 @@ public async IAsyncEnumerable ListAsync( new ListObjectsV2Request { BucketName = bucketName, + Prefix = objectKeyPrefix, + StartAfter = continuationToken is null ? startAfter : null, MaxKeys = 1000, ContinuationToken = continuationToken, }, cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); - foreach (var item in response.S3Objects) + // AWS SDK v4 represents an empty listing page with a null collection. + foreach (var item in response.S3Objects ?? []) { cancellationToken.ThrowIfCancellationRequested(); - if (TryGetJournalId(item.Key, prefix, out var id)) + if (maxObjectKey is not null && string.CompareOrdinal(item.Key, maxObjectKey) > 0) + { + yield break; + } + + if (TryGetJournalId(item.Key, range, out var id)) { - yield return id; + // ListObjectsV2 cannot project the complete journal metadata without a separate request. + yield return new JournalCatalogEntry(id); } } @@ -83,11 +131,11 @@ public async IAsyncEnumerable ListAsync( cancellationToken.ThrowIfCancellationRequested(); } - private bool TryGetJournalId(string objectKey, JournalId prefix, out JournalId journalId) + private bool TryGetJournalId(string objectKey, JournalCatalogRange range, out JournalId journalId) { - if (objectKey.EndsWith("/wal", StringComparison.Ordinal) - && _options.TryParseJournalId(objectKey[..^"/wal".Length]) is { IsDefault: false } id - && prefix.IsPrefixOf(id)) + if (objectKey.StartsWith(S3JournalStorageOptions.WalObjectKeyPrefix, StringComparison.Ordinal) + && _options.TryParseJournalId(objectKey[S3JournalStorageOptions.WalObjectKeyPrefix.Length..]) is { IsDefault: false } id + && range.Contains(id.Value)) { var journalObjectKey = _options.GetObjectKeyForJournal(id); var canonicalWalObjectKey = S3JournalStorageOptions.GetWalObjectKeyForJournal(id, journalObjectKey); diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorage.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorage.cs index 977cfdd2b7e..1634a842847 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorage.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorage.cs @@ -908,7 +908,7 @@ private static IJournalMetadata ValidateCheckpointMetadata(CheckpointReference c } } - private static IJournalMetadata CreateJournalMetadata(ETag eTag, IDictionary? metadata) + internal static IJournalMetadata CreateJournalMetadata(ETag eTag, IDictionary? metadata) => new JournalMetadata( GetFormatKeyMetadata(metadata), eTag == default ? null : eTag.ToString(), diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageLayout.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageLayout.cs new file mode 100644 index 00000000000..e595a806144 --- /dev/null +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageLayout.cs @@ -0,0 +1,25 @@ +namespace Orleans.Journaling; + +internal static class AzureBlobJournalStorageLayout +{ + private const string WalPrefix = "wal/"; + + public static string GetWalBlobName(string journalId) => WalPrefix + journalId; + + public static string GetCheckpointBlobName(string journalId, string snapshotId) + => $"checkpoints/{journalId}/{snapshotId}"; + + public static bool TryGetJournalId(string blobName, out JournalId journalId) + { + if (blobName.StartsWith(WalPrefix, StringComparison.Ordinal) + && blobName[WalPrefix.Length..] is { } value + && !string.IsNullOrWhiteSpace(value)) + { + journalId = new JournalId(value); + return true; + } + + journalId = default; + return false; + } +} diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageOptions.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageOptions.cs index e25656b9225..9a3e5cd6667 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageOptions.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageOptions.cs @@ -23,6 +23,11 @@ public sealed class AzureBlobJournalStorageOptions /// /// Gets or sets the delegate used to generate the write-ahead log blob name for a journal. /// + /// + /// The default name is wal/{journalId.Value}. Catalog discovery uses this layout in + /// . Custom delegates which participate in catalog discovery must + /// produce the same name for each journal id. + /// public Func GetWalBlobName { get; set; } = DefaultGetWalBlobName; private static readonly Func DefaultGetWalBlobName = @@ -35,7 +40,7 @@ public sealed class AzureBlobJournalStorageOptions /// The delegate receives the journal id and an opaque snapshot id generated for the checkpoint. /// The snapshot id is currently formatted as a 32-character hexadecimal string using Guid.ToString("N"). /// The returned value must be a container-relative blob name. The default value is - /// {journalId.Value}/chk.{snapshotId}. + /// checkpoints/{journalId.Value}/{snapshotId}. /// public Func GetCheckpointBlobName { get; set; } = DefaultGetCheckpointBlobName; @@ -142,7 +147,7 @@ internal static string GetDefaultWalBlobName(JournalId journalId) throw new ArgumentException("The journal id must not be the default value.", nameof(journalId)); } - return $"{journalId.Value}/wal"; + return AzureBlobJournalStorageLayout.GetWalBlobName(journalId.Value); } internal static string GetDefaultCheckpointBlobName(JournalId journalId, string snapshotId) @@ -153,7 +158,7 @@ internal static string GetDefaultCheckpointBlobName(JournalId journalId, string } ArgumentException.ThrowIfNullOrWhiteSpace(snapshotId); - return $"{journalId.Value}/chk.{snapshotId}"; + return AzureBlobJournalStorageLayout.GetCheckpointBlobName(journalId.Value, snapshotId); } /// diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs index f7492ecf136..1e2e9ddb3c7 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs @@ -56,33 +56,55 @@ public IJournalStorage CreateStorage(JournalId journalId) return new AzureBlobJournalStorage(_shared, journalId); } - public async IAsyncEnumerable ListAsync( + public async IAsyncEnumerable ListAsync( ListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - var prefix = options?.Prefix ?? default; - var container = GetDefaultContainerClient(); + var range = new JournalCatalogRange(options); + if (range.IsEmpty) + { + yield break; + } + var container = GetDefaultContainerClient(); + var maxBlobName = range.MaxId is { } maxId && System.Text.Ascii.IsValid(maxId) + ? AzureBlobJournalStorageLayout.GetWalBlobName(maxId) : null; + var startFrom = range.LowerBound is { } lowerBound && System.Text.Ascii.IsValid(lowerBound) + ? AzureBlobJournalStorageLayout.GetWalBlobName(lowerBound) : null; await foreach (var page in container.GetBlobsAsync( - traits: BlobTraits.None, - states: BlobStates.None, - prefix: prefix.IsDefault ? null : prefix.Value, - cancellationToken: cancellationToken).AsPages(pageSizeHint: 5000)) + new GetBlobsOptions + { + Traits = range.IncludeMetadata ? BlobTraits.Metadata : BlobTraits.None, + Prefix = AzureBlobJournalStorageLayout.GetWalBlobName(range.ListingPrefix ?? string.Empty), + StartFrom = startFrom, + }, + cancellationToken).AsPages(pageSizeHint: 5000)) { cancellationToken.ThrowIfCancellationRequested(); foreach (var item in page.Values) { cancellationToken.ThrowIfCancellationRequested(); - if (item.Properties.BlobType is { } blobType && blobType != BlobType.Append - || !item.Name.EndsWith("/wal", StringComparison.Ordinal)) + // Azure's flat List Blobs API returns names in lexical order. Every matching WAL + // is at or below this raw-name bound, including the WAL for MaxId itself. + if (maxBlobName is not null && string.CompareOrdinal(item.Name, maxBlobName) > 0) + { + yield break; + } + + if (item.Properties.BlobType is { } blobType && blobType != BlobType.Append) { continue; } - if (TryParseJournalId(item.Name[..^"/wal".Length], out var journalId) && prefix.IsPrefixOf(journalId)) + if (AzureBlobJournalStorageLayout.TryGetJournalId(item.Name, out var journalId) + && range.Contains(journalId.Value)) { - yield return journalId; + yield return new( + journalId, + range.IncludeMetadata + ? AzureBlobJournalStorage.CreateJournalMetadata(item.Properties.ETag!.Value, item.Metadata) + : null); } } } @@ -102,20 +124,6 @@ private BlobContainerClient GetDefaultContainerClient() => _defaultContainer ?? throw new InvalidOperationException( $"{nameof(AzureBlobJournalStorageProvider)} has not been initialized. Ensure the silo lifecycle has started before using journal storage."); - private static bool TryParseJournalId(string value, out JournalId journalId) - { - try - { - journalId = new JournalId(value); - return true; - } - catch (ArgumentException) - { - journalId = default; - return false; - } - } - private static IJournalFormat GetJournalFormat(IServiceProvider serviceProvider, string journalFormatKey) { var journalFormat = serviceProvider.GetKeyedService(journalFormatKey); diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorage.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorage.cs index 636c35e4e4f..18acb9a925a 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorage.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorage.cs @@ -1042,11 +1042,14 @@ private HeaderState CreateHeaderState(TableEntity entity) private IJournalMetadata CreateJournalMetadata(ETag eTag, TableEntity entity) { ValidateHeaderJournalId(entity); - return new JournalMetadata( + return CreateJournalMetadataSnapshot(eTag, entity); + } + + internal static IJournalMetadata CreateJournalMetadataSnapshot(ETag eTag, TableEntity entity) + => new JournalMetadata( NormalizeFormat(entity.GetString(FormatPropertyName)), eTag == default ? null : eTag.ToString(), DeserializeCallerMetadata(entity.GetString(MetadataPropertyName))); - } private void ValidateHeaderJournalId(TableEntity entity) { diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageOptions.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageOptions.cs index 8db88a544d5..eac53a18b11 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageOptions.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageOptions.cs @@ -23,15 +23,21 @@ public sealed class AzureTableJournalStorageOptions /// Gets or sets the delegate used to generate the table partition key for a journal. /// /// - /// The returned value must be a valid Azure Table partition key. The default value percent-encodes - /// reversibly and rejects values whose encoded form exceeds the Azure - /// Table partition-key limit. + /// The returned value must be a valid Azure Table partition key. The default mapping accepts only + /// printable ASCII characters (U+0020 through U+007E) in and encodes each + /// byte as two uppercase hexadecimal digits, preserving ordinal ordering and raw prefixes. + /// The 1,024-character partition-key limit permits journal ids of at most 512 characters. + /// Custom mappings are not subject to the default mapping's journal-id restrictions and use canonical + /// journal-id property filters for catalog queries, which can require a full table scan instead of + /// an indexed partition-key range. Catalog bounds need not satisfy the stored journal-id restrictions. /// public Func GetPartitionKey { get; set; } = DefaultGetPartitionKey; private static readonly Func DefaultGetPartitionKey = static journalId => GetDefaultPartitionKey(journalId); + internal bool UsesDefaultPartitionKey => GetPartitionKey == DefaultGetPartitionKey; + /// /// Options to be used when configuring the table storage client, or to use the default options. /// @@ -144,13 +150,38 @@ internal static string GetDefaultPartitionKey(JournalId journalId) throw new ArgumentException("The journal id must not be the default value.", nameof(journalId)); } - // Percent-encoding escapes every character disallowed in partition keys ('/', '\', '#', '?', - // control characters) and is reversible. - var partitionKey = Uri.EscapeDataString(journalId.Value); + if (journalId.Value.Length > 512) + { + throw new ArgumentException( + "Azure Table partition keys must not exceed 1,024 characters.", + nameof(journalId)); + } + + if (journalId.Value.AsSpan().IndexOfAnyExceptInRange(' ', '~') >= 0) + { + throw new ArgumentException( + "The default Azure Table partition key mapping requires journal ids to contain only printable ASCII characters (U+0020 through U+007E).", + nameof(journalId)); + } + + var partitionKey = EncodePartitionKey(journalId.Value); ValidatePartitionKey(partitionKey, nameof(journalId)); return partitionKey; } + internal static string EncodePartitionKey(string value) + => string.Create(checked(value.Length * 2), value, static (destination, source) => + { + const string HexDigits = "0123456789ABCDEF"; + for (var index = 0; index < source.Length; index++) + { + var character = source[index]; + var offset = index * 2; + destination[offset] = HexDigits[character >> 4]; + destination[offset + 1] = HexDigits[character & 0xF]; + } + }); + internal static void ValidateTableName(string? tableName) { if (tableName is not { Length: >= 3 and <= 63 } || !IsAsciiLetter(tableName[0])) diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageProvider.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageProvider.cs index 2263ced4712..ca507cfaa3d 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageProvider.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureTableJournalStorageProvider.cs @@ -1,4 +1,6 @@ +using System.Buffers; using System.Runtime.CompilerServices; +using System.Text; using Azure.Data.Tables; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -10,6 +12,13 @@ namespace Orleans.Journaling; internal sealed class AzureTableJournalStorageProvider : ILifecycleParticipant, IJournalStorageProvider, IJournalStorageCatalog { private static readonly string[] JournalIdSelect = [AzureTableJournalStorage.JournalIdPropertyName]; + private static readonly string[] JournalMetadataSelect = + [ + AzureTableJournalStorage.JournalIdPropertyName, + AzureTableJournalStorage.FormatPropertyName, + AzureTableJournalStorage.MetadataPropertyName, + nameof(TableEntity.Timestamp), + ]; private readonly AzureTableJournalStorageOptions _options; private readonly AzureTableJournalStorage.InitializedTableClientProvider _tableClientProvider = new(); @@ -56,27 +65,40 @@ public IJournalStorage CreateStorage(JournalId journalId) return new AzureTableJournalStorage(_shared, journalId); } - public async IAsyncEnumerable ListAsync( + public async IAsyncEnumerable ListAsync( ListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - var prefix = options?.Prefix ?? default; + var range = new JournalCatalogRange(options); + if (range.IsEmpty + || (_options.UsesDefaultPartitionKey + && range.Prefix is { } prefix + && prefix.AsSpan().IndexOfAnyExceptInRange(' ', '~') >= 0)) + { + yield break; + } + var table = _tableClientProvider.GetTableClient(); - var filter = TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"); + var filter = GetCatalogFilter(range); await foreach (var page in table.QueryAsync( filter, maxPerPage: 1000, - select: JournalIdSelect, + select: range.IncludeMetadata ? JournalMetadataSelect : JournalIdSelect, cancellationToken: cancellationToken).AsPages(pageSizeHint: 1000)) { cancellationToken.ThrowIfCancellationRequested(); foreach (var entity in page.Values) { cancellationToken.ThrowIfCancellationRequested(); - if (TryGetJournalId(entity, out var journalId) && prefix.IsPrefixOf(journalId)) + if (TryGetJournalId(entity, out var journalId) + && range.Contains(journalId.Value)) { - yield return journalId; + yield return new( + journalId, + range.IncludeMetadata + ? AzureTableJournalStorage.CreateJournalMetadataSnapshot(entity.ETag, entity) + : null); } } } @@ -84,22 +106,84 @@ public async IAsyncEnumerable ListAsync( cancellationToken.ThrowIfCancellationRequested(); } - private static bool TryGetJournalId(TableEntity entity, out JournalId journalId) + private string GetCatalogFilter(JournalCatalogRange range) { - if (entity.GetString(AzureTableJournalStorage.JournalIdPropertyName) is { } journalIdValue) + var filter = TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"); + if (_options.UsesDefaultPartitionKey) { - return TryParseJournalId(journalIdValue, out journalId); + if (range.LowerBound is { } lowerBound) + { + var lowerKey = GetPartitionKeyBound(lowerBound); + filter += TableClient.CreateQueryFilter($" and PartitionKey ge {lowerKey}"); + } + + if (range.MaxId is { } maxId) + { + var upperKey = GetPartitionKeyBound(maxId); + filter += TableClient.CreateQueryFilter($" and PartitionKey le {upperKey}"); + } + + if (range.Prefix is { } prefix) + { + // Encoded keys contain only 0..F, so G bounds every suffix of the encoded prefix. + var prefixEnd = AzureTableJournalStorageOptions.EncodePartitionKey(prefix) + "G"; + filter += TableClient.CreateQueryFilter($" and PartitionKey lt {prefixEnd}"); + } + } + else + { + // Ordinal UTF-16 boundaries can contain unpaired surrogates. Keep those constraints local. + if (range.LowerBound is { } lowerBound && IsWellFormedUnicode(lowerBound)) + { + filter += TableClient.CreateQueryFilter($" and JournalId ge {lowerBound}"); + } + + if (range.UpperBound is { } upperBound && IsWellFormedUnicode(upperBound)) + { + filter += range.Contains(upperBound) + ? TableClient.CreateQueryFilter($" and JournalId le {upperBound}") + : TableClient.CreateQueryFilter($" and JournalId lt {upperBound}"); + } } - // Legacy headers can only be listed when they use the reversible default partition mapping. - var decodedPartitionKey = Uri.UnescapeDataString(entity.PartitionKey); - if (TryParseJournalId(decodedPartitionKey, out journalId) - && string.Equals( - Uri.EscapeDataString(journalId.Value), - entity.PartitionKey, - StringComparison.Ordinal)) + return filter; + } + + private static string GetPartitionKeyBound(string value) + { + var index = value.AsSpan().IndexOfAnyExceptInRange(' ', '~'); + if (index < 0) { - return true; + return AzureTableJournalStorageOptions.EncodePartitionKey(value); + } + + // No stored id can equal an unsupported bound. At its first unsupported code unit, + // 1F sorts after the exact prefix but before any descendant; 7F sorts after all descendants. + return AzureTableJournalStorageOptions.EncodePartitionKey(value[..index]) + + (value[index] < ' ' ? "1F" : "7F"); + } + + private static bool IsWellFormedUnicode(ReadOnlySpan value) + { + while (!value.IsEmpty) + { + if (Rune.DecodeFromUtf16(value, out _, out var consumed) != OperationStatus.Done) + { + return false; + } + + value = value[consumed..]; + } + + return true; + } + + private static bool TryGetJournalId(TableEntity entity, out JournalId journalId) + { + if (entity.TryGetValue(AzureTableJournalStorage.JournalIdPropertyName, out var value) + && value is string journalIdValue) + { + return TryParseJournalId(journalIdValue, out journalId); } journalId = default; diff --git a/src/Azure/Orleans.Journaling.AzureStorage/README.md b/src/Azure/Orleans.Journaling.AzureStorage/README.md index 26336d624b9..ed0cd3b5766 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/README.md +++ b/src/Azure/Orleans.Journaling.AzureStorage/README.md @@ -7,7 +7,7 @@ Blob names are derived from the configured journal storage identity and do not u ## Using an alternative blob layout -By default, WAL blobs are named `/wal` and checkpoint blobs are named `/chk.`. Configure the blob name delegates to use an alternative layout, such as a shared prefix, file extensions, tenant-specific paths, or names which match an existing storage convention. Each delegate returns a container-relative blob name, and checkpoint names should include the supplied snapshot id to avoid collisions. +By default, WAL blobs are named `wal/` and checkpoint blobs are named `checkpoints//`. The separate prefixes let catalog discovery select WAL blobs directly, keeping retained checkpoints out of listing pages. Configure the blob name delegates to use an alternative layout, such as a shared prefix, file extensions, tenant-specific paths, or names which match an existing storage convention. Each delegate returns a container-relative blob name, and checkpoint names should include the supplied snapshot id to avoid collisions. Catalog discovery uses the default WAL layout in the configured container; custom delegates participating in discovery produce `wal/` for each journal. ```csharp siloBuilder.AddAzureBlobJournalStorage(options => @@ -33,11 +33,13 @@ siloBuilder.AddAzureTableJournalStorage(options => ## Catalog enumeration -Both Azure providers implement `IJournalStorageCatalog.ListAsync`, returning identities incrementally in service traversal order. Set `ListOptions.Prefix` to enumerate an exact journal id and its descendants. The provider handles service continuations internally and yields identities from the current page before fetching the next page. +Both Azure providers implement `IJournalStorageCatalog.ListAsync`, returning `JournalCatalogEntry` values incrementally in service traversal order. Each entry carries its journal identity in `Id`. `ListOptions.Prefix` is a raw ordinal string prefix and may end within a segment; use a trailing `/` when selecting only entries inside a namespace. `MinId` and `MaxId` are inclusive ordinal bounds, each unlimited by default. All constraints apply and are snapshotted when enumeration starts. Consumers requiring due order must sort selected ids using `StringComparer.Ordinal`. The provider handles service continuations internally and yields entries from the current page before fetching the next page. -The Blob catalog scans the configured `ContainerName` and interprets append blobs named `/wal` as journal identities. This traversal applies equally when a custom naming delegate or container factory produces the same entries. Each internal page requests up to 5000 blobs, including checkpoints and other entries before filtering. +Set `ListOptions.IncludeMetadata` to include each entry's format, ETag, and caller-owned properties in `Metadata`. The snapshot is observed together in the listing response and has the same semantics as `GetMetadataAsync`; its ETag can be supplied to `UpdateMetadataAsync` for a conditional update. Blob listing requests metadata traits, while Table listing selects the header's format and serialized caller metadata alongside the identity and timestamp. Both providers construct snapshots directly from listing pages. By default, enumeration projects identities with `Metadata` set to `null`. `IncludeMetadata` is snapshotted with the range options when enumeration starts. -Table enumeration supports custom partition mappings through the canonical journal id stored in each header. It requests up to 1000 header rows per service page; Table Storage determines the internal scan work required by that query. Prefix filtering occurs on the returned headers, so one enumerator advance can cross multiple empty or filtered pages. +The Blob catalog scans the configured `ContainerName` and interprets append blobs named `wal/` as journal identities. This traversal applies equally when a custom naming delegate or container factory produces the same entries. Requests use `wal/` followed by the raw journal-id prefix, narrowed by the common prefix of `MinId` and `MaxId` when possible, and up to 5000 blobs per page. Checkpoints occupy their own namespace and are excluded before pagination, including for unbounded catalog queries. For an ASCII lower bound, `GetBlobsOptions.StartFrom` seeks to `wal/`, preserving the minimum journal's WAL. Azure's lexical listing order permits stopping after `wal/` for an ASCII upper bound. Other bounds are enforced locally. Upper-bound termination happens while consuming results: the provider reads the page crossing the upper bound and then completes the traversal. Empty intersections issue no request. Recovery and checkpoint cleanup use the full checkpoint name published in WAL metadata. + +Table enumeration requests up to 1000 header rows per service page and reads identities from the canonical `JournalId` header property. The default partition mapping accepts printable ASCII journal ids (`0x20` through `0x7E`) and encodes each byte as two uppercase hexadecimal digits, preserving ordinal ordering and raw prefixes, including partial segments. The 1,024-character partition-key limit therefore permits at most 512 characters per journal id. Validation occurs when creating storage. This restriction belongs to the default Table mapping; custom mappings and other providers retain their journal-id contracts. Default-mapping queries combine the header row key with indexed partition-key constraints for the raw prefix, inclusive `MinId`, and inclusive `MaxId`. Query bounds retain their original ordinal meaning, including bounds outside the stored ASCII alphabet. An empty intersection issues no query. Custom mappings apply well-formed Unicode bounds to the canonical `JournalId` property; bounds containing unpaired surrogates are enforced locally. This preserves ordinal range semantics and can require a full table scan. Unbounded enumeration also scans headers across the table. Constraints are checked again before yielding, and one enumerator advance can cross multiple empty or filtered pages. Use `await foreach` or dispose a retained enumerator when stopping early. Pass a cancellation token covering the traversal lifetime; cancellation and service errors propagate to the caller. diff --git a/src/Orleans.DurableJobs/JournaledJobShardManager.cs b/src/Orleans.DurableJobs/JournaledJobShardManager.cs index 74d7952c43b..5d327dcc068 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardManager.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardManager.cs @@ -85,9 +85,9 @@ public override async Task> AssignJobShardsAsync(DateTimeOffset var newClaimCount = 0; var membershipSnapshot = _membershipService.CurrentSnapshot; - await foreach (var storageId in _catalog.ListAsync(new() { Prefix = JobShardId.StoragePrefix }, cancellationToken)) + await foreach (var entry in _catalog.ListAsync(new() { Prefix = JobShardId.StoragePrefix }, cancellationToken)) { - var descriptor = await GetDescriptorAsync(storageId, cancellationToken); + var descriptor = await GetDescriptorAsync(entry.Id, cancellationToken); if (descriptor is null || descriptor.Poisoned || descriptor.StartTime > maxDueTime) { continue; diff --git a/src/Orleans.Journaling/IJournalStorageCatalog.cs b/src/Orleans.Journaling/IJournalStorageCatalog.cs index 7a59456ad7c..38ecdb33796 100644 --- a/src/Orleans.Journaling/IJournalStorageCatalog.cs +++ b/src/Orleans.Journaling/IJournalStorageCatalog.cs @@ -4,28 +4,33 @@ namespace Orleans.Journaling; /// Provides catalog operations for journal storage instances. /// /// -/// A catalog discovers storage identities. provides storage lifecycle, +/// A catalog discovers storage identities and optional metadata snapshots. provides storage lifecycle, /// metadata, and data mutation operations. /// public interface IJournalStorageCatalog { /// - /// Enumerates journal ids matching the supplied options. + /// Enumerates journal entries matching the supplied options. /// - /// The listing options, or to list all ids. + /// The listing options, or to list all entries without metadata. /// The cancellation token. - /// Matching ids in provider traversal order. + /// Matching entries in provider traversal order. /// - /// Options are read when enumeration begins. Providers fetch storage pages internally and yield matching ids + /// Options are snapshotted when enumeration begins. The raw prefix and inclusive lower and upper bounds all apply. + /// Results are not guaranteed to be sorted; callers requiring ordering must sort the selected ids. + /// requests complete metadata snapshots available from the listing. + /// Entries carry metadata when the provider cannot project it or it was not requested. + /// Providers fetch storage pages internally and yield matching entries /// as they are discovered. Advancing the enumerator can traverse multiple empty or filtered storage pages. /// Storage services determine request latency, retries, and internal scan work. /// Enumeration observes live storage; concurrent changes follow the provider's listing semantics. /// Callers should tolerate repeated identities during concurrent changes and start a new enumeration to - /// discover later changes. Dispose the enumerator when stopping early. + /// discover later changes. Deduplicate by when unique identities + /// are required; repeated entries can carry different metadata versions. Dispose the enumerator when stopping early. /// Storage and cancellation errors propagate through enumeration. Start a new enumeration after a listing error. /// /// is canceled. - IAsyncEnumerable ListAsync( + IAsyncEnumerable ListAsync( ListOptions? options = null, CancellationToken cancellationToken = default); } diff --git a/src/Orleans.Journaling/JournalCatalogEntry.cs b/src/Orleans.Journaling/JournalCatalogEntry.cs new file mode 100644 index 00000000000..a8705dfc869 --- /dev/null +++ b/src/Orleans.Journaling/JournalCatalogEntry.cs @@ -0,0 +1,16 @@ +namespace Orleans.Journaling; + +/// +/// A journal identity and an optional metadata snapshot returned by a catalog. +/// +/// The journal identity. +/// +/// The journal format, storage ETag, and complete caller-owned metadata properties observed together, +/// or when metadata was not requested or is unavailable from the listing. +/// +/// +/// Metadata describes the journal at the time it was listed. Use its ETag for conditional updates; +/// concurrent changes can invalidate the snapshot. A non-null snapshot has the same metadata semantics +/// as . +/// +public readonly record struct JournalCatalogEntry(JournalId Id, IJournalMetadata? Metadata = null); diff --git a/src/Orleans.Journaling/JournalCatalogRange.cs b/src/Orleans.Journaling/JournalCatalogRange.cs new file mode 100644 index 00000000000..69c1b4c9452 --- /dev/null +++ b/src/Orleans.Journaling/JournalCatalogRange.cs @@ -0,0 +1,113 @@ +namespace Orleans.Journaling; + +internal readonly struct JournalCatalogRange +{ + public JournalCatalogRange(ListOptions? options) + { + IncludeMetadata = options?.IncludeMetadata ?? false; + Prefix = options?.Prefix.Value; + MinId = options?.MinId.Value; + MaxId = options?.MaxId.Value; + var prefixEnd = GetPrefixEnd(Prefix); + LowerBound = Max(Prefix, MinId); + UpperBound = Min(prefixEnd, MaxId); + IsEmpty = LowerBound is not null && UpperBound is not null + && string.CompareOrdinal(LowerBound, UpperBound) > 0; + if (MinId is not null && prefixEnd is not null + && string.CompareOrdinal(MinId, prefixEnd) >= 0) + { + IsEmpty = true; + } + + var commonPrefix = GetCommonPrefix(MinId, MaxId); + var listingPrefix = Prefix; + if (!string.IsNullOrEmpty(commonPrefix)) + { + if (listingPrefix is null || commonPrefix.StartsWith(listingPrefix, StringComparison.Ordinal)) + { + listingPrefix = commonPrefix; + } + else if (!listingPrefix.StartsWith(commonPrefix, StringComparison.Ordinal)) + { + IsEmpty = true; + } + } + + // Broaden only the native prefix to well-formed UTF-16; Contains retains the original ordinal range. + for (var index = 0; listingPrefix is not null && index < listingPrefix.Length; index++) + { + var character = listingPrefix[index]; + if (!char.IsSurrogate(character)) + { + continue; + } + + if (char.IsHighSurrogate(character) && index + 1 < listingPrefix.Length + && char.IsLowSurrogate(listingPrefix[index + 1])) + { + index++; + continue; + } + + listingPrefix = listingPrefix[..index]; + break; + } + + ListingPrefix = string.IsNullOrEmpty(listingPrefix) ? null : listingPrefix; + } + + public string? Prefix { get; } + public string? MinId { get; } + public string? MaxId { get; } + public string? ListingPrefix { get; } + public string? LowerBound { get; } + public string? UpperBound { get; } + public bool IsEmpty { get; } + public bool IncludeMetadata { get; } + + public bool Contains(string value) + => !IsEmpty + && (Prefix is null || value.StartsWith(Prefix, StringComparison.Ordinal)) + && (MinId is null || string.CompareOrdinal(value, MinId) >= 0) + && (MaxId is null || string.CompareOrdinal(value, MaxId) <= 0); + + private static string? GetPrefixEnd(string? prefix) + { + if (prefix is null) + { + return null; + } + + for (var index = prefix.Length - 1; index >= 0; index--) + { + if (prefix[index] != char.MaxValue) + { + return string.Concat(prefix.AsSpan(0, index), ((char)(prefix[index] + 1)).ToString()); + } + } + + return null; + } + + private static string? GetCommonPrefix(string? left, string? right) + { + if (left is null || right is null) + { + return null; + } + + var length = 0; + while (length < left.Length && length < right.Length && left[length] == right[length]) + { + length++; + } + + return left[..length]; + } + + private static string? Min(string? left, string? right) + => left is null ? right : right is null || string.CompareOrdinal(left, right) <= 0 ? left : right; + + private static string? Max(string? left, string? right) + => left is null ? right : right is null || string.CompareOrdinal(left, right) >= 0 ? left : right; +} diff --git a/src/Orleans.Journaling/ListOptions.cs b/src/Orleans.Journaling/ListOptions.cs index a6785ddc995..c0258b3656d 100644 --- a/src/Orleans.Journaling/ListOptions.cs +++ b/src/Orleans.Journaling/ListOptions.cs @@ -3,13 +3,40 @@ namespace Orleans.Journaling; /// /// Options for enumerating journal storage identities. /// +/// +/// All constraints apply using ordinal comparisons. Providers snapshot these options when enumeration begins. +/// public sealed class ListOptions { /// - /// Gets or sets the journal id prefix. The default value matches all ids. + /// Gets or sets the raw prefix of . The default value matches all ids. /// /// - /// A prefix matches the exact journal id and its descendant segments. + /// Prefixes can end within a path segment, for example jobs/shards/20260909. + /// Include a trailing slash to select descendants of a namespace. /// public JournalId Prefix { get; set; } + + /// + /// Gets or sets the inclusive lower bound on , compared using + /// . The default value does not impose a lower bound. + /// + public JournalId MinId { get; set; } + + /// + /// Gets or sets the inclusive upper bound on , compared using + /// . The default value does not impose an upper bound. + /// + public JournalId MaxId { get; set; } + + /// + /// Gets or sets whether the catalog includes metadata available from its listing operation. + /// Defaults to . + /// + /// + /// Providers project a complete metadata snapshot when available without a separate per-journal + /// metadata request. Otherwise, is . + /// Callers which require missing metadata can retrieve it using . + /// + public bool IncludeMetadata { get; set; } } diff --git a/src/Orleans.Journaling/Orleans.Journaling.csproj b/src/Orleans.Journaling/Orleans.Journaling.csproj index 0cacff49493..1163bc071d0 100644 --- a/src/Orleans.Journaling/Orleans.Journaling.csproj +++ b/src/Orleans.Journaling/Orleans.Journaling.csproj @@ -22,6 +22,9 @@ + + + diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 56320decafc..4babb729c9e 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -110,21 +110,59 @@ Existing data is read using its stored format metadata, or as legacy OrleansBina ## Catalog enumeration -`IJournalStorageCatalog.ListAsync` returns an `IAsyncEnumerable` in provider traversal order. Pass `ListOptions` with `Prefix` to select an exact journal id and its descendants, or omit the options to enumerate all ids. Options are read when enumeration begins. +`IJournalStorageCatalog.ListAsync` returns an `IAsyncEnumerable` in provider traversal order. +Each entry contains its `Id` and optional `Metadata`. Deduplicate by `Id` when unique identities +are required, since repeated entries can carry different metadata versions. +`ListOptions.Prefix` matches the raw beginning of `JournalId.Value`, including partial path +segments. For example, `jobs/shards/20260909` selects timestamped names for that UTC day. +Use a trailing slash, such as `jobs/shards/`, to select a namespace's descendants. +`MinId` and `MaxId` supply inclusive lower and upper bounds. All three constraints use +`StringComparison.Ordinal` and are snapshotted when enumeration begins. Default values +leave the corresponding constraint open. Disjoint constraints produce an empty result. -When updating callers of the former prefix overload, pass `new ListOptions { Prefix = prefix }`. Applications which require ordinal ordering can materialize the sequence and sort `JournalId.Value` using `StringComparer.Ordinal`. +```csharp +var options = new ListOptions +{ + Prefix = new JournalId("jobs/shards/20260909"), + MinId = new JournalId("jobs/shards/20260909T1000000000000Z-"), + MaxId = new JournalId("jobs/shards/20260909T1200000000000Z~"), + IncludeMetadata = true +}; +``` + +Providers can narrow the native prefix further using the common prefix of the lower and upper +bounds. Applications requiring a uniform result order sort the selected ids using +`StringComparer.Ordinal`. Storage providers fetch pages internally and yield matching identities as they discover them. `await foreach` advances the traversal and disposes the enumerator when the loop ends. Consumers which process identities in batches can retain one enumerator across batches, advance it serially, and dispose it after the last pending `MoveNextAsync` completes. Use a cancellation token whose lifetime covers that enumeration. -| Provider | Internal traversal | Client memory | -| --- | --- | --- | -| Volatile | Enumerates the existing concurrent storage dictionary, checks journal existence, and applies the prefix | Constant additional traversal state; the storage dictionary holds the journals. | -| Azure Blob | Requests up to 5000 blobs per service page, then applies WAL and hierarchical prefix filtering | Proportional to the current service page. | -| Azure Table | Requests up to 1000 journal headers per service page, then decodes canonical or reversible legacy ids and applies the prefix | Proportional to the current service page. | -| S3 | Requests up to 1000 bucket objects per `ListObjectsV2` page, then applies canonical WAL and prefix filtering | Proportional to the current service page; includes unordered S3 Express listings. | -| Redis | Scans primary-server metadata keys and reads canonical ids in bounded batches, suppressing repeated ids | Read-batch state plus a seen-id set which grows with the catalog. `SCAN` count remains a service work hint. | +`IncludeMetadata` defaults to false and is snapshotted with the range options. When true, providers +include complete metadata available from the listing: journal format, storage ETag, and caller-owned +properties observed together, using the same semantics as `GetMetadataAsync`. Azure Blob projects +WAL metadata, Azure Table projects header properties, and Volatile snapshots metadata under its store +lock. S3 and Redis return null metadata because their listings expose identities rather than the +complete journal metadata. Projection adds no separate per-journal metadata requests. + +Consumers requiring metadata use the supplied snapshot or call `GetMetadataAsync` when it is absent. +An empty caller-property dictionary is a valid complete snapshot. Treat the ETag as the version of that +snapshot and pass it to conditional metadata updates; concurrent changes can cause the update to fail. -One `MoveNextAsync` can traverse multiple empty or filtered storage pages before yielding an identity. A consumer's identity or metadata-read budget therefore bounds returned candidates, while storage services determine internal scan work, request latency, and retries. Table header queries can inspect additional rows internally, and S3 custom mappings require a bucket traversal. +| Provider | How narrowly discovery scans | Remaining work | +| --- | --- | --- | +| Volatile | An ordered key index selects a view covering the requested prefix and bounds. | Snapshots selected keys and checks current journal existence. | +| Azure Table, default mapping | Printable ASCII ids use two uppercase hex digits per byte, allowing direct indexed prefix and lower/upper key filters. | Queries include the journal header row condition; the service controls work inside the selected key range. | +| Azure Table, custom mapping | Canonical journal-id filters limit returned headers. | Arbitrary mappings can require a table scan because the journal-id property is not indexed. | +| Azure Blob | The `wal/` namespace and raw id prefix select WAL blobs; `StartFrom` seeks to an ASCII lower bound and ordered traversal stops at an ASCII upper bound. | The final page can contain WALs beyond the range. Checkpoints occupy a separate namespace. | +| S3 general-purpose, ordered listing enabled | The `wal/` namespace excludes checkpoints. Identity-mapped keys use native prefixes and an initial `StartAfter` marker preceding the inclusive lower bound, then stop at the upper WAL key. | The final page can overrun the range. Custom key mappings use their configured native prefix and identity filtering. | +| S3 Express directory buckets | A native slash-terminated prefix within `wal/` limits the namespace and excludes checkpoints. | Partial-name and time bounds are filtered during unordered traversal. | +| Redis | Readable key names enable native `SCAN MATCH` prefix filtering and local key-range checks before identity metadata reads for the default mapping. | `SCAN MATCH` still traverses the server keyspace. Custom key mappings read canonical ids from matching metadata hashes. | + +Blob and ordered S3 native lower/upper optimizations apply where storage ordering agrees with +ordinal identity ordering, including the fixed-width ASCII timestamp names. Other bounds +remain enforced while traversing storage. S3 ordered listing is an explicit +`UseOrderedListing` capability setting for general-purpose buckets. +The selected identity count, transferred keys, and backend scan work are separate costs: +server-side filtering can reduce transferred data while the service still examines a wider keyspace. Enumeration observes live storage. Concurrent changes follow each provider's listing semantics; callers should tolerate repeated identities during changes and use subsequent enumerations to discover later updates. Journal existence can change between discovery and a storage operation. Cancellation and storage errors propagate through enumeration. Dispose a failed enumerator and begin a new enumeration when retrying a listing operation. diff --git a/src/Orleans.Journaling/VolatileJournalStorage.cs b/src/Orleans.Journaling/VolatileJournalStorage.cs index ce337459333..a358d5a4cdc 100644 --- a/src/Orleans.Journaling/VolatileJournalStorage.cs +++ b/src/Orleans.Journaling/VolatileJournalStorage.cs @@ -14,6 +14,8 @@ public sealed class VolatileJournalStorageProvider : IJournalStorageProvider, IJ { private readonly IOptions? _options; private readonly ConcurrentDictionary _storage = new(StringComparer.Ordinal); + private readonly SortedSet _storageKeys = new(StringComparer.Ordinal); + private readonly object _catalogLock = new(); /// /// Initializes a new instance of the class using the default journal format. @@ -41,36 +43,76 @@ public IJournalStorage CreateStorage(JournalId journalId) } var journalFormatKey = GetJournalFormatKey(); - var store = _storage.GetOrAdd(journalId.Value, static key => new VolatileJournalStorage.Store(key)); + if (!_storage.TryGetValue(journalId.Value, out var store)) + { + lock (_catalogLock) + { + if (!_storage.TryGetValue(journalId.Value, out store)) + { + store = new VolatileJournalStorage.Store(journalId.Value); + _storageKeys.Add(journalId.Value); + _storage[journalId.Value] = store; + } + } + } + return new VolatileJournalStorage(store, journalFormatKey); } /// - public async IAsyncEnumerable ListAsync( + public async IAsyncEnumerable ListAsync( ListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - var prefix = options?.Prefix ?? default; - foreach (var (key, store) in _storage) + var range = new JournalCatalogRange(options); + if (range.IsEmpty) + { + yield break; + } + + string[] keys; + lock (_catalogLock) + { + if (_storageKeys.Count == 0) + { + keys = []; + } + else + { + var lower = range.LowerBound ?? _storageKeys.Min!; + var upper = range.UpperBound ?? _storageKeys.Max!; + keys = string.CompareOrdinal(lower, upper) <= 0 + ? _storageKeys.GetViewBetween(lower, upper).ToArray() + : []; + } + } + + foreach (var key in keys) { cancellationToken.ThrowIfCancellationRequested(); - var journalId = new JournalId(key); - if (!prefix.IsPrefixOf(journalId)) + if (!range.Contains(key)) { continue; } + var store = _storage[key]; + IJournalMetadata? metadata = null; lock (store.SyncRoot) { if (!store.Exists) { continue; } + + if (range.IncludeMetadata) + { + metadata = store.GetMetadata(); + } } cancellationToken.ThrowIfCancellationRequested(); - yield return journalId; + yield return new JournalCatalogEntry(new JournalId(key), metadata); } cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Redis/Orleans.Journaling.Redis/README.md b/src/Redis/Orleans.Journaling.Redis/README.md index 4596d986c65..81532e2d5c7 100644 --- a/src/Redis/Orleans.Journaling.Redis/README.md +++ b/src/Redis/Orleans.Journaling.Redis/README.md @@ -4,7 +4,7 @@ Microsoft Orleans Journaling for Redis provides a Redis-backed implementation of the Orleans Journaling storage and catalog abstractions. Orleans Durable Jobs can use Orleans Journaling as a backing store, so this provider can persist durable job state through the journaling layer. -The provider stores each journal as Redis string data plus Redis hash metadata. Per-journal reads and mutations use atomic Lua scripts. Journal discovery scans metadata keys on each connected primary Redis server and reads the canonical journal id from each hash. Configure Redis persistence, such as AOF with an appropriate `appendfsync` setting, according to the durability guarantees required by your application. +The provider stores each journal as Redis string data plus Redis hash metadata. Per-journal reads and mutations use atomic Lua scripts. Journal discovery scans metadata keys on each connected primary Redis server. The default key mapping preserves journal ids in the keys, so discovery does not read the metadata hashes. Configure Redis persistence, such as AOF with an appropriate `appendfsync` setting, according to the durability guarantees required by your application. ## Getting Started @@ -38,20 +38,34 @@ If the Redis connection is already registered in dependency injection, configure Use `IJournalStorageCatalog.ListAsync` with optional `ListOptions` to filter by a journal id prefix: ```csharp -await foreach (var journalId in catalog.ListAsync( +await foreach (var entry in catalog.ListAsync( new ListOptions { Prefix = JournalId.Create("jobs") }, cancellationToken)) { - // Process the discovered journal. + var journalId = entry.Id; } ``` -The prefix is read when enumeration begins. Discovery scans primary servers incrementally and reads canonical journal ids in batches of at most 128 metadata keys, yielding matching ids in traversal order without sorting or buffering the whole catalog. Duplicate ids are suppressed across the traversal using O(N) seen-id memory for N distinct matching ids; metadata keys are not retained for the whole traversal, so repeated scan results can cause repeated metadata reads. +`ListOptions.Prefix` matches the raw journal id using ordinal `StartsWith`, including partial segments. For example, `new JournalId("jobs/2026/0")` matches both `jobs/2026/01` and `jobs/2026/09`. `ListOptions.MinId` and `MaxId` add inclusive ordinal lower and upper bounds; their default values are unlimited. All filters apply together and are snapshotted when enumeration begins. Empty intersections perform no scans or metadata reads. -The Redis `SCAN COUNT` value of 250 is a hint, not a strict response-size or server-work bound. One enumerator advancement can traverse many empty or nonmatching scans, and Redis/client-side scan buffering is not bounded by the metadata batch size. Discovery observes live storage rather than a snapshot. +Entries contain the journal `Id` and null `Metadata`, including when `IncludeMetadata` is requested. +Redis key scans supply identities; callers requiring complete metadata retrieve it using +`IJournalStorage.GetMetadataAsync`. The listing retains its existing key-scan and canonical-id read costs. + +With the default `GetKeyName` mapping, discovery supplies an escaped `SCAN MATCH` pattern for metadata keys beginning with the encoded raw prefix. A common prefix of `MinId` and `MaxId` can narrow this pattern further. Returned keys contain the journal id, which is decoded and filtered against both bounds locally without `HGET` requests. Malformed matching keys are errors, not silently ignored entries. + +Custom `GetKeyName` mappings cannot safely translate logical journal prefixes or bounds into Redis key names. Discovery therefore scans all metadata keys for the configured key prefix and reads canonical journal ids from their `$journal-id` fields in concurrent batches of at most 128, then applies the filters locally. Missing metadata is checked atomically to distinguish deletion from corruption. + +Discovery yields ids incrementally in traversal order, without sorting or buffering the whole catalog. Redis `SCAN` is unordered, so encountering a future id does not end traversal. Consumers requiring due order must sort the selected ids using `StringComparer.Ordinal`. Duplicate ids are suppressed across the traversal using O(N) seen-id memory for N distinct matching ids. Metadata keys are not retained for the whole traversal, so custom mappings can cause repeated metadata reads for repeated scan results. + +**`SCAN MATCH` still traverses the Redis keyspace on the server.** Prefix narrowing reduces returned keys and client work, not the server-side keyspace scan. Arbitrary ordinal ranges are filtered client-side, not sought through an index. The Redis `SCAN COUNT` value of 250 is a hint, not a strict response-size or server-work bound. One enumerator advancement can traverse many empty or nonmatching scans, and Redis/client-side scan buffering is not bounded by the metadata batch size. Discovery observes live storage rather than a snapshot. With the default mapping, a concurrently deleted metadata key can still yield its journal id; consumers must handle the journal no longer existing when they read it. Storage errors and cancellation propagate without provider-level retries, restarts, or success fallbacks. A disconnected primary or the absence of any primary is an error; failures on later servers can occur after earlier ids have been yielded. Dispose the enumerator when stopping early. Cancellation is checked between scan and metadata operations, but does not abort an in-flight Redis metadata request. +## Redis key layout + +Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. The reversible key name is outside the existing SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI encoding preserves literal percent signs, separators, Unicode, and Redis glob characters in journal ids; the scan pattern also escapes glob characters in the configured key prefix. + ## Documentation - [Microsoft Orleans Documentation](https://dotnet.github.io/orleans/docs/) diff --git a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs index 704ec4a4e64..758e6e1baea 100644 --- a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs +++ b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs @@ -606,10 +606,20 @@ internal static RedisKey GetMetadataKey(string keyPrefix, string keyName) return $"{baseKey}:metadata"; } - internal static RedisValue GetMetadataKeyPattern(string keyPrefix) - => $"{EscapeRedisPattern(keyPrefix)}:journal:*:metadata"; + internal static RedisValue GetMetadataKeyPattern(string keyPrefix, string? journalIdPrefix = null) + { + // An ordinal prefix can end halfway through a surrogate pair. Broaden the scan in that + // case since escaping the incomplete pair would not match the encoded complete character. + if (journalIdPrefix is { Length: > 0 } && char.IsHighSurrogate(journalIdPrefix[^1])) + { + journalIdPrefix = journalIdPrefix[..^1]; + } - private static RedisKey GetDataKey(string keyPrefix, string keyName) + var encodedPrefix = Uri.EscapeDataString(journalIdPrefix ?? string.Empty); + return $"{EscapeRedisPattern(keyPrefix)}:journal:{{*}}:{EscapeRedisPattern(encodedPrefix)}*:metadata"; + } + + internal static RedisKey GetDataKey(string keyPrefix, string keyName) { var baseKey = GetJournalBaseKey(keyPrefix, keyName); return $"{baseKey}:data"; @@ -618,7 +628,30 @@ private static RedisKey GetDataKey(string keyPrefix, string keyName) private static string GetJournalBaseKey(string keyPrefix, string keyName) { var hashTag = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyName))); - return $"{keyPrefix}:journal:{{{hashTag}}}"; + return $"{keyPrefix}:journal:{{{hashTag}}}:{Uri.EscapeDataString(keyName)}"; + } + + internal static JournalId GetJournalIdFromMetadataKey(string keyPrefix, RedisKey metadataKey) + { + var key = metadataKey.ToString(); + var prefix = $"{keyPrefix}:journal:{{"; + const string suffix = ":metadata"; + const int hashTagLength = 64; + var keyNameOffset = prefix.Length + hashTagLength + 2; + if (key.StartsWith(prefix, StringComparison.Ordinal) + && key.EndsWith(suffix, StringComparison.Ordinal) + && key.Length > keyNameOffset + suffix.Length + && key.AsSpan(prefix.Length + hashTagLength, 2).SequenceEqual("}:")) + { + var keyName = Uri.UnescapeDataString(key[keyNameOffset..^suffix.Length]); + if (TryParseJournalId(keyName, out var journalId) + && GetMetadataKey(keyPrefix, keyName) == metadataKey) + { + return journalId; + } + } + + throw new InvalidOperationException($"Redis journal metadata key '{metadataKey}' is malformed."); } private static string EscapeRedisPattern(string value) diff --git a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageOptions.cs b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageOptions.cs index 2b46b7947a5..b423c64033b 100644 --- a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageOptions.cs +++ b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageOptions.cs @@ -40,7 +40,11 @@ public sealed class RedisJournalStorageOptions /// /// Gets or sets the delegate used to convert a journal id to the Redis key name component. /// - public Func GetKeyName { get; set; } = static journalId => journalId.Value; + /// + /// The default mapping preserves the journal id, allowing discovery to filter keys by prefix + /// and recover ids without reading metadata. Custom mappings require metadata reads to recover journal ids. + /// + public Func GetKeyName { get; set; } = DefaultGetKeyName; /// /// Gets or sets the journal length, in bytes, at which returns . @@ -81,6 +85,10 @@ internal string GetKeyNameForJournal(JournalId journalId) ArgumentException.ThrowIfNullOrWhiteSpace(keyName); return keyName; } + + internal bool UsesDefaultKeyName => GetKeyName == DefaultGetKeyName; + + private static string DefaultGetKeyName(JournalId journalId) => journalId.Value; } internal sealed class RedactRedisConfigurationOptionsAttribute : RedactAttribute diff --git a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs index ba5eeefe466..0fb8b14e82d 100644 --- a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs +++ b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs @@ -56,17 +56,23 @@ public IJournalStorage CreateStorage(JournalId journalId) return new RedisJournalStorage(GetDatabase(), _keyPrefix, keyName, _journalFormatKey, _options, journalId); } - public async IAsyncEnumerable ListAsync( + public async IAsyncEnumerable ListAsync( ListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var prefix = options?.Prefix ?? default; cancellationToken.ThrowIfCancellationRequested(); + var range = new JournalCatalogRange(options); + if (range.IsEmpty) + { + yield break; + } + + var identityKeyMapping = _options.UsesDefaultKeyName; var connection = GetConnection(); var database = GetDatabase(); var journalIds = new HashSet(); - var batch = new RedisKey[JournalIdReadBatchSize]; - var pattern = RedisJournalStorage.GetMetadataKeyPattern(_keyPrefix); + var batch = identityKeyMapping ? [] : new RedisKey[JournalIdReadBatchSize]; + var pattern = RedisJournalStorage.GetMetadataKeyPattern(_keyPrefix, identityKeyMapping ? range.ListingPrefix : null); var scannedServer = false; foreach (var endpoint in connection.GetEndPoints()) { @@ -88,6 +94,27 @@ public async IAsyncEnumerable ListAsync( database.Database, pattern, pageSize: ScanPageSize).GetAsyncEnumerator(cancellationToken); + if (identityKeyMapping) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!await metadataKeys.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + var journalId = RedisJournalStorage.GetJournalIdFromMetadataKey(_keyPrefix, metadataKeys.Current); + if (range.Contains(journalId.Value) && journalIds.Add(journalId)) + { + yield return new JournalCatalogEntry(journalId); + } + } + + continue; + } + while (true) { var count = 0; @@ -152,10 +179,10 @@ public async IAsyncEnumerable ListAsync( $"Redis journal metadata '{batch[i]}' contains an invalid '{RedisJournalStorage.JournalIdMetadataKey}' value."); } - if (prefix.IsPrefixOf(journalId) && journalIds.Add(journalId)) + if (range.Contains(journalId.Value) && journalIds.Add(journalId)) { cancellationToken.ThrowIfCancellationRequested(); - yield return journalId; + yield return new JournalCatalogEntry(journalId); } } diff --git a/src/api/AWS/Orleans.Journaling.S3/Orleans.Journaling.S3.cs b/src/api/AWS/Orleans.Journaling.S3/Orleans.Journaling.S3.cs index b39d9ca89de..8eb5ddbd5c9 100644 --- a/src/api/AWS/Orleans.Journaling.S3/Orleans.Journaling.S3.cs +++ b/src/api/AWS/Orleans.Journaling.S3/Orleans.Journaling.S3.cs @@ -30,6 +30,8 @@ public sealed partial class S3JournalStorageOptions public System.Func GetObjectKey { get { throw null; } set { } } + public System.Func? GetObjectKeyPrefix { get { throw null; } set { } } + public int MaxMetadataOnlyConflictRetries { get { throw null; } set { } } public System.TimeSpan MetadataOnlyConflictInitialBackoff { get { throw null; } set { } } @@ -44,6 +46,8 @@ public sealed partial class S3JournalStorageOptions public bool UseConditionalDelete { get { throw null; } set { } } + public bool UseOrderedListing { get { throw null; } set { } } + public bool UseS3ExpressAppend { get { throw null; } set { } } public void ConfigureS3Client(Amazon.RegionEndpoint regionEndpoint) { } diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 0b3262c7bff..7cec335ee24 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -271,7 +271,7 @@ public partial interface IJournalStorage public partial interface IJournalStorageCatalog { - System.Collections.Generic.IAsyncEnumerable ListAsync(ListOptions? options = null, System.Threading.CancellationToken cancellationToken = default); + System.Collections.Generic.IAsyncEnumerable ListAsync(ListOptions? options = null, System.Threading.CancellationToken cancellationToken = default); } public partial interface IJournalStorageConsumer @@ -364,6 +364,38 @@ protected void WriteAt(int offset, System.ReadOnlySpan value) { } protected virtual void WritePreservedEntry(JournalStreamId streamId, IPreservedJournalEntry entry) { } } + public readonly partial struct JournalCatalogEntry : System.IEquatable + { + private readonly object _dummy; + private readonly int _dummyPrimitive; + public JournalCatalogEntry(JournalId Id, IJournalMetadata? Metadata = null) { } + + public JournalId Id { get { throw null; } init { } } + + public IJournalMetadata? Metadata { get { throw null; } init { } } + + [System.Runtime.CompilerServices.CompilerGenerated] + public readonly void Deconstruct(out JournalId Id, out IJournalMetadata? Metadata) { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public readonly bool Equals(JournalCatalogEntry other) { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public override readonly bool Equals(object obj) { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public override readonly int GetHashCode() { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public static bool operator ==(JournalCatalogEntry left, JournalCatalogEntry right) { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public static bool operator !=(JournalCatalogEntry left, JournalCatalogEntry right) { throw null; } + + [System.Runtime.CompilerServices.CompilerGenerated] + public override readonly string ToString() { throw null; } + } + public sealed partial class JournaledStateManagerOptions { public static readonly System.TimeSpan DEFAULT_RETIREMENT_GRACE_PERIOD; @@ -511,6 +543,12 @@ public readonly partial struct JournalStreamWriter public sealed partial class ListOptions { + public bool IncludeMetadata { get { throw null; } set { } } + + public JournalId MaxId { get { throw null; } set { } } + + public JournalId MinId { get { throw null; } set { } } + public JournalId Prefix { get { throw null; } set { } } } @@ -545,7 +583,7 @@ public VolatileJournalStorageProvider(Microsoft.Extensions.Options.IOptions ListAsync(ListOptions? options = null, System.Threading.CancellationToken cancellationToken = default) { throw null; } + public System.Collections.Generic.IAsyncEnumerable ListAsync(ListOptions? options = null, System.Threading.CancellationToken cancellationToken = default) { throw null; } } } diff --git a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs index 4bf0f0fce15..cd32ebf78ee 100644 --- a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs @@ -1,5 +1,7 @@ using System.Net; using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; @@ -20,12 +22,44 @@ public sealed class RedisJournalStorageCatalogTests { private const string KeyPrefix = "catalog-tests"; + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ListAsync_IncludeMetadataReturnsUnavailableWithoutFetchingMetadataHashes(bool customMapping) + { + var id = JournalId.Create("redis", "projection"); + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, customMapping ? "mapped" : id.Value); + var database = Substitute.For(); + database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey).Returns((RedisValue)id.Value); + var provider = await CreateProviderAsync( + database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanKeysAsync([key]))); + var result = new List(); + await foreach (var entry in provider.ListAsync(new() { IncludeMetadata = true }, TestContext.Current.CancellationToken)) + { + result.Add(entry); + } + + var listed = Assert.Single(result); + Assert.Equal(id, listed.Id); + Assert.Null(listed.Metadata); + await database.DidNotReceiveWithAnyArgs().HashGetAllAsync(default); + await database.DidNotReceive().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()); + if (customMapping) + { + await database.Received(1).HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey); + } + else + { + await AssertNoMetadataReadsAsync(database); + } + } + [Fact] - public async Task ListAsync_YieldsFirstBatchBeforeScanCompletesAndSnapshotsPrefix() + public async Task ListAsync_CustomMappingYieldsFirstBatchBeforeScanCompletesAndSnapshotsOptions() { var firstId = JournalId.Create("redis", "list", "z"); var secondId = JournalId.Create("redis", "list", "a"); - var nonmatchingId = JournalId.Create("redis", "listing", "other"); + var nonmatchingId = JournalId.Create("redis", "other"); var firstBatch = Enumerable.Range(0, 128) .Select(index => RedisJournalStorage.GetMetadataKey(KeyPrefix, $"mapped-{index}")) .ToArray(); @@ -44,21 +78,27 @@ public async Task ListAsync_YieldsFirstBatchBeforeScanCompletesAndSnapshotsPrefi metadataReads++; return Task.FromResult((RedisValue)ids[call.ArgAt(0)].Value); }); - var provider = await CreateProviderAsync(database, CreateServer(ScanAsync(TestContext.Current.CancellationToken))); - var options = new ListOptions { Prefix = JournalId.Create("redis", "list") }; - await using var enumerator = provider.ListAsync(options, TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); + var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanAsync(TestContext.Current.CancellationToken))); + var options = new ListOptions { Prefix = nonmatchingId, MinId = firstId, MaxId = new("a") }; + var listing = provider.ListAsync(options, TestContext.Current.CancellationToken); + options.Prefix = JournalId.Create("redis", "list"); + options.MinId = secondId; + options.MaxId = firstId; + await using var enumerator = listing.GetAsyncEnumerator(TestContext.Current.CancellationToken); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal(firstId, enumerator.Current); + Assert.Equal(firstId, enumerator.Current.Id); Assert.Equal(128, scannedKeys); Assert.Equal(128, metadataReads); Assert.False(scanCompleted); Assert.False(scanDisposed); options.Prefix = JournalId.Create("redis", "listing"); + options.MinId = firstId; + options.MaxId = new("a"); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal(secondId, enumerator.Current); + Assert.Equal(secondId, enumerator.Current.Id); Assert.Equal(130, scannedKeys); Assert.Equal(130, metadataReads); Assert.True(scanCompleted); @@ -86,23 +126,77 @@ async IAsyncEnumerable ScanAsync([EnumeratorCancellation] Cancellation } } - [Fact] - public async Task ListAsync_LaterScanFailurePropagatesAfterFirstBatch() + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ListAsync_BoundsAreInclusiveAndDoNotStopUnorderedScan(bool bounded, bool customMapping) + { + var future = JournalId.Create("redis", "list", "z"); + var earlier = JournalId.Create("redis", "list", "a"); + var maximum = JournalId.Create("redis", "list", "b"); + var before = JournalId.Create("redis", "list", "0"); + var partialMatch = JournalId.Create("redis", "listing"); + var ids = Enumerable.Repeat(future, 128).Concat([earlier, maximum, before, partialMatch, JournalId.Create("redis", "other")]).ToArray(); + var keys = ids.Select((id, index) => RedisJournalStorage.GetMetadataKey(KeyPrefix, customMapping ? $"mapped-{index}" : id.Value)).ToArray(); + var database = Substitute.For(); + for (var index = 0; index < keys.Length; index++) + { + database.HashGetAsync(keys[index], RedisJournalStorage.JournalIdMetadataKey) + .Returns(Task.FromResult((RedisValue)ids[index].Value)); + } + + var scanned = 0; + var provider = await CreateProviderAsync( + database, + customMapping ? CustomMappingOptions() : new(), + CreateServer(ScanAsync(), customMapping ? null : bounded ? "redis/list/" : "redis/list")); + var result = new List(); + await foreach (var id in provider.ListAsync( + new() { Prefix = JournalId.Create("redis", "list"), MinId = bounded ? earlier : default, MaxId = bounded ? maximum : default }, + TestContext.Current.CancellationToken)) + { + result.Add(id.Id); + } + + Assert.Equal(bounded ? [earlier, maximum] : new[] { future, earlier, maximum, before, partialMatch }, result); + Assert.Equal(133, scanned); + if (!customMapping) + { + await AssertNoMetadataReadsAsync(database); + } + + async IAsyncEnumerable ScanAsync() + { + await Task.CompletedTask; + foreach (var key in keys) + { + scanned++; + yield return key; + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ListAsync_LaterScanFailurePropagatesAfterEarlierIds(bool customMapping) { var id = JournalId.Create("redis", "failure"); - var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "mapped"); + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, customMapping ? "mapped" : id.Value); var failure = new RedisConnectionException(ConnectionFailureType.UnableToConnect, "Later scan failed."); var scanDisposed = false; var scannedKeys = 0; var database = Substitute.For(); database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey) .Returns(Task.FromResult((RedisValue)id.Value)); - var provider = await CreateProviderAsync(database, CreateServer(ScanAsync())); + var provider = await CreateProviderAsync(database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanAsync())); await using var enumerator = provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal(id, enumerator.Current); - Assert.Equal(128, scannedKeys); + Assert.Equal(id, enumerator.Current.Id); + Assert.Equal(customMapping ? 128 : 1, scannedKeys); Assert.False(scanDisposed); var exception = await Assert.ThrowsAsync(() => enumerator.MoveNextAsync().AsTask()); @@ -132,12 +226,14 @@ async IAsyncEnumerable ScanAsync() } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ListAsync_CancellationOrEarlyDisposalStopsScan(bool cancel) + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ListAsync_CancellationOrEarlyDisposalStopsScan(bool cancel, bool customMapping) { var id = JournalId.Create("redis", "cancel"); - var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "mapped"); + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, customMapping ? "mapped" : id.Value); var scannedKeys = 0; var metadataReads = 0; var scanDisposed = false; @@ -149,14 +245,14 @@ public async Task ListAsync_CancellationOrEarlyDisposalStopsScan(bool cancel) return Task.FromResult((RedisValue)id.Value); }); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - var provider = await CreateProviderAsync(database, CreateServer(ScanAsync(cancellation.Token))); + var provider = await CreateProviderAsync(database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanAsync(cancellation.Token))); await using (var enumerator = provider.ListAsync(cancellationToken: cancellation.Token).GetAsyncEnumerator(cancellation.Token)) { Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal(id, enumerator.Current); - Assert.Equal(128, scannedKeys); - Assert.Equal(128, metadataReads); + Assert.Equal(id, enumerator.Current.Id); + Assert.Equal(customMapping ? 128 : 1, scannedKeys); + Assert.Equal(customMapping ? 128 : 0, metadataReads); Assert.False(scanDisposed); if (cancel) @@ -166,8 +262,8 @@ public async Task ListAsync_CancellationOrEarlyDisposalStopsScan(bool cancel) } } - Assert.Equal(128, scannedKeys); - Assert.Equal(128, metadataReads); + Assert.Equal(customMapping ? 128 : 1, scannedKeys); + Assert.Equal(customMapping ? 128 : 0, metadataReads); Assert.True(scanDisposed); async IAsyncEnumerable ScanAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -193,7 +289,7 @@ async IAsyncEnumerable ScanAsync([EnumeratorCancellation] Cancellation public async Task ListAsync_DisconnectedLaterPrimaryPropagatesAfterEarlierIds() { var id = JournalId.Create("redis", "earlier-server"); - var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "mapped"); + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value); var database = Substitute.For(); database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey) .Returns(Task.FromResult((RedisValue)id.Value)); @@ -203,7 +299,7 @@ public async Task ListAsync_DisconnectedLaterPrimaryPropagatesAfterEarlierIds() await using var enumerator = provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal(id, enumerator.Current); + Assert.Equal(id, enumerator.Current.Id); var exception = await Assert.ThrowsAsync(() => enumerator.MoveNextAsync().AsTask()); @@ -248,7 +344,7 @@ public async Task ListAsync_CancellationDuringFinalDeletedMetadataResponsePropag cancellation.Cancel(); return Task.FromResult(RedisResult.Create(new RedisValue[] { 0 })); }); - var provider = await CreateProviderAsync(database, CreateServer(ScanAsync())); + var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanAsync())); await using var enumerator = provider.ListAsync(cancellationToken: cancellation.Token).GetAsyncEnumerator(cancellation.Token); var exception = await Assert.ThrowsAnyAsync(() => enumerator.MoveNextAsync().AsTask()); @@ -282,15 +378,333 @@ async IAsyncEnumerable ScanAsync() } } - private static IServer CreateServer(IAsyncEnumerable keys) + [Fact] + public async Task ListAsync_DefaultMappingSnapshotsOptionsAndStreamsWithoutMetadataReads() + { + var first = new JournalId("jobs/2026/09/b"); + var second = new JournalId("jobs/2026/09/a"); + var unrelated = new JournalId("jobs/other"); + var scanned = 0; + var database = Substitute.For(); + var server = CreateServer(ScanAsync(), "jobs/2026/09/"); + var provider = await CreateProviderAsync(database, server); + var options = new ListOptions { Prefix = unrelated, MinId = first, MaxId = new("a") }; + var listing = provider.ListAsync(options, TestContext.Current.CancellationToken); + options.Prefix = new("jobs/2026/0"); + options.MinId = second; + options.MaxId = first; + await using var enumerator = listing.GetAsyncEnumerator(TestContext.Current.CancellationToken); + + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(first, enumerator.Current.Id); + Assert.Equal(1, scanned); + await AssertNoMetadataReadsAsync(database); + + options.Prefix = unrelated; + options.MinId = first; + options.MaxId = new("a"); + + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(second, enumerator.Current.Id); + Assert.Equal(4, scanned); + Assert.False(await enumerator.MoveNextAsync()); + _ = server.Received(1).KeysAsync(0, "catalog-tests:journal:{*}:jobs%2F2026%2F09%2F*:metadata", pageSize: 250); + await AssertNoMetadataReadsAsync(database); + + async IAsyncEnumerable ScanAsync() + { + await Task.CompletedTask; + foreach (var id in new[] { first, first, unrelated, second }) + { + scanned++; + yield return RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value); + } + } + } + + [Fact] + public async Task ListAsync_DefaultMappingReadsAllIdsFromKeysWithoutMetadataRequests() + { + var ids = new JournalId[] { new("redis/raw:*?[\\]/%2F:雪/😀"), new("redis/other") }; + var database = Substitute.For(); + var provider = await CreateProviderAsync( + database, + CreateServer(ScanKeysAsync(ids.Concat(ids).Select(id => RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value))))); + + var result = await ReadIdsAsync(provider); + + Assert.Equal(ids, result); + await AssertNoMetadataReadsAsync(database); + } + + [Fact] + public async Task ListAsync_DefaultMappingEscapesNativePatternAndMatchesPartialSegments() { + const string keyPrefix = @"catalog[*?]\tests"; + const string prefix = @"jobs/*?[x]\%/雪"; + const string expectedPattern = @"catalog\[\*\?\]\\tests:journal:{*}:jobs%2F%2A%3F%5Bx%5D%5C%25%2F%E9%9B%AA*:metadata"; + var expected = new JournalId[] { new(prefix), new(prefix + "flake/child") }; + var unrelated = new JournalId("jobs/wildcard-not-a-literal"); + var keys = expected.Append(unrelated).Select(id => RedisJournalStorage.GetMetadataKey(keyPrefix, id.Value)); + var database = Substitute.For(); + var server = CreateServer(ScanKeysAsync(keys), prefix, keyPrefix); + var provider = await CreateProviderAsync(database, new RedisJournalStorageOptions { KeyPrefix = keyPrefix }, server); + + var result = await ReadIdsAsync(provider, new() { Prefix = new(prefix) }); + + Assert.Equal(expected, result); + Assert.Equal((RedisValue)expectedPattern, RedisJournalStorage.GetMetadataKeyPattern(keyPrefix, prefix)); + _ = server.Received(1).KeysAsync(0, expectedPattern, pageSize: 250); + await AssertNoMetadataReadsAsync(database); + } + + [Theory] + [InlineData("jobs/", "jobs/123/a", "jobs/123/z", "jobs%2F123%2F")] + [InlineData(null, "jobs/123/a", "jobs/123/z", "jobs%2F123%2F")] + [InlineData("jobs/123/alpha", "jobs/123/a", "jobs/123/z", "jobs%2F123%2Falpha")] + [InlineData(null, "jobs/a", "other/z", "")] + [InlineData(null, "jobs/a", null, "")] + [InlineData(null, null, "jobs/z", "")] + public async Task ListAsync_DefaultMappingNarrowsNativePatternUsingCommonBounds( + string? prefix, string? minId, string? maxId, string expectedEncodedPrefix) + { + var database = Substitute.For(); var server = Substitute.For(); server.IsConnected.Returns(true); - server.KeysAsync(0, RedisJournalStorage.GetMetadataKeyPattern(KeyPrefix), pageSize: 250).Returns(keys); + server.KeysAsync(0, Arg.Any(), pageSize: 250).Returns(ScanKeysAsync([])); + var provider = await CreateProviderAsync(database, server); + + var result = await ReadIdsAsync(provider, new() + { + Prefix = prefix is null ? default : new(prefix), + MinId = minId is null ? default : new(minId), + MaxId = maxId is null ? default : new(maxId), + }); + + Assert.Empty(result); + _ = server.Received(1).KeysAsync(0, $"catalog-tests:journal:{{*}}:{expectedEncodedPrefix}*:metadata", pageSize: 250); + await AssertNoMetadataReadsAsync(database); + } + + [Fact] + public async Task ListAsync_DefaultMappingBroadensNativePrefixEndingInHighSurrogate() + { + var included = new JournalId("jobs/\U0001F600"); + var excluded = new JournalId("jobs/\u96EA"); + var database = Substitute.For(); + var server = CreateServer( + ScanKeysAsync(new[] { included, excluded }.Select(id => RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value))), + "jobs/"); + var provider = await CreateProviderAsync(database, server); + + var result = await ReadIdsAsync(provider, new() { Prefix = new("jobs/\uD83D") }); + + Assert.Equal([included], result); + _ = server.Received(1).KeysAsync(0, "catalog-tests:journal:{*}:jobs%2F*:metadata", pageSize: 250); + await AssertNoMetadataReadsAsync(database); + } + + [Theory] + [InlineData("jobs/", "z", "a", false)] + [InlineData("jobs/", "z", "a", true)] + [InlineData("jobs/", "other", null, false)] + [InlineData("jobs/", "other", null, true)] + [InlineData("jobs/", null, "a", false)] + [InlineData("jobs/", null, "a", true)] + public async Task ListAsync_EmptyRangePerformsNoScanOrReads(string prefix, string? minId, string? maxId, bool customMapping) + { + var database = Substitute.For(); + var server = Substitute.For(); + server.IsConnected.Returns(true); + var provider = await CreateProviderAsync(database, customMapping ? CustomMappingOptions() : new(), server); + database.ClearReceivedCalls(); + server.ClearReceivedCalls(); + + var result = await ReadIdsAsync(provider, new() + { + Prefix = new(prefix), + MinId = minId is null ? default : new(minId), + MaxId = maxId is null ? default : new(maxId), + }); + + Assert.Empty(result); + Assert.Empty(database.ReceivedCalls()); + Assert.Empty(server.ReceivedCalls()); + } + + [Theory] + [InlineData("")] + [InlineData("bad%")] + [InlineData("journal%2fchild")] + [InlineData("journal%2Fother")] + [InlineData("journal/child")] + [InlineData("%FF")] + [InlineData("%20")] + public async Task ListAsync_DefaultMappingRejectsMalformedMetadataKeysWithoutFallback(string encodedName) + { + var metadataKey = RedisJournalStorage.GetMetadataKey(KeyPrefix, "journal/child").ToString(); + metadataKey = metadataKey[..(metadataKey.IndexOf("}:", StringComparison.Ordinal) + 2)] + encodedName + ":metadata"; + var database = Substitute.For(); + var provider = await CreateProviderAsync(database, CreateServer(ScanKeysAsync([(RedisKey)metadataKey]))); + + var exception = await Assert.ThrowsAsync(() => ReadIdsAsync(provider)); + + Assert.Contains("malformed", exception.Message); + Assert.Contains(metadataKey, exception.Message); + await AssertNoMetadataReadsAsync(database); + } + + [Fact] + public void MetadataKeyPatternRequiresReadableComponentAndKeysPreserveHashTag() + { + const string keyName = "jobs/raw:*?[\\]/%2F:雪/😀"; + var expectedHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyName))); + const string encodedName = "jobs%2Fraw%3A%2A%3F%5B%5C%5D%2F%252F%3A%E9%9B%AA%2F%F0%9F%98%80"; + var metadataKey = RedisJournalStorage.GetMetadataKey(KeyPrefix, keyName); + var dataKey = RedisJournalStorage.GetDataKey(KeyPrefix, keyName); + + Assert.Equal($"{KeyPrefix}:journal:{{{expectedHash}}}:{encodedName}:metadata", metadataKey.ToString()); + Assert.Equal($"{KeyPrefix}:journal:{{{expectedHash}}}:{encodedName}:data", dataKey.ToString()); + Assert.Equal(new JournalId(keyName), RedisJournalStorage.GetJournalIdFromMetadataKey(KeyPrefix, metadataKey)); + Assert.Equal((RedisValue)"catalog-tests:journal:{*}:*:metadata", RedisJournalStorage.GetMetadataKeyPattern(KeyPrefix)); + var oldKey = (RedisKey)$"{KeyPrefix}:journal:{{{expectedHash}}}:metadata"; + Assert.Throws(() => RedisJournalStorage.GetJournalIdFromMetadataKey(KeyPrefix, oldKey)); + } + + [Fact] + public async Task ListAsync_CustomMappingReadsBatchesConcurrently() + { + var ids = Enumerable.Range(0, 128).Select(index => new JournalId($"jobs/{index:D3}")).ToArray(); + var keys = ids.Select((_, index) => RedisJournalStorage.GetMetadataKey(KeyPrefix, $"opaque-{index}")).ToArray(); + var completions = ids.Select(_ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var reads = 0; + var database = Substitute.For(); + database.HashGetAsync(Arg.Any(), RedisJournalStorage.JournalIdMetadataKey) + .Returns(call => + { + var index = Array.IndexOf(keys, call.ArgAt(0)); + reads++; + return completions[index].Task; + }); + var server = CreateServer(ScanKeysAsync(keys)); + var provider = await CreateProviderAsync(database, CustomMappingOptions(), server); + await using var enumerator = provider.ListAsync( + new() { Prefix = new("jobs/"), MinId = ids[0], MaxId = ids[^1] }, + TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); + var moveNext = enumerator.MoveNextAsync().AsTask(); + var startedReads = reads; + var completedBeforeResponses = moveNext.IsCompleted; + + for (var i = 0; i < completions.Length; i++) + { + completions[i].SetResult(ids[i].Value); + } + + Assert.True(await moveNext); + Assert.Equal(128, startedReads); + Assert.False(completedBeforeResponses); + Assert.Equal(ids[0], enumerator.Current.Id); + var result = new List { enumerator.Current.Id }; + while (await enumerator.MoveNextAsync()) + { + result.Add(enumerator.Current.Id); + } + + Assert.Equal(ids, result); + _ = server.Received(1).KeysAsync(0, "catalog-tests:journal:{*}:*:metadata", pageSize: 250); + } + + [Fact] + public async Task ListAsync_CustomMappingMetadataReadFailurePropagates() + { + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "opaque"); + var failure = new RedisConnectionException(ConnectionFailureType.UnableToConnect, "Metadata read failed."); + var database = Substitute.For(); + database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey) + .Returns(Task.FromException(failure)); + var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanKeysAsync([key]))); + + var exception = await Assert.ThrowsAsync(() => ReadIdsAsync(provider)); + + Assert.Same(failure, exception); + await database.Received(1).HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey); + await database.DidNotReceive().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Theory] + [InlineData(0, null)] + [InlineData(1, "jobs/recovered")] + [InlineData(1, " ")] + [InlineData(-1, null)] + public async Task ListAsync_CustomMappingMissingMetadataChecksDeletionOrReportsCorruption(int status, string? value) + { + var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "opaque"); + var database = Substitute.For(); + database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey).Returns(Task.FromResult(RedisValue.Null)); + var response = status == 1 ? new RedisValue[] { status, value } : [status]; + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(RedisResult.Create(response))); + var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanKeysAsync([key]))); + + if (status < 0 || string.IsNullOrWhiteSpace(value) && status == 1) + { + var exception = await Assert.ThrowsAsync(() => ReadIdsAsync(provider)); + Assert.Contains(RedisJournalStorage.JournalIdMetadataKey, exception.Message); + } + else + { + var result = await ReadIdsAsync(provider); + if (status == 0) + { + Assert.Empty(result); + } + else + { + Assert.NotNull(value); + Assert.Equal([new JournalId(value)], result); + } + } + + await database.Received(1).HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey); + await database.Received(1).ScriptEvaluateAsync( + Arg.Any(), Arg.Is(keys => keys.Length == 1 && keys[0] == key), Arg.Any()); + } + + private static async IAsyncEnumerable ScanKeysAsync(IEnumerable keys) + { + await Task.CompletedTask; + foreach (var key in keys) + { + yield return key; + } + } + + private static async Task> ReadIdsAsync(RedisJournalStorageProvider provider, ListOptions? options = null) + { + var result = new List(); + await foreach (var id in provider.ListAsync(options, TestContext.Current.CancellationToken)) + { + result.Add(id.Id); + } + + return result; + } + + private static IServer CreateServer(IAsyncEnumerable keys, string? journalIdPrefix = null, string keyPrefix = KeyPrefix) + { + var server = Substitute.For(); + server.IsConnected.Returns(true); + server.KeysAsync(0, RedisJournalStorage.GetMetadataKeyPattern(keyPrefix, journalIdPrefix), pageSize: 250).Returns(keys); return server; } private static async Task CreateProviderAsync(IDatabase database, params IServer[] servers) + => await CreateProviderAsync(database, new RedisJournalStorageOptions(), servers); + + private static async Task CreateProviderAsync( + IDatabase database, + RedisJournalStorageOptions options, + params IServer[] servers) { var connection = Substitute.For(); connection.GetDatabase().Returns(database); @@ -301,12 +715,10 @@ private static async Task CreateProviderAsync(IData connection.GetServer(endpoints[i]).Returns(servers[i]); } + options.KeyPrefix ??= KeyPrefix; + options.CreateMultiplexer = _ => Task.FromResult((connection, true)); var provider = new RedisJournalStorageProvider( - Options.Create(new RedisJournalStorageOptions - { - KeyPrefix = KeyPrefix, - CreateMultiplexer = _ => Task.FromResult((connection, true)), - }), + Options.Create(options), Options.Create(new ClusterOptions { ServiceId = "catalog-tests" }), Options.Create(new JournaledStateManagerOptions())); var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); @@ -314,4 +726,13 @@ private static async Task CreateProviderAsync(IData await lifecycle.OnStart(TestContext.Current.CancellationToken); return provider; } + + private static RedisJournalStorageOptions CustomMappingOptions() + => new() { GetKeyName = static id => $"opaque-{id.Value}" }; + + private static async Task AssertNoMetadataReadsAsync(IDatabase database) + { + await database.DidNotReceive().HashGetAsync(Arg.Any(), Arg.Any()); + await database.DidNotReceive().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } } diff --git a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs index b905c5e0f72..61acc384d53 100644 --- a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs @@ -523,14 +523,14 @@ private static ReadOnlySequence CreateSequence(byte[] first, byte[] second return new(firstSegment, 0, lastSegment, lastSegment.Memory.Length); } - private static async Task> ToListAsync( - IAsyncEnumerable source, + private static async Task> ToListAsync( + IAsyncEnumerable source, CancellationToken cancellationToken) { - var result = new List(); + var result = new List(); await foreach (var item in source.WithCancellation(cancellationToken)) { - result.Add(item); + result.Add(item.Id); } return result; diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs index 39e4c10ca79..a795387f4ef 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs @@ -645,7 +645,7 @@ public void AllowAppends() public IJournalStorage CreateStorage(JournalId journalId) => new CountingJournalStorage(this, _inner.CreateStorage(journalId)); - public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) => _inner.ListAsync(options, cancellationToken); private async ValueTask OnAppendAsync(CancellationToken cancellationToken) diff --git a/test/Orleans.Journaling.Tests/AzureBlobJournalStorageTests.cs b/test/Orleans.Journaling.Tests/AzureBlobJournalStorageTests.cs index 3caf0d1f262..da258da74e3 100644 --- a/test/Orleans.Journaling.Tests/AzureBlobJournalStorageTests.cs +++ b/test/Orleans.Journaling.Tests/AzureBlobJournalStorageTests.cs @@ -162,12 +162,58 @@ public async Task AppendAsync_WhenMimeTypeConfigured_SetsBlobContentType() } [Fact] - public void DefaultWalAndCheckpointNames_UseFixedWalAndSnapshotPrefix() + public void DefaultWalAndCheckpointNames_UseSeparatePrefixes() { var journalId = new JournalId("journals/test"); - Assert.Equal("journals/test/wal", AzureBlobJournalStorageOptions.GetDefaultWalBlobName(journalId)); - Assert.Equal("journals/test/chk.snapshot", AzureBlobJournalStorageOptions.GetDefaultCheckpointBlobName(journalId, "snapshot")); + var options = new AzureBlobJournalStorageOptions(); + Assert.Equal("wal/journals/test", options.GetWalBlobNameForJournal(journalId)); + Assert.Equal("checkpoints/journals/test/snapshot", options.GetCheckpointBlobNameForJournal(journalId, "snapshot")); + } + + [Fact] + public async Task DefaultLayout_ReplaceReplayAndDeleteFollowPublishedCheckpointNames() + { + var appendBlobs = new FakeAppendBlobStore(); + var checkpoints = new FakeBlockBlobStore(); + var options = new AzureBlobJournalStorageOptions(); + var walName = options.GetWalBlobNameForJournal(TestJournalId); + AzureBlobJournalStorage Create() => CreateStorage( + appendBlobs, + checkpoints, + journalFormatKey: "json-lines", + walBlobName: walName, + getCheckpointName: snapshotId => options.GetCheckpointBlobNameForJournal(TestJournalId, snapshotId)); + + var cancellationToken = TestContext.Current.CancellationToken; + var storage = Create(); + await storage.AppendAsync(new ReadOnlySequence([1]), cancellationToken); + await storage.ReplaceAsync(new ReadOnlySequence([2]), cancellationToken); + var firstCheckpoint = Assert.Single(checkpoints.UploadCalls).Name; + Assert.StartsWith($"checkpoints/{TestJournalId.Value}/", firstCheckpoint); + Assert.Equal(firstCheckpoint, appendBlobs.CreateCalls[^1].Metadata[AzureBlobJournalStorage.CheckpointMetadataKey]); + Assert.Equal("wal/" + TestJournalId.Value, appendBlobs.CreateCalls[^1].Name); + Assert.Equal(new ETag("\"append-1\""), appendBlobs.CreateCalls[^1].IfMatch); + + await storage.ReplaceAsync(new ReadOnlySequence([3]), cancellationToken); + var secondCheckpoint = checkpoints.UploadCalls[^1].Name; + Assert.NotEqual(firstCheckpoint, secondCheckpoint); + Assert.StartsWith($"checkpoints/{TestJournalId.Value}/", secondCheckpoint); + Assert.False(checkpoints.Exists(firstCheckpoint)); + Assert.True(checkpoints.Exists(secondCheckpoint)); + Assert.Equal(secondCheckpoint, appendBlobs.CreateCalls[^1].Metadata[AzureBlobJournalStorage.CheckpointMetadataKey]); + await storage.AppendAsync(new ReadOnlySequence([4]), cancellationToken); + + var recovered = Create(); + var consumer = new CapturingJournalStorageConsumer(); + await recovered.ReadAsync(consumer, cancellationToken); + Assert.Equal([3, 4], consumer.Bytes.ToArray()); + Assert.Equal(secondCheckpoint, Assert.Single(checkpoints.DownloadCalls).Name); + + await recovered.DeleteAsync(cancellationToken); + Assert.False(appendBlobs.Exists(walName)); + Assert.False(checkpoints.Exists(secondCheckpoint)); + Assert.Equal(2, checkpoints.DeleteCalls.Count); } [Fact] diff --git a/test/Orleans.Journaling.Tests/AzureTableJournalStorageOptionsTests.cs b/test/Orleans.Journaling.Tests/AzureTableJournalStorageOptionsTests.cs index 3fb89615961..9c4366b66e1 100644 --- a/test/Orleans.Journaling.Tests/AzureTableJournalStorageOptionsTests.cs +++ b/test/Orleans.Journaling.Tests/AzureTableJournalStorageOptionsTests.cs @@ -23,7 +23,8 @@ public void Constructor_UsesDocumentedDefaults() Assert.Equal(AzureTableJournalStorageOptions.DEFAULT_TABLE_NAME, options.TableName); Assert.Equal("journal", options.TableName); - Assert.Equal("tenant%2Fjournal", options.GetPartitionKey(journalId)); + Assert.Equal("74656E616E742F6A6F75726E616C", options.GetPartitionKey(journalId)); + Assert.True(options.UsesDefaultPartitionKey); Assert.Null(options.ClientOptions); Assert.Null(options.TableServiceClient); Assert.True(options.DeleteOldGenerations); @@ -41,21 +42,86 @@ public void Constructor_UsesDocumentedDefaults() } [Theory] - [InlineData("simple-._~09AZaz", "simple-._~09AZaz")] - [InlineData("parent/child", "parent%2Fchild")] - [InlineData("slash\\hash#question?", "slash%5Chash%23question%3F")] - [InlineData("control\0\u0001\u001F\u007F", "control%00%01%1F%7F")] - [InlineData("space + percent%", "space%20%2B%20percent%25")] - [InlineData("café/😀", "caf%C3%A9%2F%F0%9F%98%80")] - public void GetDefaultPartitionKey_PercentEncodesJournalIdValue(string value, string expected) + [InlineData("-._~09AZaz", "2D2E5F7E3039415A617A")] + [InlineData("a/b", "612F62")] + [InlineData("\\#?", "5C233F")] + [InlineData(" +%", "202B25")] + public void GetDefaultPartitionKey_EncodesPrintableAsciiBytesAsUppercaseHex(string value, string expected) { var partitionKey = AzureTableJournalStorageOptions.GetDefaultPartitionKey(new JournalId(value)); Assert.Equal(expected, partitionKey); - Assert.Equal(value, Uri.UnescapeDataString(partitionKey)); + Assert.Equal(value.Length * 2, partitionKey.Length); Assert.DoesNotContain(partitionKey, static character => character is '/' or '\\' or '#' or '?' or '\0'); } + [Fact] + public void GetDefaultPartitionKey_PreservesOrdinalOrderingAndRawPrefixesForPrintableAscii() + { + var values = new[] { "a" }.Concat(Enumerable.Range(' ', '~' - ' ' + 1).Select(value => "a" + (char)value)).ToArray(); + var options = new AzureTableJournalStorageOptions(); + foreach (var left in values) + { + var leftKey = options.GetPartitionKey(new JournalId(left)); + foreach (var right in values) + { + var rightKey = options.GetPartitionKey(new JournalId(right)); + Assert.Equal( + Math.Sign(string.CompareOrdinal(left, right)), + Math.Sign(string.CompareOrdinal(leftKey, rightKey))); + Assert.Equal( + right.StartsWith(left, StringComparison.Ordinal), + rightKey.StartsWith(leftKey, StringComparison.Ordinal)); + } + } + + Assert.Equal("6120", options.GetPartitionKey(new JournalId("a "))); + Assert.Equal("617E", options.GetPartitionKey(new JournalId("a~"))); + } + + [Theory] + [InlineData(0x00)] + [InlineData(0x1F)] + [InlineData(0x7F)] + [InlineData(0x85)] + [InlineData(0xA0)] + [InlineData(0xE9)] + [InlineData(0xD800)] + [InlineData(0xDFFF)] + [InlineData(0xFFFF)] + public void GetDefaultPartitionKey_UnsupportedCharacter_ThrowsExplicitly(int character) + { + var exception = Assert.Throws(() => + new AzureTableJournalStorageOptions().GetPartitionKeyForJournal(new JournalId("journal/" + (char)character))); + + Assert.Equal("journalId", exception.ParamName); + Assert.Contains("printable ASCII", exception.Message); + } + + [Theory] + [InlineData(0x00)] + [InlineData(0x7F)] + [InlineData(0xE9)] + [InlineData(0xD800)] + [InlineData(0xDFFF)] + [InlineData(0xFFFF)] + public void GetPartitionKeyForJournal_CustomMapper_AcceptsUnsupportedAndLongJournalIds(int character) + { + var journalId = new JournalId(new string('a', 512) + (char)character); + JournalId mappedId = default; + var options = new AzureTableJournalStorageOptions + { + GetPartitionKey = id => + { + mappedId = id; + return "custom"; + }, + }; + + Assert.Equal("custom", options.GetPartitionKeyForJournal(journalId)); + Assert.Equal(journalId, mappedId); + } + [Fact] public void GetDefaultPartitionKey_DefaultJournalId_Throws() { @@ -87,6 +153,7 @@ public void GetPartitionKeyForJournal_CustomMapper_ReceivesJournalIdAndReturnsMa Assert.Equal("partition::tenant%2Fjournal", partitionKey); Assert.Equal(expectedJournalId, receivedJournalId); Assert.Equal(1, invocationCount); + Assert.False(options.UsesDefaultPartitionKey); } [Fact] @@ -156,7 +223,7 @@ public void GetPartitionKeyForJournal_CustomMapperReturnsInvalidAzureKey_Throws( [Fact] public void GetDefaultPartitionKey_EncodedValueExceedsAzureLimit_ThrowsLocally() { - var journalId = new JournalId(new string('/', 342)); + var journalId = new JournalId(new string('a', 513)); var exception = Assert.Throws( () => AzureTableJournalStorageOptions.GetDefaultPartitionKey(journalId)); @@ -165,6 +232,16 @@ public void GetDefaultPartitionKey_EncodedValueExceedsAzureLimit_ThrowsLocally() Assert.Contains("1,024", exception.Message); } + [Fact] + public void GetDefaultPartitionKey_512PrintableAsciiCharacters_FitsAzureLimit() + { + var journalId = new JournalId(new string('~', 512)); + + var partitionKey = new AzureTableJournalStorageOptions().GetPartitionKeyForJournal(journalId); + + Assert.Equal(string.Concat(Enumerable.Repeat("7E", 512)), partitionKey); + } + [Fact] public async Task ConfigureTableServiceClient_ConnectionString_CreatesClientForConfiguredEndpoint() { diff --git a/test/Orleans.Journaling.Tests/AzureTableJournalStorageProviderTests.cs b/test/Orleans.Journaling.Tests/AzureTableJournalStorageProviderTests.cs index 0d84b34330c..37ffd0df8db 100644 --- a/test/Orleans.Journaling.Tests/AzureTableJournalStorageProviderTests.cs +++ b/test/Orleans.Journaling.Tests/AzureTableJournalStorageProviderTests.cs @@ -1,3 +1,5 @@ +using System.Text; +using System.Text.RegularExpressions; using Azure; using Azure.Core; using Azure.Data.Tables; @@ -28,8 +30,43 @@ public void CreateStorage_DefaultJournalId_Throws() Assert.StartsWith("The journal id must not be the default value.", exception.Message); } + [Theory] + [InlineData(0x00)] + [InlineData(0x1F)] + [InlineData(0x7F)] + [InlineData(0xE9)] + [InlineData(0xD800)] + [InlineData(0xDFFF)] + [InlineData(0xFFFF)] + public void CreateStorage_DefaultMapping_UnsupportedJournalIdThrowsBeforeClientAccess(int character) + { + var options = new AzureTableJournalStorageOptions(); + options.ConfigureTableServiceClient(_ => throw new InvalidOperationException("Client access is not expected.")); + using var context = CreateProvider(options); + + var exception = Assert.Throws(() => + context.Provider.CreateStorage(new JournalId("journal/" + (char)character))); + + Assert.Equal("journalId", exception.ParamName); + Assert.Contains("printable ASCII", exception.Message); + } + [Fact] - public async Task CreateStorage_UsesConfiguredPartitionMapping() + public void CreateStorage_DefaultMapping_OversizedJournalIdThrowsBeforeClientAccess() + { + using var context = CreateProvider(); + + var exception = Assert.Throws(() => + context.Provider.CreateStorage(new JournalId(new string('a', 513)))); + + Assert.Equal("journalId", exception.ParamName); + Assert.Contains("1,024", exception.Message); + } + + [Theory] + [InlineData("orders/42", "tenant!orders%2F42")] + [InlineData("café/😀\0", "tenant!caf%C3%A9%2F%F0%9F%98%80%00")] + public async Task CreateStorage_UsesConfiguredPartitionMapping(string value, string expectedPartitionKey) { var table = new FakeTableClient(); var mappedJournalId = default(JournalId); @@ -41,7 +78,7 @@ public async Task CreateStorage_UsesConfiguredPartitionMapping() }; using var context = CreateProvider(options); await StartAsync(context.Provider, TestContext.Current.CancellationToken); - var journalId = new JournalId("orders/42"); + var journalId = new JournalId(value); var created = await context.Provider.CreateStorage(journalId) .CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken); @@ -49,7 +86,7 @@ public async Task CreateStorage_UsesConfiguredPartitionMapping() Assert.True(created); Assert.Equal(journalId, mappedJournalId); var added = Assert.Single(table.AddedEntities); - Assert.Equal("tenant!orders%2F42", added.PartitionKey); + Assert.Equal(expectedPartitionKey, added.PartitionKey); Assert.Equal(AzureTableJournalStorage.HeaderRowKey, added.RowKey); Assert.Equal(journalId.Value, added[AzureTableJournalStorage.JournalIdPropertyName]); } @@ -299,19 +336,20 @@ public async Task ListAsync_MultipleHeaders_ReturnsProviderTraversalOrder() context.Provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([last, lower, upper], result); + Assert.Equal([last, lower, upper], result.Select(entry => entry.Id)); Assert.Equal(3, result.Count); } [Fact] - public async Task ListAsync_WithPrefix_ReturnsExactIdAndDescendantsOnly() + public async Task ListAsync_WithPrefix_ReturnsExactIdDescendantsAndPartialSegmentMatches() { var table = new FakeTableClient(); var prefix = new JournalId("tenant/orders"); var exact = prefix; var child = new JournalId("tenant/orders/2026"); + var partialSegment = new JournalId("tenant/orders-archive"); table.AddHeader(new JournalId("tenant/order")); - table.AddHeader(new JournalId("tenant/orders-archive")); + table.AddHeader(partialSegment); table.AddHeader(new JournalId("tenant/payments")); table.AddHeader(child); table.AddHeader(exact); @@ -321,11 +359,295 @@ public async Task ListAsync_WithPrefix_ReturnsExactIdAndDescendantsOnly() context.Provider.ListAsync(new() { Prefix = prefix }, TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([child, exact], result); + Assert.Equal([partialSegment, child, exact], result.Select(entry => entry.Id)); var query = Assert.Single(table.QueryCalls); + var prefixKey = AzureTableJournalStorageOptions.EncodePartitionKey(prefix.Value); Assert.Equal( - TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"), + TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}") + + TableClient.CreateQueryFilter($" and PartitionKey ge {prefixKey}") + + TableClient.CreateQueryFilter($" and PartitionKey lt {prefixKey + "G"}"), query.Filter); + Assert.Equal(3, query.ReturnedCount); + Assert.DoesNotContain("JournalId", query.Filter); + Assert.DoesNotContain(" or ", query.Filter); + } + + [Theory] + [InlineData(null, "jobs/2026-09-03", "jobs/2026-09-05", "jobs/2026-09-03", "jobs/2026-09-05")] + [InlineData("jobs/2026-09", "jobs/2026-09-03", "jobs/2026-09-05", "jobs/2026-09-03", "jobs/2026-09-05")] + [InlineData("jobs/2026-09", "jobs/2026-08-01", "jobs/2026-09-03", "jobs/2026-09-02", "jobs/2026-09-03")] + [InlineData("jobs/2026-09", "jobs/2026-09-03", "jobs/2026-09-03", "jobs/2026-09-03", null)] + [InlineData(null, "jobs/2026-09-06", null, "jobs/2026-09-06", "jobs/2026-10-01")] + [InlineData(null, null, "jobs/2026-09-02", "jobs/2026-08-01", "jobs/2026-09-02")] + public async Task ListAsync_RangeBounds_PruneEarlierAndFutureRowsOnServer( + string? prefix, + string? minId, + string? maxId, + string firstExpected, + string? secondExpected) + { + var table = new FakeTableClient(); + string[] values = + [ + "jobs/2026-08-01", + "jobs/2026-09-02", + "jobs/2026-09-03", + "jobs/2026-09-05", + "jobs/2026-09-06", + "jobs/2026-10-01", + ]; + foreach (var value in values) + { + table.AddHeader(new JournalId(value)); + } + + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + var options = new ListOptions + { + Prefix = prefix is null ? default : new(prefix), + MinId = minId is null ? default : new(minId), + MaxId = maxId is null ? default : new(maxId), + }; + + var result = await ToListAsync( + context.Provider.ListAsync(options, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + var expected = secondExpected is null ? new[] { firstExpected } : new[] { firstExpected, secondExpected }; + Assert.Equal(expected, result.Select(entry => entry.Id.Value)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(expected.Length, query.ReturnedCount); + Assert.Equal(1000, query.MaxPerPage); + Assert.DoesNotContain("JournalId", query.Filter); + Assert.DoesNotContain(" or ", query.Filter); + if (minId is not null) + { + var lowerBound = prefix is not null && string.CompareOrdinal(prefix, minId) > 0 ? prefix : minId; + var minKey = AzureTableJournalStorageOptions.EncodePartitionKey(lowerBound); + Assert.Contains(TableClient.CreateQueryFilter($"PartitionKey ge {minKey}"), query.Filter); + } + + if (maxId is not null) + { + var maxKey = AzureTableJournalStorageOptions.EncodePartitionKey(maxId); + Assert.Contains(TableClient.CreateQueryFilter($"PartitionKey le {maxKey}"), query.Filter); + } + } + + [Theory] + [InlineData(null, "b", "a")] + [InlineData("orders", "payments", null)] + [InlineData("orders", null, "order")] + [InlineData("orders", "ordert", "ordert")] + public async Task ListAsync_EmptyIntersection_DoesNotQuery(string? prefix, string? minId, string? maxId) + { + var table = new FakeTableClient(); + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + var options = new ListOptions + { + Prefix = prefix is null ? default : new(prefix), + MinId = minId is null ? default : new(minId), + MaxId = maxId is null ? default : new(maxId), + }; + + var result = await ToListAsync( + context.Provider.ListAsync(options, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Empty(result); + Assert.Empty(table.QueryCalls); + } + + [Theory] + [InlineData(0x00)] + [InlineData(0x1F)] + [InlineData(0x7F)] + [InlineData(0xE9)] + [InlineData(0xD800)] + [InlineData(0xDFFF)] + [InlineData(0xFFFF)] + public async Task ListAsync_UnsupportedBounds_ProjectToIndexedAsciiRange(int character) + { + var table = new FakeTableClient(); + string[] values = + [ + "a", + "range", + "range/ ", + "range/", + "range/'quoted'", + "range/child", + "range/~", + "range/~ ", + "range/~~", + "range0", + "~", + "~~", + ]; + foreach (var value in values) + { + table.AddHeader(new JournalId(value)); + } + + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + foreach (var prefix in new[] { string.Empty, "range/", "range/~" }) + { + foreach (var suffix in new[] { string.Empty, "/ignored\uFFFF" }) + { + var bound = prefix + (char)character + suffix; + var lower = await ToListAsync( + context.Provider.ListAsync(new() { MinId = new(bound) }, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + var upper = await ToListAsync( + context.Provider.ListAsync(new() { MaxId = new(bound) }, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + var expectedLower = values.Where(value => string.CompareOrdinal(value, bound) >= 0).ToArray(); + var expectedUpper = values.Where(value => string.CompareOrdinal(value, bound) <= 0).ToArray(); + Assert.Equal(expectedLower, lower.Select(entry => entry.Id.Value)); + Assert.Equal(expectedUpper, upper.Select(entry => entry.Id.Value)); + var lowerQuery = table.QueryCalls[^2]; + var upperQuery = table.QueryCalls[^1]; + Assert.Equal(expectedLower.Length, lowerQuery.ReturnedCount); + Assert.Equal(expectedUpper.Length, upperQuery.ReturnedCount); + var boundaryKey = AzureTableJournalStorageOptions.EncodePartitionKey(prefix) + + (character < ' ' ? "1F" : "7F"); + var headerFilter = TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"); + Assert.Equal( + headerFilter + TableClient.CreateQueryFilter($" and PartitionKey ge {boundaryKey}"), + lowerQuery.Filter); + Assert.Equal( + headerFilter + TableClient.CreateQueryFilter($" and PartitionKey le {boundaryKey}"), + upperQuery.Filter); + } + } + } + + [Theory] + [InlineData(0x00)] + [InlineData(0x1F)] + [InlineData(0x7F)] + [InlineData(0xE9)] + [InlineData(0xD800)] + [InlineData(0xDFFF)] + [InlineData(0xFFFF)] + public async Task ListAsync_UnsupportedPrefix_ReturnsEmptyWithoutQuery(int character) + { + var table = new FakeTableClient(); + table.AddHeader(new("range/")); + table.AddHeader(new("range/child")); + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync( + new() { Prefix = new("range/" + (char)character) }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Empty(result); + Assert.Empty(table.QueryCalls); + } + + [Theory] + [InlineData(0x00, 1)] + [InlineData(0x1F, 1)] + [InlineData(0x7F, 4)] + [InlineData(0xE9, 4)] + [InlineData(0xD800, 4)] + [InlineData(0xDFFF, 4)] + [InlineData(0xFFFF, 4)] + public async Task ListAsync_PrefixWithUnsupportedBounds_DistinguishesExactIdFromDescendants(int character, int expectedUpperCount) + { + var table = new FakeTableClient(); + string[] values = ["range/", "range/ ", "range/child", "range/~child"]; + table.AddHeader(new("range")); + foreach (var value in values) + { + table.AddHeader(new(value)); + } + + table.AddHeader(new("range0")); + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + + var bound = new JournalId("range/" + (char)character + "/ignored"); + var upper = await ToListAsync( + context.Provider.ListAsync( + new() { Prefix = new("range/"), MaxId = bound }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + var lower = await ToListAsync( + context.Provider.ListAsync( + new() { Prefix = new("range/"), MinId = bound }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal(values[..expectedUpperCount], upper.Select(entry => entry.Id.Value)); + Assert.Equal(values[expectedUpperCount..], lower.Select(entry => entry.Id.Value)); + Assert.Equal([expectedUpperCount, values.Length - expectedUpperCount], table.QueryCalls.Select(query => query.ReturnedCount)); + } + + [Fact] + public async Task ListAsync_BoundsBeyondStoredIdLengthLimit_DoNotThrowOrExcludeShorterIds() + { + var table = new FakeTableClient(); + var longest = new JournalId(new string('a', 512)); + table.AddHeader(longest); + table.AddHeader(new("b")); + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync( + new() { Prefix = new("a"), MaxId = new(longest.Value + "a") }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal([longest], result.Select(entry => entry.Id)); + Assert.Equal(1, Assert.Single(table.QueryCalls).ReturnedCount); + } + + [Fact] + public async Task ListAsync_DefaultMapping_RetainsExactLocalRangeFilter() + { + var table = new FakeTableClient(); + var included = new JournalId("range/a"); + table.AddHeader(included); + table.AddHeader(new("outside"), AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("range/b"))); + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync( + new() { Prefix = new("range/"), MinId = included, MaxId = new("range/\uFFFF") }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal([included], result.Select(entry => entry.Id)); + Assert.Equal(2, Assert.Single(table.QueryCalls).ReturnedCount); + } + + [Fact] + public async Task ListAsync_OptionsAreSnapshottedAtEnumerationStart() + { + var table = new FakeTableClient(); + table.AddHeader(new JournalId("orders/1")); + table.AddHeader(new JournalId("orders/2")); + table.AddHeader(new JournalId("orders/3")); + var options = new ListOptions { Prefix = new("unused"), MinId = new("z"), MaxId = new("z") }; + using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + var listing = context.Provider.ListAsync(options, TestContext.Current.CancellationToken); + options.Prefix = new("orders"); + options.MinId = new("orders/2"); + options.MaxId = new("orders/3"); + table.AfterQuery = () => + { + options.Prefix = new("other"); + options.MinId = new("z"); + options.MaxId = new("a"); + }; + + var result = await ToListAsync(listing, TestContext.Current.CancellationToken); + + Assert.Equal(["orders/2", "orders/3"], result.Select(entry => entry.Id.Value)); + Assert.Equal(2, Assert.Single(table.QueryCalls).ReturnedCount); } [Fact] @@ -343,8 +665,8 @@ public async Task ListAsync_DeletedHeader_IsNotReturned() context.Provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([retained], result); - Assert.DoesNotContain(deleted, result); + Assert.Equal([retained], result.Select(entry => entry.Id)); + Assert.DoesNotContain(result, entry => entry.Id == deleted); } [Fact] @@ -357,22 +679,32 @@ public async Task ListAsync_OrphanDataRowsAndMalformedHeader_AreIgnored() table.AddEntity( AzureTableJournalStorageOptions.GetDefaultPartitionKey(orphan), "g00000000000000000001-r0000000000"); - table.AddEntity("%20", AzureTableJournalStorage.HeaderRowKey); + table.AddEntity( + AzureTableJournalStorageOptions.EncodePartitionKey("missing-id"), + AzureTableJournalStorage.HeaderRowKey); + table.AddEntity( + AzureTableJournalStorageOptions.EncodePartitionKey("invalid-id"), + AzureTableJournalStorage.HeaderRowKey, + new Dictionary { [AzureTableJournalStorage.JournalIdPropertyName] = " " }); + table.AddEntity( + AzureTableJournalStorageOptions.EncodePartitionKey("wrong-type"), + AzureTableJournalStorage.HeaderRowKey, + new Dictionary { [AzureTableJournalStorage.JournalIdPropertyName] = 42 }); using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); var result = await ToListAsync( context.Provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([valid], result); - Assert.DoesNotContain(orphan, result); + Assert.Equal([valid], result.Select(entry => entry.Id)); + Assert.DoesNotContain(result, entry => entry.Id == orphan); } [Fact] public async Task ListAsync_EscapedJournalId_RoundTripsExactValue() { var table = new FakeTableClient(); - var escaped = new JournalId("tenant/slash\\hash#query?control\u0001"); + var escaped = new JournalId("tenant/slash\\hash#query?' +%~"); table.AddHeader(escaped); using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); @@ -381,48 +713,201 @@ public async Task ListAsync_EscapedJournalId_RoundTripsExactValue() TestContext.Current.CancellationToken); var listed = Assert.Single(result); - Assert.Equal(escaped, listed); + Assert.Equal(escaped, listed.Id); Assert.Equal( - "tenant%2Fslash%5Chash%23query%3Fcontrol%01", + AzureTableJournalStorageOptions.GetDefaultPartitionKey(escaped), table.Entities.Single().PartitionKey); } [Fact] - public async Task ListAsync_CustomPartitionMapping_UsesCanonicalJournalIdAndAppliesPrefix() + public async Task ListAsync_CustomPartitionMapping_UsesOnlyCanonicalJournalIdFilters() { var table = new FakeTableClient(); var included = new JournalId("tenant/orders/42"); var excluded = new JournalId("other/orders/42"); - table.AddHeader(included, "hash-a"); + table.AddHeader(included, "hash-z"); table.AddHeader(excluded, "hash-b"); + table.AddHeader(new JournalId("tenant/orders/41"), "hash-y"); + table.AddHeader(new JournalId("tenant/orders/43"), "hash-a"); var options = CreateOptions(table); - options.GetPartitionKey = static journalId => journalId.Value == "tenant/orders/42" ? "hash-a" : "hash-b"; + options.GetPartitionKey = static _ => throw new InvalidOperationException("Listing must not invoke custom partition mapping."); using var context = CreateProvider(options); await StartAsync(context.Provider, TestContext.Current.CancellationToken); var result = await ToListAsync( - context.Provider.ListAsync(new() { Prefix = new JournalId("tenant/orders") }, TestContext.Current.CancellationToken), + context.Provider.ListAsync( + new() { Prefix = new("tenant/order"), MinId = included, MaxId = included }, + TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([included], result); + Assert.Equal([included], result.Select(entry => entry.Id)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(1, query.ReturnedCount); Assert.Equal( - TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"), - Assert.Single(table.QueryCalls).Filter); + TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}") + + " and JournalId ge 'tenant/orders/42' and JournalId le 'tenant/orders/42'", + query.Filter); + Assert.DoesNotContain("PartitionKey", query.Filter); + Assert.DoesNotContain(" or ", query.Filter); } [Fact] - public async Task ListAsync_LegacyHeaderUsingDefaultPartitionMapping_IsStillReturned() + public async Task ListAsync_CustomPartitionMapping_RawPrefixExcludesSuccessor() { var table = new FakeTableClient(); - var journalId = new JournalId("legacy/orders/42"); - table.AddLegacyHeader(journalId); - using var context = await CreateStartedProviderAsync(table, TestContext.Current.CancellationToken); + var exact = new JournalId("tenant/order"); + var partialSegment = new JournalId("tenant/orders/42"); + table.AddHeader(exact, "hash-z"); + table.AddHeader(partialSegment, "hash-y"); + table.AddHeader(new JournalId("tenant/ordes"), "hash-x"); + var options = CreateOptions(table); + options.GetPartitionKey = static _ => "custom"; + using var context = CreateProvider(options); + await StartAsync(context.Provider, TestContext.Current.CancellationToken); var result = await ToListAsync( - context.Provider.ListAsync(new() { Prefix = new JournalId("legacy/orders") }, TestContext.Current.CancellationToken), + context.Provider.ListAsync( + new() { Prefix = exact, MaxId = new("tenant/ordes") }, + TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); - Assert.Equal([journalId], result); + Assert.Equal([exact, partialSegment], result.Select(entry => entry.Id)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(2, query.ReturnedCount); + Assert.Contains("JournalId ge 'tenant/order' and JournalId lt 'tenant/ordes'", query.Filter); + Assert.DoesNotContain("PartitionKey", query.Filter); + } + + [Theory] + [InlineData(0xD7FF)] + [InlineData(0x103FF)] + [InlineData(0x10FFFF)] + public async Task ListAsync_CustomMapping_PrefixSuccessorOmitsMalformedODataUpperBound(int lastScalar) + { + var prefix = new JournalId("range/" + char.ConvertFromUtf32(lastScalar)); + var child = new JournalId(prefix.Value + "/child"); + var table = new FakeTableClient(); + table.AddHeader(new("range/a"), "before"); + table.AddHeader(prefix, "exact"); + table.AddHeader(child, "child"); + table.AddHeader(new("range/\uE000"), "after"); + var options = CreateOptions(table); + options.GetPartitionKey = static _ => "custom"; + using var context = CreateProvider(options); + await StartAsync(context.Provider, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync(new() { Prefix = prefix }, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal([prefix, child], result.Select(entry => entry.Id)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(3, query.ReturnedCount); + Assert.Equal( + TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}") + + TableClient.CreateQueryFilter($" and JournalId ge {prefix.Value}"), + query.Filter); + } + + [Theory] + [InlineData(0xD800, true, "")] + [InlineData(0xD800, false, "")] + [InlineData(0xDFFF, true, "")] + [InlineData(0xDFFF, false, "")] + [InlineData(0xD800, true, "/suffix")] + [InlineData(0xD800, false, "/suffix")] + [InlineData(0xDFFF, true, "/suffix")] + [InlineData(0xDFFF, false, "/suffix")] + public async Task ListAsync_CustomMapping_MalformedExplicitBoundUsesOtherNativeBoundAndLocalRange( + int surrogate, bool malformedMinimum, string suffix) + { + var malformed = "range/" + (char)surrogate + suffix; + var minimum = malformedMinimum ? malformed : "range/"; + var maximum = malformedMinimum ? "range/\uFFFF" : malformed; + string[] ids = ["range/a", "range/b", "range/\U0001F600", "range/\uE000"]; + var table = new FakeTableClient(); + for (var i = 0; i < ids.Length; i++) + { + table.AddHeader(new(ids[i]), $"opaque-{i}"); + } + + var options = CreateOptions(table); + options.GetPartitionKey = static _ => "custom"; + using var context = CreateProvider(options); + await StartAsync(context.Provider, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync( + new() { MinId = new(minimum), MaxId = new(maximum) }, + TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal( + ids.Where(id => string.CompareOrdinal(id, minimum) >= 0 && string.CompareOrdinal(id, maximum) <= 0), + result.Select(entry => entry.Id.Value)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(ids.Length, query.ReturnedCount); + Assert.Equal( + TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}") + + (malformedMinimum + ? TableClient.CreateQueryFilter($" and JournalId le {maximum}") + : TableClient.CreateQueryFilter($" and JournalId ge {minimum}")), + query.Filter); + } + + [Fact] + public async Task ListAsync_CustomMapping_TwoMalformedBoundsUseHeaderQueryAndExactLocalPrefix() + { + var prefix = "range/" + (char)0xD83D; + var first = new JournalId("range/\U0001F600"); + var second = new JournalId("range/\U0001F601"); + var table = new FakeTableClient(); + table.AddHeader(new("range/a"), "before"); + table.AddHeader(first, "first"); + table.AddHeader(second, "second"); + table.AddHeader(new("range/\U0001F900"), "after"); + var options = CreateOptions(table); + options.GetPartitionKey = static _ => "custom"; + using var context = CreateProvider(options); + await StartAsync(context.Provider, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync(new() { Prefix = new(prefix) }, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal([first, second], result.Select(entry => entry.Id)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(4, query.ReturnedCount); + Assert.Equal(TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"), query.Filter); + } + + [Fact] + public async Task ListAsync_CustomMapping_CompleteSurrogatePairsRetainBothNativeBounds() + { + var minimum = new JournalId("range/\U0001F600"); + var maximum = new JournalId("range/\U0001F601"); + var table = new FakeTableClient(); + table.AddHeader(new("range/a"), "before"); + table.AddHeader(minimum, "minimum"); + table.AddHeader(maximum, "maximum"); + table.AddHeader(new("range/\uE000"), "after"); + var options = CreateOptions(table); + options.GetPartitionKey = static _ => "custom"; + using var context = CreateProvider(options); + await StartAsync(context.Provider, TestContext.Current.CancellationToken); + + var result = await ToListAsync( + context.Provider.ListAsync(new() { MinId = minimum, MaxId = maximum }, TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken); + + Assert.Equal([minimum, maximum], result.Select(entry => entry.Id)); + var query = Assert.Single(table.QueryCalls); + Assert.Equal(2, query.ReturnedCount); + Assert.Equal( + TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}") + + TableClient.CreateQueryFilter($" and JournalId ge {minimum.Value}") + + TableClient.CreateQueryFilter($" and JournalId le {maximum.Value}"), + query.Filter); } [Fact] @@ -595,6 +1080,7 @@ public override TableClient GetTableClient(string tableName) private sealed class FakeTableClient : TableClient { + private static readonly UTF8Encoding StrictUtf8 = new(false, true); private readonly List _entities = []; public override string Name => "journal"; @@ -622,11 +1108,6 @@ public void AddHeader(JournalId journalId, string? partitionKey = null) [AzureTableJournalStorage.JournalIdPropertyName] = journalId.Value, }); - public void AddLegacyHeader(JournalId journalId) - => AddEntity( - AzureTableJournalStorageOptions.GetDefaultPartitionKey(journalId), - AzureTableJournalStorage.HeaderRowKey); - public void RemoveHeader(JournalId journalId) => _entities.RemoveAll(entity => entity.PartitionKey == AzureTableJournalStorageOptions.GetDefaultPartitionKey(journalId) @@ -686,9 +1167,35 @@ public override AsyncPageable QueryAsync( CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - QueryCalls.Add(new(filter, select?.ToArray())); + // OData filter text must survive transport encoding without surrogate replacement. + _ = StrictUtf8.GetByteCount(filter!); + var clauses = Regex.Matches(filter!, @"(RowKey|PartitionKey|JournalId) (eq|ge|le|lt) '((?:[^']|'')*)'"); + Assert.Equal(filter, string.Join(" and ", clauses.Select(clause => clause.Value))); var values = _entities - .Where(entity => entity.RowKey == AzureTableJournalStorage.HeaderRowKey) + .Where(entity => clauses.All(clause => + { + var property = clause.Groups[1].Value; + var value = property switch + { + "PartitionKey" => entity.PartitionKey, + "RowKey" => entity.RowKey, + _ => entity.TryGetValue(property, out var stored) ? stored as string : null, + }; + if (value is null) + { + return false; + } + + var comparison = string.CompareOrdinal(value, clause.Groups[3].Value.Replace("''", "'")); + return clause.Groups[2].Value switch + { + "eq" => comparison == 0, + "ge" => comparison >= 0, + "le" => comparison <= 0, + "lt" => comparison < 0, + _ => throw new InvalidOperationException("Unexpected table filter comparison."), + }; + })) .Select(entity => { var projected = new TableEntity(entity.PartitionKey, entity.RowKey); @@ -703,6 +1210,7 @@ public override AsyncPageable QueryAsync( return (T)(ITableEntity)projected; }) .ToList(); + QueryCalls.Add(new(filter, select?.ToArray(), maxPerPage, values.Count)); var page = Page.FromValues(values, continuationToken: null, new FakeResponse(200, default)); AfterQuery?.Invoke(); return AsyncPageable.FromPages([page]); @@ -710,7 +1218,7 @@ public override AsyncPageable QueryAsync( } - private sealed record QueryCall(string? Filter, IReadOnlyList? Select); + private sealed record QueryCall(string? Filter, IReadOnlyList? Select, int? MaxPerPage, int ReturnedCount); private sealed class FakeResponse(int status, ETag eTag) : Response { diff --git a/test/Orleans.Journaling.Tests/AzureTableJournalStorageTests.cs b/test/Orleans.Journaling.Tests/AzureTableJournalStorageTests.cs index 6bc9be31722..d63aab31589 100644 --- a/test/Orleans.Journaling.Tests/AzureTableJournalStorageTests.cs +++ b/test/Orleans.Journaling.Tests/AzureTableJournalStorageTests.cs @@ -37,6 +37,41 @@ public async Task AppendAsync_RoundTripsThroughRead() Assert.Equal(2, store.DataRowCount(TestPartitionKey)); } + [Fact] + public async Task AppendAsync_DefaultMapping_LongestPrintableAsciiId_RoundTrips() + { + var store = new FakeTableStore(); + var journalId = new JournalId(new string('~', 512)); + var storage = CreateStorage(store, journalId: journalId); + + await storage.AppendAsync(new ReadOnlySequence([1, 2]), TestContext.Current.CancellationToken); + var consumer = new CapturingJournalStorageConsumer(); + await CreateStorage(store, journalId: journalId).ReadAsync(consumer, TestContext.Current.CancellationToken); + + Assert.Equal([1, 2], consumer.Bytes.ToArray()); + var header = Assert.Single(store.AddCalls); + Assert.Equal(string.Concat(Enumerable.Repeat("7E", 512)), header.PartitionKey); + Assert.Equal(journalId.Value, header.Properties[AzureTableJournalStorage.JournalIdPropertyName]); + } + + [Fact] + public async Task AppendAsync_CustomMapping_RoundTripsUnicodeJournalIdAndCanonicalHeader() + { + var store = new FakeTableStore(); + var journalId = new JournalId("café/😀\0"); + static void Configure(AzureTableJournalStorageOptions options) => options.GetPartitionKey = _ => "custom"; + var storage = CreateStorage(store, journalId: journalId, configure: Configure); + + await storage.AppendAsync(new ReadOnlySequence([1, 2]), TestContext.Current.CancellationToken); + var consumer = new CapturingJournalStorageConsumer(); + await CreateStorage(store, journalId: journalId, configure: Configure).ReadAsync(consumer, TestContext.Current.CancellationToken); + + Assert.Equal([1, 2], consumer.Bytes.ToArray()); + var header = Assert.Single(store.AddCalls); + Assert.Equal("custom", header.PartitionKey); + Assert.Equal(journalId.Value, header.Properties[AzureTableJournalStorage.JournalIdPropertyName]); + } + [Fact] public async Task AppendAsync_ChunksLargePayloadAcrossPropertiesAndRows() { @@ -186,16 +221,16 @@ public async Task CreateIfNotExistsAsync_WhenPartitionMapperCollides_RejectsDiff } [Fact] - public async Task LegacyHeader_ReadsAndMigratesThroughMetadataAppendReplaceAndDelete() + public async Task CanonicalHeader_PreservesIdentityThroughMetadataAppendReplaceAndDelete() { var store = new FakeTableStore(); store.PutEntity( TestPartitionKey, AzureTableJournalStorage.HeaderRowKey, - CreateLegacyHeaderProperties("legacy", rowCount: 1, length: 1, metadataJson: """{"owner":"alice"}""")); + CreateHeaderProperties("current", rowCount: 1, length: 1, metadataJson: """{"owner":"alice"}""")); store.PutEntity( TestPartitionKey, - FormatDataRowKey("legacy", 0), + FormatDataRowKey("current", 0), new Dictionary { ["Data00"] = new byte[] { 1 } }); var storage = CreateStorage(store, compactionRowCountThreshold: 2); @@ -206,7 +241,7 @@ public async Task LegacyHeader_ReadsAndMigratesThroughMetadataAppendReplaceAndDe Assert.Equal([1], consumer.Bytes.ToArray()); Assert.Equal("alice", Assert.IsAssignableFrom(metadata).Properties["owner"]); Assert.NotNull(await storage.UpdateMetadataAsync( - set: new Dictionary { ["migrated"] = "true" }, + set: new Dictionary { ["updated"] = "true" }, cancellationToken: CancellationToken.None)); Assert.Equal( TestJournalId.Value, @@ -747,9 +782,9 @@ public async Task UpdateMetadataAsync_PreservesProviderProperties() } [Fact] - public void DefaultPartitionKey_EscapesJournalIdValue() + public void DefaultPartitionKey_HexEncodesJournalIdValue() { - Assert.Equal("journals%2Ftest", AzureTableJournalStorageOptions.GetDefaultPartitionKey(new JournalId("journals/test"))); + Assert.Equal("6A6F75726E616C732F74657374", AzureTableJournalStorageOptions.GetDefaultPartitionKey(new JournalId("journals/test"))); } [Fact] @@ -1787,20 +1822,6 @@ private static Dictionary CreateHeaderProperties( return result; } - private static Dictionary CreateLegacyHeaderProperties( - string generation, - long rowCount, - long length, - string metadataJson) - => new() - { - [AzureTableJournalStorage.FormatPropertyName] = string.Empty, - [AzureTableJournalStorage.GenerationPropertyName] = generation, - [AzureTableJournalStorage.RowCountPropertyName] = rowCount, - [AzureTableJournalStorage.LengthPropertyName] = length, - [AzureTableJournalStorage.MetadataPropertyName] = metadataJson, - }; - private static string FormatDataRowKey(string generation, long sequence) => $"{generation}-{sequence:D12}"; private static ReadOnlySequence CreateMultiSegmentSequence(byte[] payload, params int[] segmentLengths) diff --git a/test/Orleans.Journaling.Tests/JournalCatalogRangeTests.cs b/test/Orleans.Journaling.Tests/JournalCatalogRangeTests.cs new file mode 100644 index 00000000000..221e8679b38 --- /dev/null +++ b/test/Orleans.Journaling.Tests/JournalCatalogRangeTests.cs @@ -0,0 +1,159 @@ +using System.Text; +using Xunit; + +namespace Orleans.Journaling.Tests; + +[TestSuite("BVT"), TestProvider("None"), TestCategory("BVT")] +public sealed class JournalCatalogRangeTests +{ + [Theory] + [InlineData(null, null, null, "anything", true)] + [InlineData("jobs/shards/20260909", null, null, "jobs/shards/20260909T120000-a", true)] + [InlineData("jobs/shards/20260909", null, null, "jobs/shards/20260910T120000-a", false)] + [InlineData("tenant", null, null, "tenant2", true)] + [InlineData("tenant/", null, null, "tenant2", false)] + [InlineData(null, "b", "d", "b", true)] + [InlineData(null, "b", "d", "d", true)] + [InlineData(null, "b", "d", "a", false)] + [InlineData(null, "b", "d", "e", false)] + [InlineData("a", "b", "z", "abc", false)] + [InlineData(null, "z", "a", "m", false)] + public void Contains_IntersectsRawPrefixAndInclusiveBounds(string? prefix, string? min, string? max, string id, bool expected) + { + var range = new JournalCatalogRange(new() + { + Prefix = ToId(prefix), + MinId = ToId(min), + MaxId = ToId(max) + }); + + Assert.Equal(expected, range.Contains(id)); + } + + [Fact] + public void Snapshot_RetainsBoundsAndDerivesNarrowerNativePrefix() + { + var options = new ListOptions + { + Prefix = new("jobs/"), + MinId = new("jobs/shards/20260909-a"), + MaxId = new("jobs/shards/20260909-z"), + IncludeMetadata = true + }; + var range = new JournalCatalogRange(options); + options.Prefix = new("other/"); + options.MinId = new("z"); + options.MaxId = new("z"); + options.IncludeMetadata = false; + + Assert.True(range.IncludeMetadata); + Assert.Equal("jobs/shards/20260909-", range.ListingPrefix); + Assert.Equal("jobs/shards/20260909-a", range.LowerBound); + Assert.Equal("jobs/shards/20260909-z", range.UpperBound); + Assert.True(range.Contains("jobs/shards/20260909-m")); + Assert.False(range.Contains("jobs/shards/20260908-m")); + } + + [Theory] + [InlineData("a", "b", null)] + [InlineData("b", null, "a")] + [InlineData(null, "z", "a")] + [InlineData("other/", "jobs/a", "jobs/z")] + public void DisjointConstraints_AreEmpty(string? prefix, string? min, string? max) + { + var range = new JournalCatalogRange(new() { Prefix = ToId(prefix), MinId = ToId(min), MaxId = ToId(max) }); + + Assert.True(range.IsEmpty); + } + + [Fact] + public void NativePrefix_PreservesMatchingIdsWithoutSplittingSurrogatePairs() + { + var range = new JournalCatalogRange(new() + { + MinId = new("tenant/\U0001F600-a"), + MaxId = new("tenant/\U0001F601-z") + }); + + Assert.Equal("tenant/", range.ListingPrefix); + Assert.True(range.Contains("tenant/\U0001F600-b")); + Assert.True(range.Contains("tenant/\U0001F601-a")); + } + + [Theory] + [InlineData(0xD800, "", "suffix")] + [InlineData(0xDBFF, "", "")] + [InlineData(0xDC00, "", "suffix")] + [InlineData(0xDFFF, "", "")] + [InlineData(0xD800, "tenant/", "/suffix")] + [InlineData(0xDBFF, "tenant/", "")] + [InlineData(0xDC00, "tenant/", "/suffix")] + [InlineData(0xDFFF, "tenant/", "")] + [InlineData(0xD800, "tenant/\U0001F600/", "/\U0001F601")] + [InlineData(0xDC00, "tenant/\U0001F600/", "/\U0001F601")] + public void NativePrefix_BroadensBeforeFirstUnpairedSurrogateAndKeepsOrdinalBounds( + int surrogate, string before, string after) + { + // Construct malformed UTF-16 at runtime so attribute serialization preserves the intended code unit. + var prefix = before + (char)surrogate + after; + var min = new JournalId(prefix + "b"); + var max = new JournalId(prefix + "d"); + var strictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + foreach (var fromBounds in new[] { false, true }) + { + var options = new ListOptions + { + Prefix = fromBounds ? default : new(prefix), + MinId = min, + MaxId = max + }; + var range = new JournalCatalogRange(options); + options.Prefix = new("changed"); + options.MinId = default; + options.MaxId = default; + + Assert.Equal(before.Length == 0 ? null : before, range.ListingPrefix); + Assert.Equal(fromBounds ? null : prefix, range.Prefix); + Assert.Equal(min.Value, range.MinId); + Assert.Equal(max.Value, range.MaxId); + Assert.Equal(before, strictUtf8.GetString(strictUtf8.GetBytes(range.ListingPrefix ?? string.Empty))); + Assert.True(range.Contains(min.Value)); + Assert.True(range.Contains(prefix + "c")); + Assert.True(range.Contains(max.Value)); + Assert.False(range.Contains(prefix + "a")); + Assert.False(range.Contains(prefix + "e")); + Assert.False(range.Contains(before + "other")); + } + } + + [Theory] + [InlineData("tenant/ascii")] + [InlineData("tenant/\u00e9")] + [InlineData("tenant/\U0001F600")] + [InlineData("\U00010000/\U0010FFFF")] + public void NativePrefix_PreservesCompleteCharacters(string prefix) + { + var range = new JournalCatalogRange(new() { Prefix = new(prefix) }); + + Assert.Equal(prefix, range.ListingPrefix); + Assert.True(range.Contains(prefix + "/child")); + Assert.False(range.Contains("other")); + } + + [Fact] + public void DescendantRange_IncludesMaximumAndExcludesNamespaceId() + { + const string maximum = "jobs/shards/20260909T1200000000000Z~"; + var range = new JournalCatalogRange(new() + { + Prefix = new("jobs/shards/"), + MaxId = new(maximum) + }); + + Assert.Equal(maximum, range.UpperBound); + Assert.True(range.Contains(maximum)); + Assert.False(range.Contains("jobs/shards")); + } + + private static JournalId ToId(string? value) => value is null ? default : new(value); +} diff --git a/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs b/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs index e77a73f7175..22f0a9d4fd5 100644 --- a/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs +++ b/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Globalization; +using System.Text.RegularExpressions; using Amazon.S3; using Amazon.S3.Model; using Azure; @@ -8,6 +9,7 @@ using Azure.Data.Tables.Models; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -22,30 +24,238 @@ namespace Orleans.Journaling.Tests; [TestCategory("BVT")] public sealed class JournalStorageCatalogTests { + [Theory] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + public async Task AzureListAsync_IncludeMetadataProjectsCompleteSnapshotWithoutPerJournalRequests(string kind) + { + await using var context = await CreateAsync(kind, ["tenant/a", "tenant/b", "tenant/c"]); + var entries = await DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken)); + + Assert.Equal(["tenant/a", "tenant/b", "tenant/c"], entries.Select(entry => entry.Id.Value)); + Assert.Equal(2, context.Native.Requests.Count); + Assert.Equal(0, context.Native.MetadataRequests); + AssertMetadataProjection(context.Native, kind, includeMetadata: true); + foreach (var entry in entries) + { + var metadata = Assert.IsAssignableFrom(entry.Metadata); + Assert.Equal("test", metadata.Format); + Assert.Equal(new ETag("listed").ToString(), metadata.ETag); + Assert.Equal(new Dictionary { ["owner"] = "alice" }, metadata.Properties); + + var current = await context.Provider.CreateStorage(entry.Id).GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(current); + Assert.Equal(current.Format, metadata.Format); + Assert.Equal(current.ETag, metadata.ETag); + Assert.Equal(current.Properties, metadata.Properties); + } + + Assert.Equal(entries.Count, context.Native.MetadataRequests); + } + + [Theory] + [InlineData("AzureBlob", false)] + [InlineData("AzureBlob", true)] + [InlineData("AzureTable", false)] + [InlineData("AzureTable", true)] + public async Task AzureListAsync_IncludeMetadataIsSnapshottedAcrossPages(string kind, bool includeMetadata) + { + await using var context = await CreateAsync(kind, ["tenant/a", "tenant/b", "tenant/c"]); + var options = new ListOptions { IncludeMetadata = !includeMetadata }; + var listing = context.Catalog.ListAsync(options, TestContext.Current.CancellationToken); + options.IncludeMetadata = includeMetadata; + await using (var enumerator = listing.GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(includeMetadata, enumerator.Current.Metadata is not null); + options.IncludeMetadata = !includeMetadata; + var count = 1; + while (await enumerator.MoveNextAsync()) + { + Assert.Equal(includeMetadata, enumerator.Current.Metadata is not null); + count++; + } + + Assert.Equal(3, count); + } + + Assert.Equal(2, context.Native.Requests.Count); + Assert.Equal(0, context.Native.MetadataRequests); + AssertMetadataProjection(context.Native, kind, includeMetadata); + var next = await DrainEntriesAsync(listing); + Assert.Equal(3, next.Count); + Assert.All(next, entry => Assert.Equal(!includeMetadata, entry.Metadata is not null)); + Assert.Equal(0, context.Native.MetadataRequests); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + public async Task AzureListAsync_ProjectedETagRejectsStaleMetadataUpdate(string kind) + { + await using var context = await CreateAsync(kind, ["tenant/a"]); + var entry = Assert.Single(await DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + var metadata = Assert.IsAssignableFrom(entry.Metadata); + Assert.Equal(0, context.Native.MetadataRequests); + context.Native.CurrentETag = new ETag("changed"); + + var updated = await context.Provider.CreateStorage(entry.Id).UpdateMetadataAsync( + set: new Dictionary { ["owner"] = "bob" }, + expectedETag: metadata.ETag, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Null(updated); + Assert.Equal(1, context.Native.MetadataRequests); + Assert.Equal(new ETag("listed").ToString(), metadata.ETag); + Assert.Equal("alice", metadata.Properties["owner"]); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + public async Task AzureListAsync_EmptyMetadataStillReturnsCompleteVersionedSnapshot(string kind) + { + var blob = Blob("wal/tenant/a"); + blob.Metadata.Clear(); + var header = Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant/a")), "tenant/a"); + header[AzureTableJournalStorage.FormatPropertyName] = string.Empty; + header[AzureTableJournalStorage.MetadataPropertyName] = "{}"; + await using var context = await CreateAsync(kind, [], blobs: [blob], headers: [header]); + + var entry = Assert.Single(await DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + + Assert.Equal(new JournalId("tenant/a"), entry.Id); + var metadata = Assert.IsAssignableFrom(entry.Metadata); + Assert.Null(metadata.Format); + Assert.Equal(new ETag("listed").ToString(), metadata.ETag); + Assert.Empty(metadata.Properties); + Assert.Equal(0, context.Native.MetadataRequests); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + public async Task AzureListAsync_MetadataSnapshotRetainsObservedPropertiesAfterStorageChanges(string kind) + { + var blob = Blob("wal/tenant/a"); + var header = Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant/a")), "tenant/a"); + await using var context = await CreateAsync(kind, [], blobs: [blob], headers: [header]); + var entry = Assert.Single(await DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + var metadata = Assert.IsAssignableFrom(entry.Metadata); + + blob.Metadata["owner"] = "bob"; + header[AzureTableJournalStorage.MetadataPropertyName] = """{"owner":"bob"}"""; + context.Native.CurrentETag = new ETag("changed"); + var current = await context.Provider.CreateStorage(entry.Id).GetMetadataAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(current); + Assert.Equal("bob", current.Properties["owner"]); + Assert.Equal(new ETag("changed").ToString(), current.ETag); + Assert.Equal("test", metadata.Format); + Assert.Equal(new ETag("listed").ToString(), metadata.ETag); + Assert.Equal(new Dictionary { ["owner"] = "alice" }, metadata.Properties); + Assert.Equal(1, context.Native.MetadataRequests); + } + + [Theory] + [InlineData("Volatile")] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + [InlineData("S3")] + public async Task ListAsync_DefaultProjectionReturnsIdsWithoutMetadata(string kind) + { + await using var context = await CreateAsync(kind, ["tenant/a"]); + var entry = Assert.Single(await DrainEntriesAsync(context.Catalog.ListAsync( + cancellationToken: TestContext.Current.CancellationToken))); + + Assert.Equal(new JournalId("tenant/a"), entry.Id); + Assert.Null(entry.Metadata); + Assert.Equal(0, context.Native.MetadataRequests); + if (kind is "AzureBlob" or "AzureTable") + { + AssertMetadataProjection(context.Native, kind, includeMetadata: false); + } + } + + [Fact] + public async Task S3ListAsync_IncludeMetadataRetainsIdentityOnlyProjection() + { + await using var context = await CreateAsync("S3", ["tenant/a"]); + var entry = Assert.Single(await DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + + Assert.Equal(new JournalId("tenant/a"), entry.Id); + Assert.Null(entry.Metadata); + context.AssertNoS3MetadataRequests(); + } + + [Theory] + [InlineData("{", typeof(InvalidOperationException))] + [InlineData("{\"owner\":null}", typeof(ArgumentNullException))] + public async Task AzureTableListAsync_MalformedProjectedMetadataPropagates(string json, Type exceptionType) + { + var header = Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant/a")), "tenant/a"); + header[AzureTableJournalStorage.MetadataPropertyName] = json; + await using var context = await CreateAsync("AzureTable", [], headers: [header]); + + var exception = await Record.ExceptionAsync(() => DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + + Assert.IsType(exceptionType, exception); + Assert.Equal(0, context.Native.MetadataRequests); + Assert.Equal(["tenant/a"], await DrainAsync(context.Catalog.ListAsync( + cancellationToken: TestContext.Current.CancellationToken))); + } + + [Fact] + public async Task AzureBlobListAsync_MalformedProjectedMetadataPropagates() + { + var blob = Blob("wal/tenant/a"); + blob.Metadata["owner"] = null!; + await using var context = await CreateAsync("AzureBlob", [], blobs: [blob]); + + await Assert.ThrowsAsync(() => DrainEntriesAsync(context.Catalog.ListAsync( + new() { IncludeMetadata = true }, TestContext.Current.CancellationToken))); + + Assert.Equal(0, context.Native.MetadataRequests); + Assert.Equal(["tenant/a"], await DrainAsync(context.Catalog.ListAsync( + cancellationToken: TestContext.Current.CancellationToken))); + } + [Theory] [InlineData("Volatile")] [InlineData("AzureBlob")] [InlineData("AzureTable")] [InlineData("S3")] - public async Task ListAsync_OptionsSelectExactAndDescendantIdentities(string kind) + public async Task ListAsync_PrefixUsesRawOrdinalStartsWith(string kind) { string[] ids = [ "tenant/z", "tenant2", "tenant", "tenant/a", "tenant/a/child", "tenantish/child", - "tenant%2Fone", "tenant%2Fone/child", " leading space ", "raw\uD800", "raw\uD801", - "unicode/\u00E9", "percent%2f", @"back\slash", "tenant/", + "tenant%2Fone", "tenant%2Fone/child", " leading space ", + "percent%2f", @"back\slash", "tenant/", ]; + if (kind != "AzureTable") + { + ids = [.. ids, "raw\uD800", "raw\uD801", "unicode/\u00E9"]; + } await using var context = await CreateAsync(kind, ids); AssertMembership(ids, await DrainAsync(context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken))); AssertMembership(ids, await DrainAsync(context.Catalog.ListAsync(new(), TestContext.Current.CancellationToken))); AssertMembership( - ["tenant", "tenant/z", "tenant/a", "tenant/a/child", "tenant/"], + ["tenant", "tenant/z", "tenant/a", "tenant/a/child", "tenant/", "tenant2", "tenantish/child", "tenant%2Fone", "tenant%2Fone/child"], await DrainAsync(context.Catalog.ListAsync(new() { Prefix = new("tenant") }, TestContext.Current.CancellationToken))); AssertMembership( ["tenant%2Fone", "tenant%2Fone/child"], await DrainAsync(context.Catalog.ListAsync( new() { Prefix = JournalId.Create("tenant/one") }, TestContext.Current.CancellationToken))); + AssertMembership(["tenant/a", "tenant/a/child"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/a") }, TestContext.Current.CancellationToken))); } [Theory] @@ -56,23 +266,82 @@ await DrainAsync(context.Catalog.ListAsync( public async Task ListAsync_OptionsAreReadAtEnumerationStartAndRemainStable(string kind) { await using var context = await CreateAsync(kind, ["tenant/z", "tenant/a", "tenant/b", "other/q"]); - var options = new ListOptions { Prefix = new("other") }; + var options = new ListOptions { Prefix = new("other"), MinId = new("other"), MaxId = new("other") }; var listing = context.Catalog.ListAsync(options, TestContext.Current.CancellationToken); options.Prefix = new("tenant"); + options.MinId = new("tenant/a"); + options.MaxId = new("tenant/b"); await using var enumerator = listing.GetAsyncEnumerator(TestContext.Current.CancellationToken); Assert.True(await enumerator.MoveNextAsync()); - var result = new List { enumerator.Current.Value }; + var result = new List { enumerator.Current.Id.Value }; options.Prefix = new("other"); + options.MinId = new("other/q"); + options.MaxId = default; while (await enumerator.MoveNextAsync()) { - result.Add(enumerator.Current.Value); + result.Add(enumerator.Current.Id.Value); } - AssertMembership(["tenant/z", "tenant/a", "tenant/b"], result); + AssertMembership(["tenant/a", "tenant/b"], result); AssertMembership(["other/q"], await DrainAsync(listing)); } + [Theory] + [InlineData("Volatile")] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + [InlineData("S3")] + public async Task ListAsync_MinAndMaxAreInclusiveAndIntersectRawPrefix(string kind) + { + string[] ids = ["tenant/z", "tenant/a", "tenant/b", "tenant/b-extra", "tenant/c", "tenant2/a", "other"]; + await using var context = await CreateAsync(kind, ids); + + AssertMembership(["tenant/b", "tenant/b-extra", "tenant/c"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MinId = new("tenant/b"), MaxId = new("tenant/c") }, TestContext.Current.CancellationToken))); + AssertMembership(["tenant/b", "tenant/b-extra", "tenant/c"], await DrainAsync(context.Catalog.ListAsync( + new() { MinId = new("tenant/b"), MaxId = new("tenant/c") }, TestContext.Current.CancellationToken))); + AssertMembership(["tenant/b", "tenant/b-extra"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/b"), MinId = new("tenant/a"), MaxId = new("tenant/c") }, TestContext.Current.CancellationToken))); + } + + [Theory] + [InlineData("Volatile")] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + [InlineData("S3")] + public async Task ListAsync_EmptyRangeDoesNotRequestStorage(string kind) + { + await using var context = await CreateAsync(kind, ["tenant/a"]); + Assert.Empty(await DrainAsync(context.Catalog.ListAsync( + new() { MinId = new("z"), MaxId = new("a") }, TestContext.Current.CancellationToken))); + Assert.Empty(await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MinId = new("z") }, TestContext.Current.CancellationToken))); + Assert.Empty(await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("a") }, TestContext.Current.CancellationToken))); + Assert.Empty(context.Native.Requests); + } + + [Theory] + [InlineData("Volatile")] + [InlineData("AzureBlob")] + [InlineData("AzureTable")] + [InlineData("S3")] + public async Task ListAsync_MaxIdIsInclusiveOrdinalAndCombinedWithPrefix(string kind) + { + string[] ids = ["tenant/z", "tenant/a", "tenant/B", "tenant", "tenant/a/child", "other", "tenant2"]; + await using var context = await CreateAsync(kind, ids); + + AssertMembership(ids, await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = default }, TestContext.Current.CancellationToken))); + AssertMembership(["other", "tenant", "tenant/B", "tenant/a"], await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = new("tenant/a") }, TestContext.Current.CancellationToken))); + AssertMembership(["tenant", "tenant/B", "tenant/a"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/a") }, TestContext.Current.CancellationToken))); + Assert.Empty(await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("other") }, TestContext.Current.CancellationToken))); + } + [Theory] [InlineData("AzureBlob")] [InlineData("AzureTable")] @@ -88,10 +357,10 @@ public async Task ListAsync_YieldsCurrentPageBeforeFetchingFailingTail(string ki Assert.Empty(context.Native.Requests); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal("tenant/z", enumerator.Current.Value); + Assert.Equal(kind == "AzureBlob" ? "tenant/a" : "tenant/z", enumerator.Current.Id.Value); Assert.Single(context.Native.Requests); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal("tenant/a", enumerator.Current.Value); + Assert.Equal(kind == "AzureBlob" ? "tenant/b" : "tenant/a", enumerator.Current.Id.Value); Assert.Single(context.Native.Requests); var actual = await Assert.ThrowsAsync(() => enumerator.MoveNextAsync().AsTask()); @@ -105,7 +374,7 @@ public async Task ListAsync_YieldsCurrentPageBeforeFetchingFailingTail(string ki } context.Native.Failure = null; - Assert.Equal( + AssertMembership( ["tenant/z", "tenant/a", "tenant/b"], await DrainAsync(context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken))); Assert.Null(context.Native.Requests[2].Cursor); @@ -121,7 +390,7 @@ public async Task ListAsync_EarlyDisposalStopsFetchingPages(string kind) await using (var enumerator = context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken)) { Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal("z", enumerator.Current.Value); + Assert.Equal(kind == "AzureBlob" ? "a" : "z", enumerator.Current.Id.Value); Assert.Single(context.Native.Requests); } @@ -133,19 +402,18 @@ public async Task ListAsync_EarlyDisposalStopsFetchingPages(string kind) [InlineData("AzureBlob")] [InlineData("AzureTable")] [InlineData("S3")] - public async Task ListAsync_AdvanceCrossesEmptyAndFilteredNativePages(string kind) + public async Task ListAsync_AdvanceCrossesEmptyNativePages(string kind) { await using var context = await CreateAsync(kind, ["tenant2", "tenantish/child", "tenant/valid"]); context.Native.EmptyFirstPage = true; await using var enumerator = context.Catalog.ListAsync( - new() { Prefix = new("tenant") }, TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); + new() { Prefix = new("tenant/") }, TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); Assert.True(await enumerator.MoveNextAsync()); - Assert.Equal("tenant/valid", enumerator.Current.Value); - Assert.Equal(3, context.Native.Requests.Count); - Assert.Equal([0, 2, 1], context.Native.Requests.Select(request => request.ResultCount)); + Assert.Equal("tenant/valid", enumerator.Current.Id.Value); + Assert.Equal([0, 1], context.Native.Requests.Select(request => request.ResultCount)); Assert.False(await enumerator.MoveNextAsync()); - Assert.Equal(3, context.Native.Requests.Count); + Assert.Equal(2, context.Native.Requests.Count); } [Theory] @@ -186,12 +454,13 @@ public async Task ListAsync_CancellationAfterNativeResponseIncludesEmptyPages(st await using var context = await CreateAsync(kind, empty ? [] : ["z", "a"]); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); context.Native.BeforeResponse = cancellation.Cancel; - await using var enumerator = context.Catalog.ListAsync(cancellationToken: cancellation.Token).GetAsyncEnumerator(cancellation.Token); + await using var enumerator = context.Catalog.ListAsync( + new() { MaxId = new("0") }, cancellation.Token).GetAsyncEnumerator(cancellation.Token); var actual = await Assert.ThrowsAnyAsync(() => enumerator.MoveNextAsync().AsTask()); Assert.Equal(cancellation.Token, actual.CancellationToken); var request = Assert.Single(context.Native.Requests); - Assert.Equal(empty ? 0 : 2, request.ResultCount); + Assert.Equal(empty || kind == "AzureTable" ? 0 : 2, request.ResultCount); Assert.Equal(cancellation.Token, request.CancellationToken); } } @@ -204,7 +473,7 @@ public async Task ListAsync_NativeRequestsCarryPrefixContinuationAndCancellation { await using var context = await CreateAsync(kind, ["tenant/z", "tenant/a", "tenant/b"]); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - Assert.Equal( + AssertMembership( ["tenant/z", "tenant/a", "tenant/b"], await DrainAsync(context.Catalog.ListAsync(new() { Prefix = new("tenant") }, cancellation.Token))); @@ -216,13 +485,12 @@ public async Task ListAsync_NativeRequestsCarryPrefixContinuationAndCancellation { Assert.Equal(kind == "AzureBlob" ? 5000 : 1000, request.Maximum); Assert.Equal(cancellation.Token, request.CancellationToken); - Assert.Equal(kind == "AzureBlob" ? "tenant" : null, request.Prefix); + Assert.Equal(kind == "AzureBlob" ? "wal/tenant" : kind == "S3" ? "wal/" : null, request.Prefix); }); if (kind == "AzureTable") { - Assert.Equal( - TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"), - context.Native.Filter); + Assert.Contains($"PartitionKey ge '{AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant"))}'", context.Native.Filter); + Assert.DoesNotContain(" or ", context.Native.Filter); Assert.Equal([AzureTableJournalStorage.JournalIdPropertyName], Assert.IsType(context.Native.Select)); } } @@ -279,9 +547,9 @@ public async Task AzureBlobListAsync_EquivalentConfigurationStreamsSameCatalog(s var result = await DrainAsync(context.Catalog.ListAsync( new() { Prefix = new("jobs/shards") }, TestContext.Current.CancellationToken)); - Assert.Equal(["jobs/shards/z", "jobs/shards/a", "jobs/shards/child"], result); + Assert.Equal(["jobs/shards/a", "jobs/shards/child", "jobs/shards/z", "jobs/shards2"], result); Assert.Equal(2, context.Native.Requests.Count); - Assert.All(context.Native.Requests, request => Assert.Equal("jobs/shards", request.Prefix)); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/jobs/shards", request.Prefix)); } [Fact] @@ -289,33 +557,401 @@ public async Task AzureBlobListAsync_FiltersAppendWalEntries() { BlobItem[] blobs = [ - Blob("tenant/z/wal"), Blob("tenant/block/wal", BlobType.Block), - Blob("tenant/page/wal", BlobType.Page), Blob("tenant/chk.1"), - Blob("tenant/child/wal"), Blob("tenant/case/WAL"), Blob("/wal"), - Blob(" \t /wal"), Blob("tenant2/wal"), Blob("tenant/wal"), + Blob("wal/tenant/z"), Blob("wal/tenant/block", BlobType.Block), + Blob("wal/tenant/page", BlobType.Page), Blob("checkpoints/tenant/snapshot", BlobType.Block), + Blob("wal/tenant/child"), Blob("WAL/tenant/case"), Blob("wal/"), + Blob("wal/ \t "), Blob("wal/tenant2"), Blob("wal/tenant"), ]; await using var context = await CreateAsync("AzureBlob", [], blobs: blobs); Assert.Equal( - ["tenant/z", "tenant/child", "tenant"], + ["tenant", "tenant/child", "tenant/z", "tenant2"], await DrainAsync(context.Catalog.ListAsync(new() { Prefix = new("tenant") }, TestContext.Current.CancellationToken))); - Assert.Equal(8, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.Equal(6, context.Native.Requests.Sum(request => request.ResultCount)); + } + + [Theory] + [InlineData(null)] + [InlineData("tenant/")] + public async Task AzureBlobListAsync_SeparatePrefixExcludesRetainedCheckpointsFromEveryPage(string? prefix) + { + string[] ids = ["tenant", "tenant/a", "tenant/b"]; + var blobs = ids.Select(id => Blob($"wal/{id}")) + .Concat(ids.SelectMany(id => Enumerable.Range(0, 256) + .Select(index => Blob($"checkpoints/{id}/{index}", BlobType.Block)))) + .Append(Blob("unrelated/tenant")) + .ToArray(); + await using var context = await CreateAsync("AzureBlob", [], blobs: blobs); + var expected = prefix is null ? ids : ids[1..]; + + Assert.Equal(expected, await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = prefix is null ? default : new(prefix) }, TestContext.Current.CancellationToken))); + Assert.Equal(expected.Length, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.Equal(prefix is null ? 2 : 1, context.Native.Requests.Count); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/" + prefix, request.Prefix)); + } + + [Fact] + public async Task AzureBlobListAsync_WalPrefixRequiresANonWhitespaceJournalId() + { + await using var context = await CreateAsync("AzureBlob", [], + blobs: [Blob("wal/"), Blob("wal/ \t "), Blob("wal/wal/tenant"), Blob("wal/tenant/wal")]); + + Assert.Equal(["tenant/wal", "wal/tenant"], await DrainAsync( + context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken))); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/", request.Prefix)); + } + + [Fact] + public async Task AzureBlobListAsync_MaxIdStopsBeforeFetchingFutureTail() + { + await using var context = await CreateAsync("AzureBlob", ["a", "b", "z", "zz", "zzz"]); + context.Native.Failure = new InvalidOperationException("future tail must not be requested"); + context.Native.FailureAtRequest = 3; + context.Native.EmptyFirstPage = true; + + Assert.Equal(["a"], await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = new("a") }, TestContext.Current.CancellationToken))); + Assert.Equal([0, 2], context.Native.Requests.Select(request => request.ResultCount)); + Assert.Equal(1, context.Native.DisposedEnumerators); + } + + [Fact] + public async Task AzureBlobListAsync_MaxIdIncludesExactJournalAndExcludesCheckpointsBeforePagination() + { + await using var context = await CreateAsync("AzureBlob", [], + blobs: [Blob("checkpoints/a/1", BlobType.Block), Blob("wal/a"), Blob("checkpoints/b/1", BlobType.Block), Blob("wal/b"), Blob("wal/z")]); + context.Native.Failure = new InvalidOperationException("future tail must not be requested"); + context.Native.FailureAtRequest = 3; + + Assert.Equal(["a"], await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = new("a") }, TestContext.Current.CancellationToken))); + Assert.Equal(2, Assert.Single(context.Native.Requests).ResultCount); + Assert.Equal("wal/", context.Native.Requests[0].Prefix); + Assert.Equal(1, context.Native.DisposedEnumerators); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("S3")] + public async Task OrderedListAsync_TimePrefixedNamespaceStopsBeforeFuturePages(string kind) + { + const string prefix = "jobs/shards/"; + const string overdue = prefix + "20250101T0000000000000Z-11111111111111111111111111111111"; + const string due = prefix + "20260909T2100000000000Z-22222222222222222222222222222222"; + const string maximum = prefix + "20260909T2100000000000Z~"; + var ids = new List { overdue, due, maximum }; + ids.AddRange(Enumerable.Range(0, 256).Select(index => + $"{prefix}20260909T2100010000000Z-{index.ToString("x32", CultureInfo.InvariantCulture)}")); + ids.Add("jobs/shards"); + + await using var context = await CreateAsync(kind, ids.ToArray(), configureS3: options => options.UseOrderedListing = true); + context.Native.Failure = new InvalidOperationException("future tail must not be requested"); + context.Native.FailureAtRequest = 3; + + var result = await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new(prefix), MaxId = new(maximum) }, TestContext.Current.CancellationToken)); + + Assert.Equal([overdue, due, maximum], result); + Assert.Equal(2, context.Native.Requests.Count); + Assert.All(context.Native.Requests, request => + { + Assert.Equal("wal/" + prefix, request.Prefix); + Assert.Equal(kind == "AzureBlob" ? "wal/" + prefix : null, request.LowerStart); + Assert.Equal(kind == "AzureBlob" ? 5000 : 1000, request.Maximum); + Assert.Equal(2, request.ResultCount); + }); + Assert.Equal(kind == "AzureBlob" ? 1 : 0, context.Native.DisposedEnumerators); + } + + [Theory] + [InlineData("Append")] + [InlineData("Block")] + [InlineData("Missing")] + public async Task AzureBlobListAsync_RawPrefixFiltersWalTypeAndDoesNotDuplicateIds(string exactWal) + { + var blobs = new List { Blob("wal/tenant/wal-child"), Blob("wal/tenant/z") }; + if (exactWal != "Missing") + { + blobs.Add(Blob("wal/tenant", exactWal == "Append" ? BlobType.Append : BlobType.Block)); + } + + await using var context = await CreateAsync("AzureBlob", [], blobs: blobs.ToArray()); + var result = await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/z") }, TestContext.Current.CancellationToken)); + + Assert.Equal(exactWal == "Append" ? ["tenant", "tenant/wal-child", "tenant/z"] : new[] { "tenant/wal-child", "tenant/z" }, result); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/tenant", request.Prefix)); + } + + [Theory] + [InlineData("a!")] + [InlineData("a/0")] + [InlineData("a/w")] + [InlineData("a/wal!")] + [InlineData("\u00e9")] + public async Task AzureBlobListAsync_BoundedPrefixPreservesShorterDescendants(string suffix) + { + string[] ids = ["tenant", "tenant/a", "tenant/a!", "tenant/a/0", "tenant/a/w", "tenant/a/wal!", "tenant/\u00e9", "tenant/\uffff"]; + var maximum = "tenant/" + suffix; + await using var context = await CreateAsync("AzureBlob", ids); + + AssertMembership(ids.Where(id => string.CompareOrdinal(id, maximum) <= 0).ToArray(), + await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AzureBlobListAsync_BoundedPrefixDisposalOrCancellationStopsListing(bool cancel) + { + await using var context = await CreateAsync("AzureBlob", ["tenant", "tenant/a"]); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + await using (var enumerator = context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/z") }, cancellation.Token).GetAsyncEnumerator(cancellation.Token)) + { + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal("tenant", enumerator.Current.Id.Value); + if (cancel) + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => enumerator.MoveNextAsync().AsTask()); + Assert.Equal(cancellation.Token, exception.CancellationToken); + } + } + + Assert.Equal("wal/tenant", Assert.Single(context.Native.Requests).Prefix); + Assert.Equal(1, context.Native.DisposedEnumerators); + } + + [Fact] + public async Task AzureBlobListAsync_BoundedPrefixCancellationAfterEmptyPagePropagates() + { + await using var context = await CreateAsync("AzureBlob", ["tenant/a"]); + context.Native.EmptyFirstPage = true; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + context.Native.BeforeResponse = cancellation.Cancel; + + var exception = await Assert.ThrowsAnyAsync(() => DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/z") }, cancellation.Token))); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + var request = Assert.Single(context.Native.Requests); + Assert.Equal("wal/tenant", request.Prefix); + Assert.Equal(0, request.ResultCount); + } + + [Fact] + public async Task AzureBlobListAsync_BoundedPrefixCrossesEmptyAndFilteredPages() + { + await using var context = await CreateAsync("AzureBlob", ["tenant", "tenant/a", "tenant/b"]); + context.Native.EmptyFirstPage = true; + + Assert.Equal(["tenant", "tenant/a"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/a") }, TestContext.Current.CancellationToken))); + Assert.Equal([0, 2, 1], context.Native.Requests.Select(request => request.ResultCount)); + Assert.Equal(1, context.Native.DisposedEnumerators); + } + + [Theory] + [InlineData("tenant")] + [InlineData("tenant!")] + public async Task AzureBlobListAsync_BoundedPrefixMaximumPreservesExactId(string maximum) + { + await using var context = await CreateAsync("AzureBlob", ["tenant", "tenant/a"]); + + Assert.Equal(["tenant"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + Assert.Equal("wal/tenant", Assert.Single(context.Native.Requests).Prefix); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public async Task AzureBlobListAsync_BoundedPrefixListingErrorPropagates(int failedRequest) + { + await using var context = await CreateAsync("AzureBlob", ["tenant/a", "tenant/b", "tenant/c"]); + var failure = new RequestFailedException(503, "listing failed"); + context.Native.Failure = failure; + context.Native.FailureAtRequest = failedRequest; + + Assert.Same(failure, await Assert.ThrowsAsync(() => DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MaxId = new("tenant/z") }, TestContext.Current.CancellationToken)))); + Assert.Equal(failedRequest, context.Native.Requests.Count); + } + + [Theory] + [InlineData("AzureBlob", false)] + [InlineData("AzureBlob", true)] + [InlineData("S3", false)] + [InlineData("S3", true)] + public async Task OrderedListAsync_SeeksMinimumBeforeFetchingAnyPageAndIncludesBothEndpoints(string kind, bool includePrefix) + { + const string day = "jobs/shards/20260909"; + const string common = day + "T1200000000000Z-"; + const string minimum = common + "80000000000000000000000000000000"; + const string maximum = common + "c0000000000000000000000000000000"; + var ids = Enumerable.Range(0, 256).Select(index => common + index.ToString("x32", CultureInfo.InvariantCulture)) + .Concat([minimum, maximum, common + "f0000000000000000000000000000000"]).ToArray(); + await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + + Assert.Equal([minimum, maximum], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = includePrefix ? new(day) : default, MinId = new(minimum), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + Assert.Equal(2, context.Native.Requests.Count); + Assert.Equal([2, 1], context.Native.Requests.Select(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => + { + Assert.Equal("wal/" + common, request.Prefix); + Assert.Equal(kind == "AzureBlob" ? "wal/" + minimum : request.Cursor is null ? "wal/" + minimum[..^1] : null, request.LowerStart); + }); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("S3")] + public async Task OrderedListAsync_PushesPartialDayPrefixWithoutAddingSeparator(string kind) + { + const string prefix = "jobs/shards/202609"; + string[] ids = ["jobs/shards/20260831-a", prefix + "01-a", prefix + "09-b", "jobs/shards/20261001-a"]; + await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + + Assert.Equal(ids[1..3], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new(prefix) }, TestContext.Current.CancellationToken))); + var request = Assert.Single(context.Native.Requests); + Assert.Equal("wal/" + prefix, request.Prefix); + Assert.Equal(kind == "AzureBlob" ? "wal/" + prefix : null, request.LowerStart); + Assert.Equal(2, request.ResultCount); + } + + [Theory] + [InlineData("journals", null, null, "wal/journals", null)] + [InlineData("journals", "journal", "journals/zeta", "wal/journals", null)] + [InlineData("journals", "journals", "journals/zeta", "wal/journals", null)] + [InlineData("journals", "journals/alpha", "journals/alpha", "wal/journals/alpha", null)] + [InlineData("journals/", "journals/alpha", "journals/zeta", "wal/journals/", "wal/journals/alph")] + public async Task S3ListAsync_OrderedSeekOnlyNarrowsBeyondNativePrefix( + string prefix, string? minId, string? maxId, string nativePrefix, string? startAfter) + { + string[] ids = ["before/a", "journals/alpha", "journals/zeta", "other/beta"]; + await using var context = await CreateAsync("S3", ids, configureS3: options => options.UseOrderedListing = true); + + var result = await DrainAsync(context.Catalog.ListAsync( + new() + { + Prefix = new(prefix), + MinId = minId is null ? default : new(minId), + MaxId = maxId is null ? default : new(maxId) + }, + TestContext.Current.CancellationToken)); + + Assert.Equal(maxId == "journals/alpha" ? new[] { "journals/alpha" } : ["journals/alpha", "journals/zeta"], result); + Assert.All(context.Native.Requests, request => + { + Assert.Equal(nativePrefix, request.Prefix); + Assert.Equal(request.Cursor is null ? startAfter : null, request.LowerStart); + }); + Assert.Equal(result.Count, context.Native.Requests.Sum(request => request.ResultCount)); + } + + [Theory] + [InlineData("AzureBlob")] + [InlineData("S3")] + public async Task OrderedListAsync_UnicodeBoundsDoNotUseUnsafeNativeOrdering(string kind) + { + string[] ids = ["a", "\ud800\udc00", "\ue000", "\uffff"]; + await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + + AssertMembership(ids[1..3], await DrainAsync(context.Catalog.ListAsync( + new() { MinId = new(ids[1]), MaxId = new(ids[2]) }, TestContext.Current.CancellationToken))); + Assert.Equal(2, context.Native.Requests.Count); + Assert.Equal(4, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => Assert.Null(request.LowerStart)); + } + + [Theory] + [InlineData("a!")] + [InlineData("a/0")] + [InlineData("a/w")] + [InlineData("a/wal!")] + public async Task AzureBlobListAsync_MaxIdPreservesShorterIdsInJournalOrder(string maximum) + { + string[] ids = ["a", "a!", "a/0", "a/w", "a/wal!", "z"]; + await using var context = await CreateAsync("AzureBlob", ids); + AssertMembership(ids.Where(id => string.CompareOrdinal(id, maximum) <= 0).ToArray(), + await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + } + + [Fact] + public async Task AzureBlobListAsync_NonAsciiMaxIdFiltersWithoutLexicalCutoff() + { + await using var context = await CreateAsync("AzureBlob", ["a", "\u00e9", "\uffff"]); + Assert.Equal(["a", "\u00e9"], await DrainAsync(context.Catalog.ListAsync( + new() { MaxId = new("\u00e9") }, TestContext.Current.CancellationToken))); + Assert.Equal(2, context.Native.Requests.Count); + Assert.Equal(3, context.Native.Requests.Sum(request => request.ResultCount)); } [Fact] - public async Task AzureTableListAsync_PreservesCanonicalAndReversibleLegacyIds() + public async Task AzureTableListAsync_PushesBoundsIntoIndexedPartitionKeyQuery() + { + const string prefix = "jobs/shards/202609"; + const string due = prefix + "09-a"; + const string maximum = prefix + "09~"; + const string future = prefix + "10-a"; + await using var context = await CreateAsync("AzureTable", [prefix + "08-a", due, maximum, future, "other/id"]); + + Assert.Equal([due, maximum], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new(prefix), MinId = new(due), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + Assert.Equal(2, Assert.Single(context.Native.Requests).ResultCount); + Assert.Contains($"PartitionKey ge '{AzureTableJournalStorageOptions.GetDefaultPartitionKey(new(due))}'", context.Native.Filter); + Assert.Contains($"PartitionKey le '{AzureTableJournalStorageOptions.GetDefaultPartitionKey(new(maximum))}'", context.Native.Filter); + Assert.Contains($"PartitionKey lt '{AzureTableJournalStorageOptions.GetDefaultPartitionKey(new(prefix))}G'", context.Native.Filter); + Assert.DoesNotContain("JournalId ", context.Native.Filter); + Assert.DoesNotContain(" or ", context.Native.Filter); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AzureTableListAsync_CustomMappingFiltersCanonicalIdsAndEscapesQueryLiterals(bool includeMetadata) + { + const string prefix = "tenant'one"; + string[] ids = [prefix, prefix + "/a", prefix + "/a/child", prefix + "/z"]; + await using var context = await CreateAsync("AzureTable", [], + headers: ids.Select((id, index) => Header($"opaque-{index}", id)).ToArray(), + configureTable: options => options.GetPartitionKey = id => $"opaque-{Array.IndexOf(ids, id.Value)}"); + + var entries = await DrainEntriesAsync(context.Catalog.ListAsync( + new() { Prefix = new(prefix), MaxId = new(prefix + "/a/child"), IncludeMetadata = includeMetadata }, + TestContext.Current.CancellationToken)); + + Assert.Equal(ids[..3], entries.Select(entry => entry.Id.Value)); + Assert.All(entries, entry => Assert.Equal(includeMetadata, entry.Metadata is not null)); + AssertMetadataProjection(context.Native, "AzureTable", includeMetadata); + Assert.Equal(0, context.Native.MetadataRequests); + Assert.Contains("JournalId ge 'tenant''one'", context.Native.Filter); + Assert.Contains("JournalId le 'tenant''one/a/child'", context.Native.Filter); + Assert.DoesNotContain("PartitionKey ", context.Native.Filter); + Assert.Equal(3, context.Native.Requests.Sum(request => request.ResultCount)); + } + + [Fact] + public async Task AzureTableListAsync_OnlyCanonicalHeaderIdsAreReturned() { TableEntity[] headers = [ - Header("opaque!1", "tenant/z"), Header("other%2Fid", "tenant/a"), - Header("legacy%2F%C3%A9"), Header("legacy%2fchild"), Header("%41"), - Header("bad%ZZ"), Header("fallback%2Fid", ""), Header("%20%09"), + Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant/z")), "tenant/z"), + Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("tenant/a")), "tenant/a"), + Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new("missing/id"))), + Header("invalid", ""), Header("whitespace", " \t "), new("orphan", "data") { [AzureTableJournalStorage.JournalIdPropertyName] = "ignored" }, ]; await using var context = await CreateAsync("AzureTable", [], headers: headers); Assert.Equal( - ["tenant/z", "tenant/a", "legacy/\u00E9"], + ["tenant/z", "tenant/a"], await DrainAsync(context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken))); - Assert.Equal(8, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.Equal(5, context.Native.Requests.Sum(request => request.ResultCount)); } [Fact] @@ -324,8 +960,8 @@ public async Task S3ListAsync_UsesCanonicalMappingAfterPrefixFilter() var mapped = new List(); string[] keys = [ - "current/tenant/z/wal", "legacy/tenant/z/wal", "current/tenant/a/chk.1", - "current/tenant/a/wal", "current/other/x/wal", "invalid/wal", "default/wal", + "wal/current/tenant/z", "wal/current/tenant/alias", "checkpoints/current/tenant/a/1", + "wal/current/tenant/a", "wal/current/other/x", "wal/invalid", "wal/default", ]; await using var context = await CreateAsync("S3", [], keys: keys, configureS3: options => { @@ -334,8 +970,10 @@ public async Task S3ListAsync_UsesCanonicalMappingAfterPrefixFilter() mapped.Add(id.Value); return $"current/{id.Value}"; }; + options.GetObjectKeyPrefix = id => $"current/{id.Value}"; options.TryParseJournalId = value => value switch { + "current/tenant/alias" => new JournalId("tenant/z"), "invalid" => null, "default" => default(JournalId), _ => new JournalId(value[(value.IndexOf('/') + 1)..]), @@ -346,6 +984,127 @@ public async Task S3ListAsync_UsesCanonicalMappingAfterPrefixFilter() ["tenant/z", "tenant/a"], await DrainAsync(context.Catalog.ListAsync(new() { Prefix = new("tenant") }, TestContext.Current.CancellationToken))); Assert.Equal(["tenant/z", "tenant/z", "tenant/a"], mapped); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/current/", request.Prefix)); + } + + [Fact] + public async Task S3ListAsync_MaxIdDoesNotStopUnorderedDirectoryBucketTraversal() + { + await using var context = await CreateAsync("S3", ["tenant/z", "tenant/y", "tenant/a", "tenant/b"]); + context.Native.EmptyFirstPage = true; + + Assert.Equal(["tenant/a", "tenant/b"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant"), MinId = new("tenant/a"), MaxId = new("tenant/b") }, TestContext.Current.CancellationToken))); + Assert.Equal([0, 2, 2], context.Native.Requests.Select(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => Assert.Null(request.LowerStart)); + } + + [Theory] + [InlineData("tenant", "tenant-a", "tenant-b", "wal/")] + [InlineData("jobs/shards/202609", "jobs/shards/20260909-a", "jobs/shards/20260909-b", "wal/jobs/shards/")] + public async Task S3ListAsync_DirectoryModeWidensPartialPrefixWithoutLosingBoundedIds( + string prefix, string minimum, string maximum, string? nativePrefix) + { + string[] ids = [maximum + "-future", minimum[..^1] + "0", maximum, minimum, minimum + "-child"]; + await using var context = await CreateAsync("S3", ids, configureS3: options => options.UseOrderedListing = false); + + Assert.Equal([maximum, minimum, minimum + "-child"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new(prefix), MinId = new(minimum), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); + Assert.Equal(5, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => + { + Assert.Equal(nativePrefix, request.Prefix); + Assert.Null(request.LowerStart); + }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task S3ListAsync_CustomMappingNeverUsesNativeIdentityBounds(bool ordered) + { + string[] ids = ["tenant/z", "tenant/a", "tenant/b", "tenant/0"]; + var mapping = ids.Select((id, index) => (id, key: $"current/{index}/{id}")).ToDictionary(item => item.id, item => item.key); + await using var context = await CreateAsync("S3", [], keys: ids.Select(id => "wal/" + mapping[id]).ToArray(), configureS3: options => + { + options.UseOrderedListing = ordered; + options.GetObjectKey = id => mapping[id.Value]; + options.GetObjectKeyPrefix = _ => "current/"; + options.TryParseJournalId = key => new JournalId(key["current/0/".Length..]); + }); + + Assert.Equal(["tenant/a", "tenant/b"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/"), MinId = new("tenant/a"), MaxId = new("tenant/b") }, TestContext.Current.CancellationToken))); + Assert.Equal(4, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => + { + Assert.Equal("wal/current/", request.Prefix); + Assert.Null(request.LowerStart); + }); + } + + [Theory] + [InlineData(false, "wal/current/tenant/")] + [InlineData(true, "wal/current/tenant/a")] + public async Task S3ListAsync_CustomMapperAcceptsRawPartialPrefixes(bool ordered, string nativePrefix) + { + await using var context = await CreateAsync("S3", [], keys: + ["wal/current/tenant/aa", "wal/current/tenant/ab", "wal/current/tenant/ac"], configureS3: options => + { + options.UseOrderedListing = ordered; + options.GetObjectKey = id => "current/" + id.Value; + options.GetObjectKeyPrefix = prefix => "current/" + prefix.Value; + options.TryParseJournalId = key => new JournalId(key["current/".Length..]); + }); + + Assert.Equal(["tenant/aa", "tenant/ab"], await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/a"), MinId = new("tenant/aa"), MaxId = new("tenant/ab") }, TestContext.Current.CancellationToken))); + Assert.Equal(3, context.Native.Requests.Sum(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => + { + Assert.Equal(nativePrefix, request.Prefix); + Assert.Null(request.LowerStart); + }); + } + + [Fact] + public async Task S3ListAsync_CustomMappingWithBoundsOnlyDoesNotRequirePrefixMapper() + { + await using var context = await CreateAsync("S3", [], keys: ["wal/current/tenant/a", "wal/current/tenant/z"], configureS3: options => + { + options.UseOrderedListing = true; + options.GetObjectKey = id => "current/" + id.Value; + options.TryParseJournalId = key => new JournalId(key["current/".Length..]); + }); + + Assert.Equal(["tenant/a"], await DrainAsync(context.Catalog.ListAsync( + new() { MinId = new("tenant/a"), MaxId = new("tenant/b") }, TestContext.Current.CancellationToken))); + var request = Assert.Single(context.Native.Requests); + Assert.Equal("wal/", request.Prefix); + Assert.Null(request.LowerStart); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task S3ListAsync_CustomMappingRequiresValidExplicitPrefix(string? mappedPrefix) + { + await using var context = await CreateAsync("S3", [], keys: ["wal/current/tenant"], configureS3: options => + { + options.GetObjectKey = id => "current/" + id.Value; + options.TryParseJournalId = key => new JournalId(key["current/".Length..]); + if (mappedPrefix is not null) + { + options.GetObjectKeyPrefix = _ => mappedPrefix; + } + }); + + var failure = await Assert.ThrowsAsync(() => DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant") }, TestContext.Current.CancellationToken))); + Assert.Contains(nameof(S3JournalStorageOptions.GetObjectKeyPrefix), failure.Message); + Assert.Empty(context.Native.Requests); + Assert.Equal(["tenant"], await DrainAsync(context.Catalog.ListAsync(cancellationToken: TestContext.Current.CancellationToken))); + Assert.Equal("wal/", Assert.Single(context.Native.Requests).Prefix); } [Theory] @@ -391,25 +1150,65 @@ private static void AssertMembership(string[] expected, IReadOnlyCollection> DrainAsync(IAsyncEnumerable source) + private static async Task> DrainAsync(IAsyncEnumerable source) { var result = new List(); - await foreach (var id in source) + await foreach (var entry in source) + { + result.Add(entry.Id.Value); + } + + return result; + } + + private static async Task> DrainEntriesAsync(IAsyncEnumerable source) + { + var result = new List(); + await foreach (var entry in source) { - result.Add(id.Value); + result.Add(entry); } return result; } + private static void AssertMetadataProjection(NativeState state, string kind, bool includeMetadata) + { + if (kind == "AzureBlob") + { + Assert.Equal(includeMetadata ? BlobTraits.Metadata : BlobTraits.None, Assert.Single(state.BlobTraits)); + } + else + { + Assert.Equal(includeMetadata + ? [AzureTableJournalStorage.JournalIdPropertyName, AzureTableJournalStorage.FormatPropertyName, + AzureTableJournalStorage.MetadataPropertyName, nameof(TableEntity.Timestamp)] + : new[] { AzureTableJournalStorage.JournalIdPropertyName }, state.Select); + } + } + private static BlobItem Blob(string name, BlobType type = BlobType.Append) => BlobsModelFactory.BlobItem( name: name, deleted: false, - properties: BlobsModelFactory.BlobItemProperties(accessTierInferred: false, blobType: type)); + properties: BlobsModelFactory.BlobItemProperties(accessTierInferred: false, blobType: type, eTag: new ETag("listed")), + metadata: new Dictionary + { + [AzureBlobJournalStorage.FormatMetadataKey] = "test", + [AzureBlobJournalStorage.CheckpointMetadataKey] = "checkpoints/previous/snapshot", + [AzureBlobJournalStorage.CheckpointOffsetMetadataKey] = "0", + [AzureBlobJournalStorage.WalGenerationMetadataKey] = "generation", + ["owner"] = "alice", + }); private static TableEntity Header(string partition, string? id = null) { - var result = new TableEntity(partition, AzureTableJournalStorage.HeaderRowKey); + var result = new TableEntity(partition, AzureTableJournalStorage.HeaderRowKey) + { + ETag = new ETag("listed"), + Timestamp = DateTimeOffset.UnixEpoch, + [AzureTableJournalStorage.FormatPropertyName] = "test", + [AzureTableJournalStorage.MetadataPropertyName] = """{"owner":"alice"}""", + }; if (id is not null) { result[AzureTableJournalStorage.JournalIdPropertyName] = id; @@ -421,9 +1220,10 @@ private static TableEntity Header(string partition, string? id = null) private static async Task CreateAsync( string kind, string[] ids, bool initialize = true, string? blobLayout = null, BlobItem[]? blobs = null, TableEntity[]? headers = null, string[]? keys = null, - Action? configureS3 = null) + Action? configureS3 = null, + Action? configureTable = null) { - var context = new ProviderContext(kind, ids, blobLayout, blobs, headers, keys, configureS3); + var context = new ProviderContext(kind, ids, blobLayout, blobs, headers, keys, configureS3, configureTable); try { if (initialize) @@ -457,7 +1257,8 @@ private sealed class ProviderContext : IAsyncDisposable public ProviderContext( string kind, string[] ids, string? blobLayout, BlobItem[]? blobs, - TableEntity[]? headers, string[]? keys, Action? configureS3) + TableEntity[]? headers, string[]? keys, Action? configureS3, + Action? configureTable) { var services = new ServiceCollection(); services.AddKeyedSingleton("test", new TestFormat()); @@ -469,12 +1270,12 @@ public ProviderContext( Provider = new VolatileJournalStorageProvider(); break; case "AzureBlob": - var container = new FakeContainer(Native, blobs ?? ids.Select(id => Blob($"{id}/wal")).ToArray()); + var container = new FakeContainer(Native, blobs ?? ids.Select(id => Blob($"wal/{id}")).ToArray()); var blobOptions = new AzureBlobJournalStorageOptions { ContainerName = "journals" }; blobOptions.ConfigureBlobServiceClient(_ => Task.FromResult(new FakeBlobService(container))); if (blobLayout == "EquivalentDelegate") { - blobOptions.GetWalBlobName = id => $"{id.Value}/wal"; + blobOptions.GetWalBlobName = id => $"wal/{id.Value}"; } else if (blobLayout == "WrapperFactory") { @@ -485,8 +1286,9 @@ public ProviderContext( Options.Create(blobOptions), manager, _services, NullLogger.Instance); break; case "AzureTable": - var table = new FakeTable(Native, headers ?? ids.Select((id, index) => Header($"mapped-{index}", id)).ToArray()); + var table = new FakeTable(Native, headers ?? ids.Select(id => Header(AzureTableJournalStorageOptions.GetDefaultPartitionKey(new(id)), id)).ToArray()); var tableOptions = new AzureTableJournalStorageOptions { TableName = "journals" }; + configureTable?.Invoke(tableOptions); tableOptions.ConfigureTableServiceClient(_ => Task.FromResult(new FakeTableService(table))); Provider = new AzureTableJournalStorageProvider( Options.Create(tableOptions), manager, _services, NullLogger.Instance); @@ -495,14 +1297,40 @@ public ProviderContext( _client = Substitute.For(); _client.HeadBucketAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(new HeadBucketResponse())); - var objects = (keys ?? ids.Select(id => $"{id}/wal").ToArray()).Select(key => new S3Object { Key = key }).ToArray(); + var s3Options = new S3JournalStorageOptions { BucketName = "journals", S3Client = _client, UseOrderedListing = false }; + configureS3?.Invoke(s3Options); + var objects = (keys ?? ids.Select(id => $"wal/{id}").ToArray()).Select(key => new S3Object { Key = key }).ToArray(); + var continuationRecords = new Dictionary(); _client.ListObjectsV2Async(Arg.Any(), Arg.Any()).Returns(call => { var request = call.Arg(); Assert.Equal("journals", request.BucketName); - Assert.Null(request.Prefix); - Assert.Null(request.StartAfter); - var page = Native.Fetch(objects, request.ContinuationToken, request.MaxKeys, request.Prefix, call.Arg()); + S3Object[] records; + if (request.ContinuationToken is { } token) + { + Assert.Null(request.StartAfter); + records = continuationRecords[token]; + } + else + { + var matching = objects.Where(item => (request.Prefix is null + || item.Key.StartsWith(request.Prefix, StringComparison.Ordinal)) + && (request.StartAfter is null || string.CompareOrdinal(item.Key, request.StartAfter) > 0)); + if (s3Options.UseOrderedListing) + { + matching = matching.OrderBy(item => item.Key, StringComparer.Ordinal); + } + + records = matching.ToArray(); + } + + var page = Native.Fetch(records, request.ContinuationToken, request.MaxKeys, request.Prefix, + call.Arg(), request.StartAfter); + if (page.NextCursor is { } nextToken) + { + continuationRecords[nextToken] = records; + } + return Task.FromResult(new ListObjectsV2Response { S3Objects = page.Values.ToList(), @@ -510,8 +1338,6 @@ public ProviderContext( NextContinuationToken = page.NextCursor, }); }); - var s3Options = new S3JournalStorageOptions { BucketName = "journals", S3Client = _client }; - configureS3?.Invoke(s3Options); Provider = new S3JournalStorageProvider( Options.Create(s3Options), manager, _services, NullLogger.Instance); break; @@ -530,6 +1356,10 @@ public ProviderContext( public IJournalStorageProvider Provider { get; } public IJournalStorageCatalog Catalog { get; } + public void AssertNoS3MetadataRequests() + => Assert.DoesNotContain(_client!.ReceivedCalls(), call => call.GetMethodInfo().Name + is nameof(IAmazonS3.GetObjectMetadataAsync) or nameof(IAmazonS3.GetObjectAsync)); + public Task InitializeAsync() => Provider is VolatileJournalStorageProvider ? Task.CompletedTask : _lifecycle.StartAsync(TestContext.Current.CancellationToken); @@ -547,7 +1377,7 @@ public async ValueTask DisposeAsync() } private sealed record NativeRequest( - string? Cursor, int? Maximum, string? Prefix, CancellationToken CancellationToken, int ResultCount, string? NextCursor); + string? Cursor, int? Maximum, string? Prefix, CancellationToken CancellationToken, int ResultCount, string? NextCursor, string? LowerStart); private sealed record NativePage(IReadOnlyList Values, string? NextCursor); private sealed class NativeState @@ -563,13 +1393,16 @@ private sealed class NativeState public Exception? SetupFailure { get; set; } public string? Filter { get; set; } public string[]? Select { get; set; } + public List BlobTraits { get; } = []; + public int MetadataRequests { get; set; } + public ETag? CurrentETag { get; set; } - public NativePage Fetch(T[] records, string? cursor, int? maximum, string? prefix, CancellationToken cancellationToken) + public NativePage Fetch(T[] records, string? cursor, int? maximum, string? prefix, CancellationToken cancellationToken, string? lowerStart = null) { var offset = cursor is null ? 0 : int.Parse(cursor.AsSpan("native:".Length), CultureInfo.InvariantCulture); - var count = cursor is null && EmptyFirstPage ? 0 : Math.Min(2, records.Length - offset); + var count = cursor is null && EmptyFirstPage ? 0 : Math.Min(Math.Min(2, maximum ?? 2), records.Length - offset); var next = offset + count < records.Length ? $"native:{offset + count}" : null; - Requests.Add(new(cursor, maximum, prefix, cancellationToken, count, next)); + Requests.Add(new(cursor, maximum, prefix, cancellationToken, count, next, lowerStart)); if (Requests.Count == FailureAtRequest && Failure is { } failure) { throw failure; @@ -582,7 +1415,7 @@ public NativePage Fetch(T[] records, string? cursor, int? maximum, string? } } - private sealed class FakePageable(NativeState state, T[] records, int? maximum, string? prefix, CancellationToken token) : AsyncPageable + private sealed class FakePageable(NativeState state, T[] records, int? maximum, string? prefix, CancellationToken token, string? lowerStart = null) : AsyncPageable where T : notnull { public override async IAsyncEnumerable> AsPages(string? continuationToken = null, int? pageSizeHint = null) @@ -592,7 +1425,7 @@ public override async IAsyncEnumerable> AsPages(string? continuationToke do { await Task.CompletedTask; - var page = state.Fetch(records, continuationToken, pageSizeHint ?? maximum, prefix, token); + var page = state.Fetch(records, continuationToken, pageSizeHint ?? maximum, prefix, token, lowerStart); yield return Page.FromValues(page.Values, page.NextCursor, new FakeResponse()); continuationToken = page.NextCursor; } @@ -616,6 +1449,9 @@ public override BlobContainerClient GetBlobContainerClient(string blobContainerN private sealed class FakeContainer(NativeState state, BlobItem[] records) : BlobContainerClient { + protected override AppendBlobClient GetAppendBlobClientCore(string blobName) + => new FakeAppendBlob(state, records, blobName); + public override Task> CreateIfNotExistsAsync( PublicAccessType publicAccessType = PublicAccessType.None, IDictionary? metadata = null, @@ -631,15 +1467,40 @@ public override Task> CreateIfNotExistsAsync( BlobsModelFactory.BlobContainerInfo(new ETag("created"), DateTimeOffset.UnixEpoch), new FakeResponse())); } - public override AsyncPageable GetBlobsAsync( - BlobTraits traits = BlobTraits.None, BlobStates states = BlobStates.None, - string? prefix = null, CancellationToken cancellationToken = default) + public override AsyncPageable GetBlobsAsync(GetBlobsOptions options, CancellationToken cancellationToken = default) { - Assert.Equal(BlobTraits.None, traits); - Assert.Equal(BlobStates.None, states); + state.BlobTraits.Add(options.Traits); + Assert.Equal(BlobStates.None, options.States); return new FakePageable( - state, records.Where(item => prefix is null || item.Name.StartsWith(prefix, StringComparison.Ordinal)).ToArray(), - null, prefix, cancellationToken); + state, records.Where(item => (options.Prefix is null || item.Name.StartsWith(options.Prefix, StringComparison.Ordinal)) + && (options.StartFrom is null || string.CompareOrdinal(item.Name, options.StartFrom) >= 0)) + .OrderBy(item => item.Name, StringComparer.Ordinal) + .Select(item => BlobsModelFactory.BlobItem( + name: item.Name, deleted: false, properties: item.Properties, + metadata: options.Traits.HasFlag(BlobTraits.Metadata) + ? new Dictionary(item.Metadata) : null)) + .ToArray(), + null, options.Prefix, cancellationToken, options.StartFrom); + } + } + + private sealed class FakeAppendBlob(NativeState state, BlobItem[] records, string name) : AppendBlobClient + { + public override Task> GetPropertiesAsync( + BlobRequestConditions conditions = default!, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + state.MetadataRequests++; + var item = Assert.Single(records, item => item.Name == name); + var eTag = state.CurrentETag ?? item.Properties.ETag!.Value; + if (conditions?.IfMatch is { } expected && expected != eTag) + { + throw new RequestFailedException(412, "The metadata snapshot is stale."); + } + + return Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobProperties(eTag: eTag, blobType: BlobType.Append, metadata: item.Metadata), + new FakeResponse())); } } @@ -661,6 +1522,22 @@ public override TableClient GetTableClient(string tableName) private sealed class FakeTable(NativeState state, TableEntity[] records) : TableClient { + public override Task> GetEntityAsync( + string partitionKey, string rowKey, IEnumerable? select = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + state.MetadataRequests++; + var entity = Assert.Single(records, entity => entity.PartitionKey == partitionKey && entity.RowKey == rowKey); + var snapshot = new TableEntity(new Dictionary(entity)) + { + PartitionKey = partitionKey, + RowKey = rowKey, + ETag = state.CurrentETag ?? entity.ETag, + Timestamp = entity.Timestamp, + }; + return Task.FromResult(Response.FromValue((T)(ITableEntity)snapshot, new FakeResponse())); + } + public override Task> CreateIfNotExistsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Response.FromValue(new TableItem("journals"), new FakeResponse())); @@ -671,20 +1548,96 @@ public override AsyncPageable QueryAsync( state.Filter = filter; state.Select = select?.ToArray(); Assert.Equal(1000, maxPerPage); - Assert.Equal(TableClient.CreateQueryFilter($"RowKey eq {AzureTableJournalStorage.HeaderRowKey}"), filter); - Assert.Equal([AzureTableJournalStorage.JournalIdPropertyName], Assert.IsType(state.Select)); - var values = records.Where(entity => entity.RowKey == AzureTableJournalStorage.HeaderRowKey).Select(entity => + Assert.NotNull(filter); + Assert.NotNull(state.Select); + var values = records.Where(entity => MatchesFilter(entity, filter)).Select(entity => { var projected = new TableEntity(entity.PartitionKey, entity.RowKey); - if (entity.TryGetValue(AzureTableJournalStorage.JournalIdPropertyName, out var value)) + foreach (var property in state.Select) { - projected[AzureTableJournalStorage.JournalIdPropertyName] = value; + if (property == nameof(TableEntity.Timestamp)) + { + projected.Timestamp = entity.Timestamp; + projected.ETag = entity.ETag; + } + else if (entity.TryGetValue(property, out var value)) + { + projected[property] = value; + } } return (T)(ITableEntity)projected; }).ToArray(); return new FakePageable(state, values, maxPerPage, null, cancellationToken); } + + private static bool MatchesFilter(TableEntity entity, string filter) + { + var tokens = Regex.Matches(filter, "'(?:[^']|'')*'|[()]|[^\\s()]+").Select(match => match.Value).ToArray(); + var index = 0; + var result = ParseOr(); + Assert.Equal(tokens.Length, index); + return result; + + bool ParseOr() + { + var value = ParseAnd(); + while (index < tokens.Length && tokens[index] == "or") + { + index++; + value |= ParseAnd(); + } + + return value; + } + + bool ParseAnd() + { + var value = ParseComparison(); + while (index < tokens.Length && tokens[index] == "and") + { + index++; + value &= ParseComparison(); + } + + return value; + } + + bool ParseComparison() + { + if (tokens[index] == "(") + { + index++; + var value = ParseOr(); + Assert.Equal(")", tokens[index++]); + return value; + } + + var name = tokens[index++]; + var operation = tokens[index++]; + var expected = tokens[index++][1..^1].Replace("''", "'", StringComparison.Ordinal); + var actual = name switch + { + "PartitionKey" => entity.PartitionKey, + "RowKey" => entity.RowKey, + _ => entity.GetString(name), + }; + if (actual is null) + { + return false; + } + + var comparison = string.CompareOrdinal(actual, expected); + return operation switch + { + "eq" => comparison == 0, + "ge" => comparison >= 0, + "le" => comparison <= 0, + "lt" => comparison < 0, + _ => throw new InvalidOperationException($"Unexpected filter operator: {operation}"), + }; + } + } } private sealed class TestFormat : IJournalFormat diff --git a/test/Orleans.Journaling.Tests/S3JournalStorageTests.cs b/test/Orleans.Journaling.Tests/S3JournalStorageTests.cs index 9031175da4f..1043049f2d6 100644 --- a/test/Orleans.Journaling.Tests/S3JournalStorageTests.cs +++ b/test/Orleans.Journaling.Tests/S3JournalStorageTests.cs @@ -189,8 +189,8 @@ public async Task ListAsync_CustomObjectKeyMapping_FiltersParsedJournalIds() IsTruncated = false, S3Objects = [ - new S3Object { Key = "tenant/journals/alpha/wal" }, - new S3Object { Key = "tenant/other/beta/wal" }, + new S3Object { Key = "wal/tenant/journals/alpha" }, + new S3Object { Key = "wal/tenant/other/beta" }, ], }; client.ListObjectsV2Async(Arg.Any(), Arg.Any()) @@ -198,25 +198,382 @@ public async Task ListAsync_CustomObjectKeyMapping_FiltersParsedJournalIds() var options = CreateOptions(); options.S3Client = client; options.GetObjectKey = static id => $"tenant/{id.Value}"; + options.GetObjectKeyPrefix = static prefix => $"tenant/{prefix.Value}"; + options.UseOrderedListing = true; options.TryParseJournalId = static key => key.StartsWith("tenant/", StringComparison.Ordinal) ? new JournalId(key["tenant/".Length..]) : null; var provider = CreateProvider(options); await provider.InitializeAsync(CancellationToken.None); - var listed = new List(); - await foreach (var journalId in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) + var listed = new List(); + await foreach (var entry in provider.ListAsync( + new() { Prefix = new("journals"), MinId = new("journals/alpha"), MaxId = new("journals/gamma") }, + CancellationToken.None)) { - listed.Add(journalId); + listed.Add(entry); } - Assert.Equal(["journals/alpha"], listed.Select(static id => id.Value)); + Assert.Equal(["journals/alpha"], listed.Select(static entry => entry.Id.Value)); await client.Received(1).ListObjectsV2Async( - Arg.Is(request => request.Prefix == null), + Arg.Is(request => request.Prefix == "wal/tenant/journals/" && request.StartAfter == null), Arg.Any()); await provider.CloseAsync(CancellationToken.None); } + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task ListAsync_ListsOnlyWalNamespaceWithoutMetadataRequests(bool ordered, bool customMapping, bool prefixed) + { + var cancellationToken = TestContext.Current.CancellationToken; + var basePrefix = customMapping ? "tenant/" : string.Empty; + string[] objectKeys = + [ + $"wal/{basePrefix}journals/alpha", + $"wal/{basePrefix}journals/beta", + $"checkpoints/{basePrefix}journals/alpha/snapshot", + $"{basePrefix}journals/legacy/wal", + "unrelated", + ]; + var requests = new List(); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + requests.Add(request); + return Task.FromResult(new ListObjectsV2Response + { + IsTruncated = false, + S3Objects = objectKeys + .Where(key => key.StartsWith(request.Prefix, StringComparison.Ordinal)) + .Select(key => new S3Object { Key = key, ETag = "object-etag" }) + .ToList(), + }); + }); + var options = CreateOptions(); + options.S3Client = client; + options.UseOrderedListing = ordered; + var parsedKeys = new List(); + if (customMapping) + { + options.GetObjectKey = static id => $"tenant/{id.Value}"; + if (prefixed) + { + options.GetObjectKeyPrefix = static prefix => $"tenant/{prefix.Value}"; + } + + options.TryParseJournalId = key => + { + parsedKeys.Add(key); + return key.StartsWith("tenant/", StringComparison.Ordinal) + ? new JournalId(key["tenant/".Length..]) : null; + }; + } + + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + var listed = new List(); + await foreach (var entry in provider.ListAsync( + new() { Prefix = prefixed ? new("journ") : default, IncludeMetadata = true }, + cancellationToken)) + { + listed.Add(entry); + } + + Assert.Equal(["journals/alpha", "journals/beta"], listed.Select(entry => entry.Id.Value)); + Assert.All(listed, entry => Assert.Null(entry.Metadata)); + var listingRequest = Assert.Single(requests); + Assert.Equal(prefixed ? $"wal/{basePrefix}{(ordered ? "journ" : string.Empty)}" : "wal/", listingRequest.Prefix); + Assert.Null(listingRequest.StartAfter); + Assert.Null(listingRequest.ContinuationToken); + if (customMapping) + { + Assert.Equal(["tenant/journals/alpha", "tenant/journals/beta"], parsedKeys); + } + + await client.DidNotReceive().GetObjectMetadataAsync(Arg.Any(), Arg.Any()); + await client.DidNotReceive().GetObjectAsync(Arg.Any(), Arg.Any()); + await provider.CloseAsync(cancellationToken); + } + + [Theory] + [InlineData("journals/", "journals/a1", "journals/c", "wal/journals/", "wal/journals/a")] + [InlineData(null, "journals/a1", "journals/c", "wal/journals/", "wal/journals/a")] + [InlineData("journals/", "journals/a", "journals/c", "wal/journals/", null)] + [InlineData(null, "journals/a", "journals/a2", "wal/journals/a", null)] + [InlineData(null, "a", "c", "wal/", null)] + public async Task ListAsync_OrderedBoundsIncludeExactLowerId( + string? prefix, string minId, string maxId, string nativePrefix, string? startAfter) + { + var cancellationToken = TestContext.Current.CancellationToken; + // Include the shorter key used as the exclusive marker, the minimum itself, and both adjacent keys. + string[] objectKeys = [$"wal/{minId[..^1]}", $"wal/{minId[..^1]}!", $"wal/{minId}", $"wal/{minId}0", $"wal/{maxId}", $"wal/{maxId}0"]; + var requests = new List(); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + requests.Add(request); + Assert.Null(request.ContinuationToken); + return Task.FromResult(new ListObjectsV2Response + { + IsTruncated = true, + NextContinuationToken = "beyond-upper-bound", + S3Objects = objectKeys + .Where(key => key.StartsWith(request.Prefix, StringComparison.Ordinal) + && (request.StartAfter is null || string.CompareOrdinal(key, request.StartAfter) > 0)) + .Order(StringComparer.Ordinal) + .Select(key => new S3Object { Key = key }) + .ToList(), + }); + }); + var options = CreateOptions(); + options.S3Client = client; + options.UseOrderedListing = true; + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + var listed = new List(); + await foreach (var entry in provider.ListAsync( + new() { Prefix = prefix is null ? default : new(prefix), MinId = new(minId), MaxId = new(maxId) }, + cancellationToken)) + { + listed.Add(entry); + } + + Assert.Equal([minId, $"{minId}0", maxId], listed.Select(entry => entry.Id.Value)); + var listingRequest = Assert.Single(requests); + Assert.Equal(nativePrefix, listingRequest.Prefix); + Assert.Equal(startAfter, listingRequest.StartAfter); + await provider.CloseAsync(cancellationToken); + } + + [Fact] + public async Task ListAsync_UnorderedUpperBoundRetainsAllOverdueIdsAcrossPages() + { + var cancellationToken = TestContext.Current.CancellationToken; + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => Task.FromResult(call.Arg().ContinuationToken is null + ? new ListObjectsV2Response + { + S3Objects = [new S3Object { Key = "wal/journals/9999999" }], + IsTruncated = true, + NextContinuationToken = "overdue", + } + : new ListObjectsV2Response + { + S3Objects = + [ + new S3Object { Key = "wal/journals/1000000" }, + new S3Object { Key = "wal/journals/0000001" }, + ], + IsTruncated = false, + })); + var options = CreateOptions(); + options.S3Client = client; + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + var listed = new List(); + await foreach (var entry in provider.ListAsync( + new() { Prefix = new("journals/"), MaxId = new("journals/1000000") }, + cancellationToken)) + { + listed.Add(entry); + } + + Assert.Equal(["journals/1000000", "journals/0000001"], listed.Select(entry => entry.Id.Value)); + await client.Received(2).ListObjectsV2Async( + Arg.Is(request => request.Prefix == "wal/journals/" && request.StartAfter == null), + cancellationToken); + await provider.CloseAsync(cancellationToken); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ListAsync_NullObjectCollectionHonorsContinuation(bool ordered, bool hasContinuation) + { + var cancellationToken = TestContext.Current.CancellationToken; + var requests = new List(); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + requests.Add(request); + Assert.Equal(cancellationToken, call.Arg()); + return Task.FromResult(requests.Count switch + { + 1 => new ListObjectsV2Response + { + S3Objects = null, + IsTruncated = hasContinuation, + NextContinuationToken = hasContinuation ? "next" : null + }, + 2 when hasContinuation => new ListObjectsV2Response + { + S3Objects = [new S3Object { Key = "wal/journals/alpha" }], + IsTruncated = true, + NextContinuationToken = "last" + }, + 3 when hasContinuation => new ListObjectsV2Response + { + S3Objects = null, + IsTruncated = false + }, + _ => throw new InvalidOperationException("Unexpected extra listing request.") + }); + }); + var options = CreateOptions(); + options.S3Client = client; + options.UseOrderedListing = ordered; + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + + var listed = new List(); + await foreach (var entry in provider.ListAsync(new() { Prefix = new("journals/") }, cancellationToken)) + { + listed.Add(entry); + } + + Assert.Equal(hasContinuation ? new[] { "journals/alpha" } : [], listed.Select(entry => entry.Id.Value)); + Assert.Equal(hasContinuation ? new string?[] { null, "next", "last" } : [null], + requests.Select(request => request.ContinuationToken)); + Assert.All(requests, request => + { + Assert.Equal("journaling-tests", request.BucketName); + Assert.Equal("wal/journals/", request.Prefix); + Assert.Null(request.StartAfter); + }); + await provider.CloseAsync(cancellationToken); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ListAsync_OrderedSeekIsOnlySentOnInitialPage(bool emptyFirstPage) + { + var cancellationToken = TestContext.Current.CancellationToken; + var requests = new List(); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + requests.Add(request); + Assert.Equal(cancellationToken, call.Arg()); + Assert.False(request.StartAfter is not null && request.ContinuationToken is not null); + return Task.FromResult(requests.Count switch + { + 1 => new ListObjectsV2Response + { + S3Objects = emptyFirstPage ? null : [new S3Object { Key = "wal/journals/b100" }], + IsTruncated = true, + NextContinuationToken = "opaque-next" + }, + 2 => new ListObjectsV2Response + { + S3Objects = [new S3Object { Key = "wal/journals/c100" }], + IsTruncated = true, + NextContinuationToken = "opaque-last" + }, + 3 => new ListObjectsV2Response + { + S3Objects = [new S3Object { Key = "wal/journals/d100" }], + IsTruncated = false + }, + _ => throw new InvalidOperationException("Unexpected extra listing request.") + }); + }); + var options = CreateOptions(); + options.S3Client = client; + options.UseOrderedListing = true; + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + var listed = new List(); + await foreach (var entry in provider.ListAsync( + new() { Prefix = new("journals/"), MinId = new("journals/b100"), MaxId = new("journals/d100") }, + cancellationToken)) + { + listed.Add(entry); + } + + Assert.Equal(emptyFirstPage ? new[] { "journals/c100", "journals/d100" } : ["journals/b100", "journals/c100", "journals/d100"], + listed.Select(entry => entry.Id.Value)); + Assert.Equal(new string?[] { "wal/journals/b10", null, null }, requests.Select(request => request.StartAfter)); + Assert.Equal(new string?[] { null, "opaque-next", "opaque-last" }, requests.Select(request => request.ContinuationToken)); + Assert.All(requests, request => Assert.Equal("wal/journals/", request.Prefix)); + await provider.CloseAsync(cancellationToken); + } + + [Fact] + public async Task ListAsync_NullObjectCollectionObservesResponseCancellation() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(_ => + { + cancellation.Cancel(); + return Task.FromResult(new ListObjectsV2Response + { + S3Objects = null, + IsTruncated = true, + NextContinuationToken = "next" + }); + }); + var options = CreateOptions(); + options.S3Client = client; + var provider = CreateProvider(options); + await provider.InitializeAsync(TestContext.Current.CancellationToken); + + await using var enumerator = provider.ListAsync(cancellationToken: cancellation.Token).GetAsyncEnumerator(cancellation.Token); + var exception = await Assert.ThrowsAnyAsync(() => enumerator.MoveNextAsync().AsTask()); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + await client.Received(1).ListObjectsV2Async(Arg.Any(), cancellation.Token); + await provider.CloseAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task ListAsync_NullObjectCollectionPreservesLaterServiceFailure() + { + var cancellationToken = TestContext.Current.CancellationToken; + var failure = new AmazonS3Exception("The continuation request failed."); + var client = CreateTrackingClient(); + client.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(call => call.Arg().ContinuationToken is null + ? Task.FromResult(new ListObjectsV2Response + { + S3Objects = null, + IsTruncated = true, + NextContinuationToken = "next" + }) + : Task.FromException(failure)); + var options = CreateOptions(); + options.S3Client = client; + var provider = CreateProvider(options); + await provider.InitializeAsync(cancellationToken); + + await using var enumerator = provider.ListAsync(cancellationToken: cancellationToken).GetAsyncEnumerator(cancellationToken); + Assert.Same(failure, await Assert.ThrowsAsync(() => enumerator.MoveNextAsync().AsTask())); + await client.Received(2).ListObjectsV2Async(Arg.Any(), cancellationToken); + await client.Received(1).ListObjectsV2Async( + Arg.Is(request => request.ContinuationToken == "next"), cancellationToken); + await provider.CloseAsync(cancellationToken); + } + [Fact] public async Task ListAsync_WhenAliasMapsToCanonicalJournalId_ReturnsCanonicalIdentityOnce() { @@ -227,13 +584,14 @@ public async Task ListAsync_WhenAliasMapsToCanonicalJournalId_ReturnsCanonicalId IsTruncated = false, S3Objects = [ - new S3Object { Key = "current/journals/alpha/wal" }, - new S3Object { Key = "legacy/journals/alpha/wal" }, + new S3Object { Key = "wal/current/journals/alpha" }, + new S3Object { Key = "wal/legacy/journals/alpha" }, ], })); var options = CreateOptions(); options.S3Client = client; options.GetObjectKey = static id => $"current/{id.Value}"; + options.GetObjectKeyPrefix = static prefix => $"current/{prefix.Value}"; options.TryParseJournalId = static key => key.EndsWith("journals/alpha", StringComparison.Ordinal) ? new JournalId("journals/alpha") @@ -241,13 +599,13 @@ public async Task ListAsync_WhenAliasMapsToCanonicalJournalId_ReturnsCanonicalId var provider = CreateProvider(options); await provider.InitializeAsync(CancellationToken.None); - var listed = new List(); - await foreach (var journalId in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) + var listed = new List(); + await foreach (var entry in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) { - listed.Add(journalId); + listed.Add(entry); } - Assert.Equal(["journals/alpha"], listed.Select(static id => id.Value)); + Assert.Equal(["journals/alpha"], listed.Select(static entry => entry.Id.Value)); await provider.CloseAsync(CancellationToken.None); } @@ -261,12 +619,13 @@ public async Task ListAsync_WhenOnlyAliasMapsToJournalId_DoesNotReturnDanglingId IsTruncated = false, S3Objects = [ - new S3Object { Key = "legacy/journals/alpha/wal" }, + new S3Object { Key = "wal/legacy/journals/alpha" }, ], })); var options = CreateOptions(); options.S3Client = client; options.GetObjectKey = static id => $"current/{id.Value}"; + options.GetObjectKeyPrefix = static prefix => $"current/{prefix.Value}"; options.TryParseJournalId = static key => key.EndsWith("journals/alpha", StringComparison.Ordinal) ? new JournalId("journals/alpha") @@ -274,10 +633,10 @@ public async Task ListAsync_WhenOnlyAliasMapsToJournalId_DoesNotReturnDanglingId var provider = CreateProvider(options); await provider.InitializeAsync(CancellationToken.None); - var listed = new List(); - await foreach (var journalId in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) + var listed = new List(); + await foreach (var entry in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) { - listed.Add(journalId); + listed.Add(entry); } Assert.Empty(listed); @@ -292,7 +651,7 @@ public async Task ListAsync_WhenParserReturnsDefaultJournalId_SkipsObject() .Returns(Task.FromResult(new ListObjectsV2Response { IsTruncated = false, - S3Objects = [new S3Object { Key = "invalid/wal" }], + S3Objects = [new S3Object { Key = "wal/invalid" }], })); var options = CreateOptions(); options.S3Client = client; @@ -300,10 +659,10 @@ public async Task ListAsync_WhenParserReturnsDefaultJournalId_SkipsObject() var provider = CreateProvider(options); await provider.InitializeAsync(CancellationToken.None); - var listed = new List(); - await foreach (var journalId in provider.ListAsync(cancellationToken: CancellationToken.None)) + var listed = new List(); + await foreach (var entry in provider.ListAsync(cancellationToken: CancellationToken.None)) { - listed.Add(journalId); + listed.Add(entry); } Assert.Empty(listed); @@ -354,6 +713,53 @@ private sealed class TestSiloBuilder : ISiloBuilder [TestCategory("BVT")] public sealed class S3JournalStorageRequestTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReplaceAsync_SeparatesWalAndCheckpointNamespaces(bool customMapping) + { + var cancellationToken = TestContext.Current.CancellationToken; + var client = Substitute.For(); + var requests = new List(); + var metadataRequests = new List(); + client.PutObjectAsync(Arg.Do(requests.Add), Arg.Any()) + .Returns(_ => Task.FromResult(new PutObjectResponse { ETag = $"etag-{requests.Count}" })); + client.GetObjectMetadataAsync(Arg.Do(metadataRequests.Add), Arg.Any()) + .Returns(_ => Task.FromResult(CreateWalProperties(requests[0], "etag-1"))); + var options = new S3JournalStorageOptions { BucketName = BucketName }; + if (customMapping) + { + options.GetObjectKey = static id => $"tenant/{id.Value}"; + } + + var storage = CreateStorage(client, options); + await storage.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + await storage.ReplaceAsync(new ReadOnlySequence([1, 2]), cancellationToken); + + var baseKey = customMapping ? "tenant/journals/test" : "journals/test"; + Assert.Equal(3, requests.Count); + Assert.Equal($"wal/{baseKey}", requests[0].Key); + Assert.Equal("*", requests[0].IfNoneMatch); + var checkpointPrefix = $"checkpoints/{baseKey}/"; + Assert.StartsWith(checkpointPrefix, requests[1].Key); + Assert.True(Guid.TryParseExact(requests[1].Key[checkpointPrefix.Length..], "N", out _)); + Assert.Equal($"wal/{baseKey}", requests[2].Key); + Assert.Equal("etag-1", requests[2].IfMatch); + Assert.Equal(requests[1].Key, requests[2].Metadata[S3JournalStorage.CheckpointMetadataKey]); + // S3 Express refreshes WAL properties, then queries the append-part count. + Assert.Collection(metadataRequests, + request => Assert.Null(request.PartNumber), + request => Assert.Equal(1, request.PartNumber)); + Assert.All(metadataRequests, request => + { + Assert.Equal(BucketName, request.BucketName); + Assert.Equal("etag-1", request.EtagToMatch); + }); + await client.Received(2).GetObjectMetadataAsync( + Arg.Is(request => request.Key == $"wal/{baseKey}"), + cancellationToken); + } + [Fact] public async Task AppendAsync_DefaultS3ExpressPath_UsesWriteOffsetAndAdvancesState() { @@ -1179,11 +1585,11 @@ public async Task ReplaceAsync_WhenWalPublicationIsDefinitivelyRejected_DeletesU client.PutObjectAsync( Arg.Do(request => { - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { createRequest = request; } - else if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + else if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { checkpointName = request.Key; } @@ -1192,12 +1598,12 @@ public async Task ReplaceAsync_WhenWalPublicationIsDefinitivelyRejected_DeletesU .Returns(call => { var request = call.Arg(); - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "etag-1" }); } - if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "checkpoint-etag" }); } @@ -1244,11 +1650,11 @@ public async Task ReplaceAsync_WhenRejectedPublicationIsAlreadyVisible_Preserves client.PutObjectAsync( Arg.Do(request => { - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { createRequest = request; } - else if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + else if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { checkpointName = request.Key; } @@ -1257,12 +1663,12 @@ public async Task ReplaceAsync_WhenRejectedPublicationIsAlreadyVisible_Preserves .Returns(call => { var request = call.Arg(); - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "etag-1" }); } - if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "checkpoint-etag" }); } @@ -1311,11 +1717,11 @@ public async Task ReplaceAsync_WhenWalPublicationIsAmbiguous_PreservesCheckpoint client.PutObjectAsync( Arg.Do(request => { - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { createRequest = request; } - else if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + else if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { checkpointName = request.Key; } @@ -1324,12 +1730,12 @@ public async Task ReplaceAsync_WhenWalPublicationIsAmbiguous_PreservesCheckpoint .Returns(call => { var request = call.Arg(); - if (request.IfNoneMatch == "*" && request.Key.EndsWith("/wal", StringComparison.Ordinal)) + if (request.IfNoneMatch == "*" && request.Key.StartsWith("wal/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "etag-1" }); } - if (request.Key.Contains("/chk.", StringComparison.Ordinal)) + if (request.Key.StartsWith("checkpoints/", StringComparison.Ordinal)) { return Task.FromResult(new PutObjectResponse { ETag = "checkpoint-etag" }); } @@ -1589,13 +1995,13 @@ public async Task ListAsync_ReturnsJournalIdsMatchingPrefix() await CreateStorage("journals/alpha").AppendAsync(new ReadOnlySequence([1]), CancellationToken.None); await CreateStorage("other/beta").AppendAsync(new ReadOnlySequence([1]), CancellationToken.None); - var listed = new List(); - await foreach (var journalId in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) + var listed = new List(); + await foreach (var entry in provider.ListAsync(new() { Prefix = new JournalId("journals") }, CancellationToken.None)) { - listed.Add(journalId); + listed.Add(entry); } - Assert.Equal(["journals/alpha", "journals/zeta"], listed.Select(static id => id.Value).Order(StringComparer.Ordinal)); + Assert.Equal(["journals/alpha", "journals/zeta"], listed.Select(static entry => entry.Id.Value).Order(StringComparer.Ordinal)); } [Fact] @@ -1693,7 +2099,7 @@ private S3JournalStorageOptions CreateOptions() UseConditionalDelete = false, StorageClass = null, MetadataOnlyConflictInitialBackoff = TimeSpan.Zero, - GetObjectKey = id => id.Value, + UseOrderedListing = true, }; } diff --git a/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs b/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs index 3b78f041902..d2b540f5524 100644 --- a/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs +++ b/test/Orleans.Journaling.Tests/VolatileJournalStorageProviderTests.cs @@ -8,6 +8,127 @@ namespace Orleans.Journaling.Tests; [TestCategory("BVT")] public sealed class VolatileJournalStorageProviderTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ListAsync_MetadataProjectionFlagIsSnapshottedAtEnumerationStart(bool includeMetadata) + { + var provider = new VolatileJournalStorageProvider(); + var cancellationToken = TestContext.Current.CancellationToken; + foreach (var id in new[] { "a", "b" }) + { + await provider.CreateStorage(new(id)).CreateIfNotExistsAsync( + new Dictionary { ["owner"] = id }, cancellationToken); + } + + var options = new ListOptions { IncludeMetadata = !includeMetadata }; + var listing = provider.ListAsync(options, cancellationToken); + options.IncludeMetadata = includeMetadata; + await using var enumerator = listing.GetAsyncEnumerator(cancellationToken); + foreach (var id in new[] { "a", "b" }) + { + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(new JournalId(id), enumerator.Current.Id); + if (includeMetadata) + { + var metadata = Assert.IsType(enumerator.Current.Metadata); + var stored = await provider.CreateStorage(new(id)).GetMetadataAsync(cancellationToken); + Assert.NotNull(stored); + Assert.Equal(stored.Format, metadata.Format); + Assert.Equal(stored.ETag, metadata.ETag); + Assert.Equal(stored.Properties, metadata.Properties); + } + else + { + Assert.Null(enumerator.Current.Metadata); + } + + options.IncludeMetadata = !includeMetadata; + } + + Assert.False(await enumerator.MoveNextAsync()); + } + + [Fact] + public async Task ListAsync_MetadataSnapshotPreservesPropertiesAndRejectsStaleConditionalUpdate() + { + var provider = new VolatileJournalStorageProvider(); + var id = new JournalId("metadata/snapshot"); + var storage = provider.CreateStorage(id); + var cancellationToken = TestContext.Current.CancellationToken; + await storage.CreateIfNotExistsAsync(new Dictionary { ["owner"] = "first" }, cancellationToken); + await using var enumerator = provider.ListAsync( + new() { IncludeMetadata = true }, cancellationToken).GetAsyncEnumerator(cancellationToken); + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(id, enumerator.Current.Id); + var snapshot = Assert.IsType(enumerator.Current.Metadata); + Assert.NotNull(snapshot.ETag); + + var updated = await storage.UpdateMetadataAsync( + new Dictionary { ["owner"] = "second", ["added"] = "value" }, + expectedETag: snapshot.ETag, cancellationToken: cancellationToken); + Assert.NotNull(updated); + Assert.NotEqual(snapshot.ETag, updated.ETag); + Assert.Equal(new Dictionary { ["owner"] = "first" }, snapshot.Properties); + Assert.Null(await storage.UpdateMetadataAsync( + new Dictionary { ["owner"] = "stale" }, + expectedETag: snapshot.ETag, cancellationToken: cancellationToken)); + var current = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(current); + Assert.Equal(updated.ETag, current.ETag); + Assert.Equal(updated.Properties, current.Properties); + } + + [Fact] + public async Task ListAsync_UsesRawPrefixAndSnapshotsInclusiveRange() + { + var provider = new VolatileJournalStorageProvider(); + string[] ids = ["jobs/20260908-a", "jobs/20260909-a", "jobs/20260909-b", "jobs/20260909-c", "jobs/20260910-a", "other"]; + foreach (var value in ids) + { + await provider.CreateStorage(new(value)).CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken); + } + + var options = new ListOptions { Prefix = new("other") }; + var listing = provider.ListAsync(options, TestContext.Current.CancellationToken); + options.Prefix = new("jobs/20260909"); + options.MinId = new("jobs/20260909-b"); + options.MaxId = new("jobs/20260909-c"); + await using var enumerator = listing.GetAsyncEnumerator(TestContext.Current.CancellationToken); + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal("jobs/20260909-b", enumerator.Current.Id.Value); + + options.Prefix = new("other"); + options.MinId = default; + options.MaxId = default; + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal("jobs/20260909-c", enumerator.Current.Id.Value); + Assert.False(await enumerator.MoveNextAsync()); + Assert.Equal([new JournalId("other")], await ToListAsync(listing, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ListAsync_RangeIndexIncludesRecreatedStoresAndExcludesDeletedStores() + { + var provider = new VolatileJournalStorageProvider(); + var lower = new JournalId("range/b"); + var upper = new JournalId("range/d"); + foreach (var name in new[] { "range/a", "range/b", "range/c", "range/d", "range/e" }) + { + await provider.CreateStorage(new(name)).CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken); + } + + var deleted = provider.CreateStorage(new("range/c")); + await deleted.DeleteAsync(TestContext.Current.CancellationToken); + var options = new ListOptions { MinId = lower, MaxId = upper }; + Assert.Equal([lower, upper], await ToListAsync(provider.ListAsync(options, TestContext.Current.CancellationToken), TestContext.Current.CancellationToken)); + + await deleted.CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal([lower, new JournalId("range/c"), upper], await ToListAsync(provider.ListAsync(options, TestContext.Current.CancellationToken), TestContext.Current.CancellationToken)); + options.MinId = new("z"); + Assert.Empty(await ToListAsync(provider.ListAsync(options, TestContext.Current.CancellationToken), TestContext.Current.CancellationToken)); + } + [Fact] public async Task CreateIfNotExists_ListAndGetMetadataUseJournalIds() { @@ -144,14 +265,14 @@ await Assert.ThrowsAsync( cancellationToken: TestContext.Current.CancellationToken).AsTask()); } - private static async Task> ToListAsync( - IAsyncEnumerable source, + private static async Task> ToListAsync( + IAsyncEnumerable source, CancellationToken cancellationToken) { - var result = new List(); + var result = new List(); await foreach (var item in source.WithCancellation(cancellationToken)) { - result.Add(item); + result.Add(item.Id); } return result; From b7a42292d2cc57530069e8a4c69c075fac3a3451 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 02:19:26 -0700 Subject: [PATCH 2/9] docs(journaling): clarify alpha layout upgrade prerequisites --- src/AWS/Orleans.Journaling.S3/README.md | 7 ++++++ .../Orleans.Journaling.AzureStorage/README.md | 7 ++++++ src/Orleans.Journaling/README.md | 24 +++++++++++++++++++ src/Redis/Orleans.Journaling.Redis/README.md | 7 ++++++ 4 files changed, 45 insertions(+) diff --git a/src/AWS/Orleans.Journaling.S3/README.md b/src/AWS/Orleans.Journaling.S3/README.md index bf5d0a78ecc..1fd400959b0 100644 --- a/src/AWS/Orleans.Journaling.S3/README.md +++ b/src/AWS/Orleans.Journaling.S3/README.md @@ -10,6 +10,13 @@ Buckets should be created ahead of time for AWS S3 Express One Zone. `CreateBuck Metadata updates rewrite the current WAL using a conditional single-object upload. Publish a checkpoint to compact the WAL before updating metadata when the replacement object would exceed S3's 5 GB (5,000,000,000 byte) single-upload limit. Checkpoint snapshots use the same upload limit. +## Adopting the alpha storage layout + +Deploy the new WAL/checkpoint layout into a fresh bucket. For state retained from an earlier alpha +deployment, complete the +[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) +using the earlier package and configuration before switching traffic. + ## Object layout `GetObjectKey` maps a logical journal id to its base object key (the identity mapping by default). WAL and checkpoint objects use separate namespaces: diff --git a/src/Azure/Orleans.Journaling.AzureStorage/README.md b/src/Azure/Orleans.Journaling.AzureStorage/README.md index ed0cd3b5766..0ff7529735a 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/README.md +++ b/src/Azure/Orleans.Journaling.AzureStorage/README.md @@ -5,6 +5,13 @@ Microsoft Orleans Journaling for Azure Storage provides an Azure Storage impleme Blob names are derived from the configured journal storage identity and do not use journal format file extensions. Azure append blobs store the journal format key in blob metadata and, when the selected journal format provides a MIME type, are created with that content type. The WAL blob name and checkpoint blob name can be customized using `AzureBlobJournalStorageOptions.GetWalBlobName` and `GetCheckpointBlobName`. +## Adopting the alpha storage layouts + +Deploy the new Blob WAL/checkpoint layout into a fresh container and the new default Table partition +encoding into a fresh table. For state retained from an earlier alpha deployment, complete the +[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) +using the earlier package and configuration before switching traffic. + ## Using an alternative blob layout By default, WAL blobs are named `wal/` and checkpoint blobs are named `checkpoints//`. The separate prefixes let catalog discovery select WAL blobs directly, keeping retained checkpoints out of listing pages. Configure the blob name delegates to use an alternative layout, such as a shared prefix, file extensions, tenant-specific paths, or names which match an existing storage convention. Each delegate returns a container-relative blob name, and checkpoint names should include the supplied snapshot id to avoid collisions. Catalog discovery uses the default WAL layout in the configured container; custom delegates participating in discovery produce `wal/` for each journal. diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 4babb729c9e..2a7432d2788 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -108,6 +108,30 @@ Inside the operation payload array, element 0 is the command name, followed by c Existing data is read using its stored format metadata, or as legacy OrleansBinary data when metadata is absent, and migrated to the configured write format by the next snapshot write. +## Adopting the alpha storage layouts + +**Deploy these alpha layouts into fresh storage namespaces.** Use a new Azure Blob container, +Azure Table table, S3 bucket, or Redis `KeyPrefix`. The Blob and S3 WAL/checkpoint namespaces, +default Table partition encoding, and Redis key suffixes change the persisted addresses of journals. +Earlier-layout journals remain at their original addresses and require the earlier provider +package and configuration for recovery. Pointing the new defaults at that storage can make those +journals appear missing. + +For an existing alpha deployment whose state must be retained, prepare and rehearse an +application-owned export/import before upgrading: + +1. Quiesce application writes and job execution, and preserve the original storage, package versions, + and configuration together with a recoverable backup. +2. Recover and export the required state using the earlier application and provider. Import it through + the upgraded application into the fresh namespace, preserving logical identities, caller-owned + metadata, and pending work as required by the application. +3. Verify catalog completeness and recovered application state, including pending jobs, before switching + traffic and resuming execution. Retain the original deployment and backup through cutover, and account + for post-cutover writes in the application's rollback procedure. + +The application supplies and validates that state-transfer procedure. Keep the earlier deployment +in service until it is ready. All silos sharing a journal namespace must use the same storage layout. + ## Catalog enumeration `IJournalStorageCatalog.ListAsync` returns an `IAsyncEnumerable` in provider traversal order. diff --git a/src/Redis/Orleans.Journaling.Redis/README.md b/src/Redis/Orleans.Journaling.Redis/README.md index 81532e2d5c7..e23f11ffe46 100644 --- a/src/Redis/Orleans.Journaling.Redis/README.md +++ b/src/Redis/Orleans.Journaling.Redis/README.md @@ -6,6 +6,13 @@ Microsoft Orleans Journaling for Redis provides a Redis-backed implementation of The provider stores each journal as Redis string data plus Redis hash metadata. Per-journal reads and mutations use atomic Lua scripts. Journal discovery scans metadata keys on each connected primary Redis server. The default key mapping preserves journal ids in the keys, so discovery does not read the metadata hashes. Configure Redis persistence, such as AOF with an appropriate `appendfsync` setting, according to the durability guarantees required by your application. +## Adopting the alpha storage layout + +Deploy the new reversible key layout with a fresh `KeyPrefix`. This layout applies to both default and +custom `GetKeyName` mappings. For state retained from an earlier alpha deployment, complete the +[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) +using the earlier package and configuration before switching traffic. + ## Getting Started Install the package: From e7d3efd1c177dbdaba0f2953b7a465c8ed939c1e Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 07:52:50 -0700 Subject: [PATCH 3/9] docs(journaling): describe current storage layouts --- src/AWS/Orleans.Journaling.S3/README.md | 7 ------ .../Orleans.Journaling.AzureStorage/README.md | 7 ------ src/Orleans.Journaling/README.md | 24 ------------------- src/Redis/Orleans.Journaling.Redis/README.md | 9 +------ 4 files changed, 1 insertion(+), 46 deletions(-) diff --git a/src/AWS/Orleans.Journaling.S3/README.md b/src/AWS/Orleans.Journaling.S3/README.md index 1fd400959b0..bf5d0a78ecc 100644 --- a/src/AWS/Orleans.Journaling.S3/README.md +++ b/src/AWS/Orleans.Journaling.S3/README.md @@ -10,13 +10,6 @@ Buckets should be created ahead of time for AWS S3 Express One Zone. `CreateBuck Metadata updates rewrite the current WAL using a conditional single-object upload. Publish a checkpoint to compact the WAL before updating metadata when the replacement object would exceed S3's 5 GB (5,000,000,000 byte) single-upload limit. Checkpoint snapshots use the same upload limit. -## Adopting the alpha storage layout - -Deploy the new WAL/checkpoint layout into a fresh bucket. For state retained from an earlier alpha -deployment, complete the -[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) -using the earlier package and configuration before switching traffic. - ## Object layout `GetObjectKey` maps a logical journal id to its base object key (the identity mapping by default). WAL and checkpoint objects use separate namespaces: diff --git a/src/Azure/Orleans.Journaling.AzureStorage/README.md b/src/Azure/Orleans.Journaling.AzureStorage/README.md index 0ff7529735a..ed0cd3b5766 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/README.md +++ b/src/Azure/Orleans.Journaling.AzureStorage/README.md @@ -5,13 +5,6 @@ Microsoft Orleans Journaling for Azure Storage provides an Azure Storage impleme Blob names are derived from the configured journal storage identity and do not use journal format file extensions. Azure append blobs store the journal format key in blob metadata and, when the selected journal format provides a MIME type, are created with that content type. The WAL blob name and checkpoint blob name can be customized using `AzureBlobJournalStorageOptions.GetWalBlobName` and `GetCheckpointBlobName`. -## Adopting the alpha storage layouts - -Deploy the new Blob WAL/checkpoint layout into a fresh container and the new default Table partition -encoding into a fresh table. For state retained from an earlier alpha deployment, complete the -[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) -using the earlier package and configuration before switching traffic. - ## Using an alternative blob layout By default, WAL blobs are named `wal/` and checkpoint blobs are named `checkpoints//`. The separate prefixes let catalog discovery select WAL blobs directly, keeping retained checkpoints out of listing pages. Configure the blob name delegates to use an alternative layout, such as a shared prefix, file extensions, tenant-specific paths, or names which match an existing storage convention. Each delegate returns a container-relative blob name, and checkpoint names should include the supplied snapshot id to avoid collisions. Catalog discovery uses the default WAL layout in the configured container; custom delegates participating in discovery produce `wal/` for each journal. diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 2a7432d2788..4babb729c9e 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -108,30 +108,6 @@ Inside the operation payload array, element 0 is the command name, followed by c Existing data is read using its stored format metadata, or as legacy OrleansBinary data when metadata is absent, and migrated to the configured write format by the next snapshot write. -## Adopting the alpha storage layouts - -**Deploy these alpha layouts into fresh storage namespaces.** Use a new Azure Blob container, -Azure Table table, S3 bucket, or Redis `KeyPrefix`. The Blob and S3 WAL/checkpoint namespaces, -default Table partition encoding, and Redis key suffixes change the persisted addresses of journals. -Earlier-layout journals remain at their original addresses and require the earlier provider -package and configuration for recovery. Pointing the new defaults at that storage can make those -journals appear missing. - -For an existing alpha deployment whose state must be retained, prepare and rehearse an -application-owned export/import before upgrading: - -1. Quiesce application writes and job execution, and preserve the original storage, package versions, - and configuration together with a recoverable backup. -2. Recover and export the required state using the earlier application and provider. Import it through - the upgraded application into the fresh namespace, preserving logical identities, caller-owned - metadata, and pending work as required by the application. -3. Verify catalog completeness and recovered application state, including pending jobs, before switching - traffic and resuming execution. Retain the original deployment and backup through cutover, and account - for post-cutover writes in the application's rollback procedure. - -The application supplies and validates that state-transfer procedure. Keep the earlier deployment -in service until it is ready. All silos sharing a journal namespace must use the same storage layout. - ## Catalog enumeration `IJournalStorageCatalog.ListAsync` returns an `IAsyncEnumerable` in provider traversal order. diff --git a/src/Redis/Orleans.Journaling.Redis/README.md b/src/Redis/Orleans.Journaling.Redis/README.md index e23f11ffe46..edb922a063a 100644 --- a/src/Redis/Orleans.Journaling.Redis/README.md +++ b/src/Redis/Orleans.Journaling.Redis/README.md @@ -6,13 +6,6 @@ Microsoft Orleans Journaling for Redis provides a Redis-backed implementation of The provider stores each journal as Redis string data plus Redis hash metadata. Per-journal reads and mutations use atomic Lua scripts. Journal discovery scans metadata keys on each connected primary Redis server. The default key mapping preserves journal ids in the keys, so discovery does not read the metadata hashes. Configure Redis persistence, such as AOF with an appropriate `appendfsync` setting, according to the durability guarantees required by your application. -## Adopting the alpha storage layout - -Deploy the new reversible key layout with a fresh `KeyPrefix`. This layout applies to both default and -custom `GetKeyName` mappings. For state retained from an earlier alpha deployment, complete the -[application-owned state transfer and cutover](../../Orleans.Journaling/README.md#adopting-the-alpha-storage-layouts) -using the earlier package and configuration before switching traffic. - ## Getting Started Install the package: @@ -71,7 +64,7 @@ Storage errors and cancellation propagate without provider-level retries, restar ## Redis key layout -Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. The reversible key name is outside the existing SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI encoding preserves literal percent signs, separators, Unicode, and Redis glob characters in journal ids; the scan pattern also escapes glob characters in the configured key prefix. +Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. This layout applies to both default and custom `GetKeyName` mappings. The reversible key name is outside the SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI encoding preserves literal percent signs, separators, Unicode, and Redis glob characters in journal ids; the scan pattern also escapes glob characters in the configured key prefix. ## Documentation From 59ade16d0500573c7cf9565d65e8b396b10446e9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 10:52:20 -0700 Subject: [PATCH 4/9] fix(journaling): preserve catalog range discovery --- .../docs/grains/journaling/azure-storage.md | 4 +- src/AWS/Orleans.Journaling.S3/README.md | 4 +- .../S3JournalStorageProvider.cs | 5 +- .../AzureBlobJournalStorageProvider.cs | 39 +++- .../Orleans.Journaling.AzureStorage/README.md | 6 +- .../JournaledJobShardManager.cs | 2 +- src/Orleans.Journaling/README.md | 9 +- src/Redis/Orleans.Journaling.Redis/README.md | 2 + .../JournaledJobShardManagerTests.cs | 33 +++- .../JournalStorageCatalogTests.cs | 185 ++++++++++++++++-- 10 files changed, 254 insertions(+), 35 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/azure-storage.md b/docs/site/src/content/docs/grains/journaling/azure-storage.md index 88e556ee1c8..02c0f17dce1 100644 --- a/docs/site/src/content/docs/grains/journaling/azure-storage.md +++ b/docs/site/src/content/docs/grains/journaling/azure-storage.md @@ -29,7 +29,9 @@ Customize ` for each journal. The separate checkpoint namespace keeps checkpoints out of listing pages. Raw journal-id prefixes and ASCII lower bounds narrow the native listing, and an ASCII upper bound terminates ordered consumption after the boundary page. +The journal catalog selects append blobs in the `wal/` namespace of the configured container. Custom naming delegates participating in catalog discovery produce `wal/` for each journal. The separate checkpoint namespace keeps checkpoints out of listing pages. Raw journal-id prefixes and conservative ASCII bounds narrow the native listing on both flat-namespace and hierarchical-namespace (HNS) accounts. + +HNS recursive listings sort `/` before other characters. The catalog preserves each bound's shared listing prefix and widens the remaining suffix when punctuation or directory separators affect ordering. For example, an inclusive range from `a!` through `a/0` starts at `wal/a` and completes after crossing `wal/a0`. Every returned candidate is checked against the original ordinal range. Shared prefixes keep timestamp scans narrow; wider boundaries can transfer additional candidates. Both account types use the same listing algorithm and configured Blob client. Catalog callers can request a metadata snapshot with each identity. Blob listings project the WAL's format, ETag, and caller-owned metadata in the listing response. The snapshot can replace a separate metadata read; conditional updates use its ETag to detect concurrent changes. diff --git a/src/AWS/Orleans.Journaling.S3/README.md b/src/AWS/Orleans.Journaling.S3/README.md index bf5d0a78ecc..d6ea9bdd7ae 100644 --- a/src/AWS/Orleans.Journaling.S3/README.md +++ b/src/AWS/Orleans.Journaling.S3/README.md @@ -19,7 +19,9 @@ Metadata updates rewrite the current WAL using a conditional single-object uploa For example, journal `jobs/00001234` uses `wal/jobs/00001234` and checkpoint objects under `checkpoints/jobs/00001234/`. Checkpoint names are stored in the WAL metadata. Catalog requests always stay under `wal/`, so checkpoints do not consume listing pages. -S3 Express directory buckets benefit from slash-delimited prefixes, but this layout does not introduce time partitions or a discovery horizon. Unordered listings scan the selected WAL directory and retain every matching overdue journal, however old. Applications can supply hierarchical base keys, provided their prefix and reverse mappings satisfy the catalog contract. +S3 Express directory buckets benefit from slash-delimited prefixes. Unordered listings scan the selected WAL directory and retain every matching overdue journal, however old. Applications can supply hierarchical base keys, provided their prefix and reverse mappings satisfy the catalog contract. + +**Alpha layout upgrade:** When upgrading from the previous `/wal` layout, drain durable jobs before deploying the new version, then recreate journals using the new layout. ## Catalog enumeration diff --git a/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs b/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs index 9e50144a618..8c9cd40d779 100644 --- a/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs +++ b/src/AWS/Orleans.Journaling.S3/S3JournalStorageProvider.cs @@ -59,9 +59,8 @@ public async IAsyncEnumerable ListAsync( var ordered = _options.UseOrderedListing; var identityMapping = _options.UsesDefaultObjectKey; var listingPrefix = range.ListingPrefix; - if (!identityMapping - && (string.IsNullOrWhiteSpace(listingPrefix) - || _options.GetObjectKeyPrefix is null && range.Prefix is null)) + if (string.IsNullOrWhiteSpace(listingPrefix) && (!identityMapping || _options.GetObjectKeyPrefix is not null) + || !identityMapping && _options.GetObjectKeyPrefix is null && range.Prefix is null) { // A common prefix inferred from bounds can be whitespace, which cannot be represented // as a JournalId for a custom mapper. Bounds alone also do not require a prefix mapper. diff --git a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs index 1e2e9ddb3c7..c0b17f22ae5 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs +++ b/src/Azure/Orleans.Journaling.AzureStorage/AzureBlobJournalStorageProvider.cs @@ -68,15 +68,14 @@ public async IAsyncEnumerable ListAsync( } var container = GetDefaultContainerClient(); - var maxBlobName = range.MaxId is { } maxId && System.Text.Ascii.IsValid(maxId) - ? AzureBlobJournalStorageLayout.GetWalBlobName(maxId) : null; - var startFrom = range.LowerBound is { } lowerBound && System.Text.Ascii.IsValid(lowerBound) - ? AzureBlobJournalStorageLayout.GetWalBlobName(lowerBound) : null; + var prefix = range.ListingPrefix ?? string.Empty; + var maxBlobName = GetNativeBound(range.MaxId, prefix, isUpperBound: true); + var startFrom = GetNativeBound(range.LowerBound, prefix, isUpperBound: false); await foreach (var page in container.GetBlobsAsync( new GetBlobsOptions { Traits = range.IncludeMetadata ? BlobTraits.Metadata : BlobTraits.None, - Prefix = AzureBlobJournalStorageLayout.GetWalBlobName(range.ListingPrefix ?? string.Empty), + Prefix = AzureBlobJournalStorageLayout.GetWalBlobName(prefix), StartFrom = startFrom, }, cancellationToken).AsPages(pageSizeHint: 5000)) @@ -85,8 +84,7 @@ public async IAsyncEnumerable ListAsync( foreach (var item in page.Values) { cancellationToken.ThrowIfCancellationRequested(); - // Azure's flat List Blobs API returns names in lexical order. Every matching WAL - // is at or below this raw-name bound, including the WAL for MaxId itself. + // The native bound encloses every ordinal match in both flat and HNS traversal. if (maxBlobName is not null && string.CompareOrdinal(item.Name, maxBlobName) > 0) { yield break; @@ -112,6 +110,33 @@ public async IAsyncEnumerable ListAsync( cancellationToken.ThrowIfCancellationRequested(); } + private static string? GetNativeBound(string? bound, string prefix, bool isUpperBound) + { + if (bound is null || !System.Text.Ascii.IsValid(bound)) + { + return null; + } + + var index = 0; + while (index < bound.Length && index < prefix.Length && bound[index] == prefix[index]) + { + index++; + } + + // HNS sorts '/' first. Beyond the shared prefix, keep boundary characters above '/' + // so both service orderings agree, widening the requested interval where necessary. + for (; index < bound.Length; index++) + { + if (bound[index] <= '/') + { + var widened = isUpperBound ? string.Concat(bound.AsSpan(0, index), "0") : bound[..index]; + return AzureBlobJournalStorageLayout.GetWalBlobName(widened); + } + } + + return AzureBlobJournalStorageLayout.GetWalBlobName(bound); + } + public void Participate(ISiloLifecycle observer) { observer.Subscribe( diff --git a/src/Azure/Orleans.Journaling.AzureStorage/README.md b/src/Azure/Orleans.Journaling.AzureStorage/README.md index ed0cd3b5766..caf5de26ea3 100644 --- a/src/Azure/Orleans.Journaling.AzureStorage/README.md +++ b/src/Azure/Orleans.Journaling.AzureStorage/README.md @@ -37,7 +37,11 @@ Both Azure providers implement `IJournalStorageCatalog.ListAsync`, returning `Jo Set `ListOptions.IncludeMetadata` to include each entry's format, ETag, and caller-owned properties in `Metadata`. The snapshot is observed together in the listing response and has the same semantics as `GetMetadataAsync`; its ETag can be supplied to `UpdateMetadataAsync` for a conditional update. Blob listing requests metadata traits, while Table listing selects the header's format and serialized caller metadata alongside the identity and timestamp. Both providers construct snapshots directly from listing pages. By default, enumeration projects identities with `Metadata` set to `null`. `IncludeMetadata` is snapshotted with the range options when enumeration starts. -The Blob catalog scans the configured `ContainerName` and interprets append blobs named `wal/` as journal identities. This traversal applies equally when a custom naming delegate or container factory produces the same entries. Requests use `wal/` followed by the raw journal-id prefix, narrowed by the common prefix of `MinId` and `MaxId` when possible, and up to 5000 blobs per page. Checkpoints occupy their own namespace and are excluded before pagination, including for unbounded catalog queries. For an ASCII lower bound, `GetBlobsOptions.StartFrom` seeks to `wal/`, preserving the minimum journal's WAL. Azure's lexical listing order permits stopping after `wal/` for an ASCII upper bound. Other bounds are enforced locally. Upper-bound termination happens while consuming results: the provider reads the page crossing the upper bound and then completes the traversal. Empty intersections issue no request. Recovery and checkpoint cleanup use the full checkpoint name published in WAL metadata. +The Blob catalog scans the configured `ContainerName` and interprets append blobs named `wal/` as journal identities. This traversal applies equally when a custom naming delegate or container factory produces the same entries. Requests use `wal/` followed by the raw journal-id prefix, narrowed by the common prefix of `MinId` and `MaxId` when possible, and up to 5000 blobs per page. Checkpoints occupy their own namespace and are excluded before pagination, including for unbounded catalog queries. Recovery and checkpoint cleanup use the full checkpoint name published in WAL metadata. + +ASCII bounds narrow traversal using `GetBlobsOptions.StartFrom` and an upper cutoff that work with both flat namespaces and hierarchical namespaces (HNS). HNS recursive listings sort `/` before other characters. After the prefix shared by a bound and the native listing prefix, the provider widens the bound at the first character at or below `/`: the lower bound ends before that character, and the upper bound replaces it with `0` and ends there. Other ASCII bounds retain their exact values. For example, a query from `a!` through `a/0` starts at `wal/a` and stops after `wal/a0`. The provider checks every candidate against the original inclusive ordinal bounds before yielding it. Non-ASCII bounds are enforced locally. + +Shared prefixes keep timestamp range scans narrow. Ranges crossing punctuation or directory boundaries can transfer additional candidates. Upper-bound termination happens while consuming results: the provider reads the page crossing the conservative upper bound and then completes the traversal. Empty intersections issue no request. The same range planning applies to both account types using the configured Blob client and ordinary listing requests. Table enumeration requests up to 1000 header rows per service page and reads identities from the canonical `JournalId` header property. The default partition mapping accepts printable ASCII journal ids (`0x20` through `0x7E`) and encodes each byte as two uppercase hexadecimal digits, preserving ordinal ordering and raw prefixes, including partial segments. The 1,024-character partition-key limit therefore permits at most 512 characters per journal id. Validation occurs when creating storage. This restriction belongs to the default Table mapping; custom mappings and other providers retain their journal-id contracts. Default-mapping queries combine the header row key with indexed partition-key constraints for the raw prefix, inclusive `MinId`, and inclusive `MaxId`. Query bounds retain their original ordinal meaning, including bounds outside the stored ASCII alphabet. An empty intersection issues no query. Custom mappings apply well-formed Unicode bounds to the canonical `JournalId` property; bounds containing unpaired surrogates are enforced locally. This preserves ordinal range semantics and can require a full table scan. Unbounded enumeration also scans headers across the table. Constraints are checked again before yielding, and one enumerator advance can cross multiple empty or filtered pages. diff --git a/src/Orleans.DurableJobs/JournaledJobShardManager.cs b/src/Orleans.DurableJobs/JournaledJobShardManager.cs index 5d327dcc068..d7dbd929887 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardManager.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardManager.cs @@ -85,7 +85,7 @@ public override async Task> AssignJobShardsAsync(DateTimeOffset var newClaimCount = 0; var membershipSnapshot = _membershipService.CurrentSnapshot; - await foreach (var entry in _catalog.ListAsync(new() { Prefix = JobShardId.StoragePrefix }, cancellationToken)) + await foreach (var entry in _catalog.ListAsync(new() { Prefix = new(JobShardId.StoragePrefix.Value + "/") }, cancellationToken)) { var descriptor = await GetDescriptorAsync(entry.Id, cancellationToken); if (descriptor is null || descriptor.Poisoned || descriptor.StartTime > maxDueTime) diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 4babb729c9e..c0c42b9cd7c 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -152,14 +152,15 @@ snapshot and pass it to conditional metadata updates; concurrent changes can cau | Volatile | An ordered key index selects a view covering the requested prefix and bounds. | Snapshots selected keys and checks current journal existence. | | Azure Table, default mapping | Printable ASCII ids use two uppercase hex digits per byte, allowing direct indexed prefix and lower/upper key filters. | Queries include the journal header row condition; the service controls work inside the selected key range. | | Azure Table, custom mapping | Canonical journal-id filters limit returned headers. | Arbitrary mappings can require a table scan because the journal-id property is not indexed. | -| Azure Blob | The `wal/` namespace and raw id prefix select WAL blobs; `StartFrom` seeks to an ASCII lower bound and ordered traversal stops at an ASCII upper bound. | The final page can contain WALs beyond the range. Checkpoints occupy a separate namespace. | +| Azure Blob | The `wal/` namespace and raw id prefix select WAL blobs; conservative ASCII bounds narrow traversal safely in both flat and hierarchical namespaces. | Widened bounds and the final page can include additional candidates, filtered against the original ordinal range. Checkpoints occupy a separate namespace. | | S3 general-purpose, ordered listing enabled | The `wal/` namespace excludes checkpoints. Identity-mapped keys use native prefixes and an initial `StartAfter` marker preceding the inclusive lower bound, then stop at the upper WAL key. | The final page can overrun the range. Custom key mappings use their configured native prefix and identity filtering. | | S3 Express directory buckets | A native slash-terminated prefix within `wal/` limits the namespace and excludes checkpoints. | Partial-name and time bounds are filtered during unordered traversal. | | Redis | Readable key names enable native `SCAN MATCH` prefix filtering and local key-range checks before identity metadata reads for the default mapping. | `SCAN MATCH` still traverses the server keyspace. Custom key mappings read canonical ids from matching metadata hashes. | -Blob and ordered S3 native lower/upper optimizations apply where storage ordering agrees with -ordinal identity ordering, including the fixed-width ASCII timestamp names. Other bounds -remain enforced while traversing storage. S3 ordered listing is an explicit +Blob native bounds preserve the shared listing prefix and widen suffixes where punctuation or +directory separators affect storage ordering. Ordered S3 native lower/upper optimizations apply +where storage ordering agrees with ordinal identity ordering, including fixed-width ASCII +timestamp names. All original bounds remain enforced while traversing storage. S3 ordered listing is an explicit `UseOrderedListing` capability setting for general-purpose buckets. The selected identity count, transferred keys, and backend scan work are separate costs: server-side filtering can reduce transferred data while the service still examines a wider keyspace. diff --git a/src/Redis/Orleans.Journaling.Redis/README.md b/src/Redis/Orleans.Journaling.Redis/README.md index edb922a063a..43f4dada9e7 100644 --- a/src/Redis/Orleans.Journaling.Redis/README.md +++ b/src/Redis/Orleans.Journaling.Redis/README.md @@ -66,6 +66,8 @@ Storage errors and cancellation propagate without provider-level retries, restar Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. This layout applies to both default and custom `GetKeyName` mappings. The reversible key name is outside the SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI encoding preserves literal percent signs, separators, Unicode, and Redis glob characters in journal ids; the scan pattern also escapes glob characters in the configured key prefix. +**Alpha layout upgrade:** When upgrading from the previous hash-only layout, drain durable jobs before deploying the new version, then recreate journals using the new layout. + ## Documentation - [Microsoft Orleans Documentation](https://dotnet.github.io/orleans/docs/) diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs index a795387f4ef..217c490241a 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs @@ -115,6 +115,31 @@ public async Task EmptyShard_IsDeletedWhenUnregistered() cancellationToken), cancellationToken)); } + [Fact] + public async Task AssignJobShardsAsync_OpensOnlyShardNamespaceDescendants() + { + var cancellationToken = TestContext.Current.CancellationToken; + var storageProvider = new CountingJournalStorageProvider(delayAppends: false); + using var services = CreateServices(storageProvider); + var membership = new TestClusterMembershipService(); + var silo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5011), 0); + membership.SetSiloStatus(silo, SiloStatus.Active); + var manager = CreateManager(services, membership, silo); + var start = new DateTimeOffset(2026, 9, 9, 0, 0, 0, TimeSpan.Zero); + var shard = await manager.CreateShardAsync(start, start.AddHours(1), new Dictionary(), cancellationToken); + foreach (var id in new[] { "jobs/shards", "jobs/shards-extra/unrelated", "jobs/shards2/unrelated" }) + { + await storageProvider.CreateStorage(new(id)).CreateIfNotExistsAsync(cancellationToken: cancellationToken); + } + + storageProvider.OpenedJournalIds.Clear(); + var assigned = await manager.AssignJobShardsAsync(start.AddHours(1), int.MaxValue, cancellationToken); + + Assert.Same(shard, Assert.Single(assigned)); + Assert.Equal(((JournaledJobShard)shard).StorageId, Assert.Single(storageProvider.OpenedJournalIds)); + await manager.UnregisterShardAsync(shard, cancellationToken); + } + [Fact] public async Task ClosedLocalShard_CanStillPersistRemovals() { @@ -624,6 +649,8 @@ public Task AppendStarted public int AppendCount => Volatile.Read(ref _appendCount); + public ConcurrentBag OpenedJournalIds { get; } = new(); + public void BlockAppends() { lock (_appendGate) @@ -643,7 +670,11 @@ public void AllowAppends() } } - public IJournalStorage CreateStorage(JournalId journalId) => new CountingJournalStorage(this, _inner.CreateStorage(journalId)); + public IJournalStorage CreateStorage(JournalId journalId) + { + OpenedJournalIds.Add(journalId); + return new CountingJournalStorage(this, _inner.CreateStorage(journalId)); + } public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) => _inner.ListAsync(options, cancellationToken); diff --git a/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs b/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs index 22f0a9d4fd5..c442b37a30d 100644 --- a/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs +++ b/test/Orleans.Journaling.Tests/JournalStorageCatalogTests.cs @@ -24,6 +24,19 @@ namespace Orleans.Journaling.Tests; [TestCategory("BVT")] public sealed class JournalStorageCatalogTests { + private static readonly IComparer HnsBlobNameComparer = Comparer.Create(static (left, right) => + { + for (var index = 0; index < Math.Min(left.Length, right.Length); index++) + { + if (left[index] != right[index]) + { + return left[index] == '/' ? -1 : right[index] == '/' ? 1 : left[index].CompareTo(right[index]); + } + } + + return left.Length.CompareTo(right.Length); + }); + [Theory] [InlineData("AzureBlob")] [InlineData("AzureTable")] @@ -631,9 +644,105 @@ public async Task AzureBlobListAsync_MaxIdIncludesExactJournalAndExcludesCheckpo } [Theory] - [InlineData("AzureBlob")] - [InlineData("S3")] - public async Task OrderedListAsync_TimePrefixedNamespaceStopsBeforeFuturePages(string kind) + [InlineData(null, "a!", null, "wal/", "wal/a")] + [InlineData(null, null, "a/0", "wal/", null)] + [InlineData(null, "a!", "a/0", "wal/a", "wal/a")] + [InlineData("a/", "a/0", "a/9", "wal/a/", "wal/a/0")] + [InlineData("a!", "a!", "a!extra", "wal/a!", "wal/a!")] + [InlineData("a/", "0", "z", "wal/a/", "wal/a/")] + public async Task AzureBlobListAsync_ConservativeBoundsPreserveOrdinalMembership( + string? prefix, string? minimum, string? maximum, string nativePrefix, string? startFrom) + { + string[] ids = ["a!", "a!extra", "a.", "a/0", "a/9", "a0", "a00", "b", "z", "zz"]; + var expected = ids.Where(id => (prefix is null || id.StartsWith(prefix, StringComparison.Ordinal)) + && (minimum is null || string.CompareOrdinal(id, minimum) >= 0) + && (maximum is null || string.CompareOrdinal(id, maximum) <= 0)).ToArray(); + foreach (var hierarchicalNamespace in new[] { false, true }) + { + await using var context = await CreateAsync("AzureBlob", ids); + context.Native.HierarchicalNamespace = hierarchicalNamespace; + + AssertMembership(expected, await DrainAsync(context.Catalog.ListAsync( + new() + { + Prefix = prefix is null ? default : new(prefix), + MinId = minimum is null ? default : new(minimum), + MaxId = maximum is null ? default : new(maximum) + }, TestContext.Current.CancellationToken))); + Assert.NotEmpty(context.Native.Requests); + Assert.All(context.Native.Requests, request => + { + Assert.Equal(nativePrefix, request.Prefix); + Assert.Equal(startFrom, request.LowerStart); + }); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AzureBlobListAsync_ConservativeBoundsStopAfterBoundaryPage(bool hierarchicalNamespace) + { + var ids = new[] { "a!", "a/0", "a/9", "a0" } + .Concat(Enumerable.Range(0, 256).Select(index => $"a0{index:D3}")).ToArray(); + await using var context = await CreateAsync("AzureBlob", ids); + context.Native.HierarchicalNamespace = hierarchicalNamespace; + context.Native.EmptyFirstPage = true; + context.Native.Failure = new InvalidOperationException("The tail beyond the conservative upper bound must not be requested."); + context.Native.FailureAtRequest = 5; + + var entries = await DrainEntriesAsync(context.Catalog.ListAsync( + new() { MinId = new("a!"), MaxId = new("a/0"), IncludeMetadata = true }, TestContext.Current.CancellationToken)); + + Assert.Equal(hierarchicalNamespace ? new[] { "a/0", "a!" } : ["a!", "a/0"], entries.Select(entry => entry.Id.Value)); + Assert.Equal([0, 2, 2, 2], context.Native.Requests.Select(request => request.ResultCount)); + Assert.All(context.Native.Requests, request => + { + Assert.Equal("wal/a", request.Prefix); + Assert.Equal("wal/a", request.LowerStart); + Assert.Equal(TestContext.Current.CancellationToken, request.CancellationToken); + }); + AssertMetadataProjection(context.Native, "AzureBlob", includeMetadata: true); + Assert.All(entries, entry => + { + var metadata = Assert.IsAssignableFrom(entry.Metadata); + Assert.Equal("test", metadata.Format); + Assert.Equal(new ETag("listed").ToString(), metadata.ETag); + Assert.Equal(new Dictionary { ["owner"] = "alice" }, metadata.Properties); + }); + Assert.Equal(1, context.Native.DisposedEnumerators); + Assert.Equal(0, context.Native.MetadataRequests); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AzureBlobListAsync_AsciiBoundCharactersPreserveOrdinalMembership(bool hierarchicalNamespace) + { + string[] ids = + [ + "tenant/a!", "tenant/a!extra", "tenant/a-", "tenant/a.", "tenant/a/0", "tenant/a/9", + "tenant/a0", "tenant/aA", "tenant/a\u007f", "tenant/a\u00e9", "tenant/a\U0001F600", "tenant/a\uffff" + ]; + await using var context = await CreateAsync("AzureBlob", ids); + context.Native.HierarchicalNamespace = hierarchicalNamespace; + for (var character = 0; character < 128; character++) + { + var bound = "tenant/a" + (char)character; + AssertMembership(ids.Where(id => string.CompareOrdinal(id, bound) >= 0).ToArray(), + await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/"), MinId = new(bound) }, TestContext.Current.CancellationToken))); + AssertMembership(ids.Where(id => string.CompareOrdinal(id, bound) <= 0).ToArray(), + await DrainAsync(context.Catalog.ListAsync( + new() { Prefix = new("tenant/"), MaxId = new(bound) }, TestContext.Current.CancellationToken))); + } + } + + [Theory] + [InlineData("AzureBlob", false)] + [InlineData("AzureBlob", true)] + [InlineData("S3", false)] + public async Task OrderedListAsync_TimePrefixedNamespaceStopsBeforeFuturePages(string kind, bool hierarchicalNamespace) { const string prefix = "jobs/shards/"; const string overdue = prefix + "20250101T0000000000000Z-11111111111111111111111111111111"; @@ -645,6 +754,7 @@ public async Task OrderedListAsync_TimePrefixedNamespaceStopsBeforeFuturePages(s ids.Add("jobs/shards"); await using var context = await CreateAsync(kind, ids.ToArray(), configureS3: options => options.UseOrderedListing = true); + context.Native.HierarchicalNamespace = hierarchicalNamespace; context.Native.Failure = new InvalidOperationException("future tail must not be requested"); context.Native.FailureAtRequest = 3; @@ -781,11 +891,14 @@ public async Task AzureBlobListAsync_BoundedPrefixListingErrorPropagates(int fai } [Theory] - [InlineData("AzureBlob", false)] - [InlineData("AzureBlob", true)] - [InlineData("S3", false)] - [InlineData("S3", true)] - public async Task OrderedListAsync_SeeksMinimumBeforeFetchingAnyPageAndIncludesBothEndpoints(string kind, bool includePrefix) + [InlineData("AzureBlob", false, false)] + [InlineData("AzureBlob", false, true)] + [InlineData("AzureBlob", true, false)] + [InlineData("AzureBlob", true, true)] + [InlineData("S3", false, false)] + [InlineData("S3", true, false)] + public async Task OrderedListAsync_SeeksMinimumBeforeFetchingAnyPageAndIncludesBothEndpoints( + string kind, bool includePrefix, bool hierarchicalNamespace) { const string day = "jobs/shards/20260909"; const string common = day + "T1200000000000Z-"; @@ -794,6 +907,7 @@ public async Task OrderedListAsync_SeeksMinimumBeforeFetchingAnyPageAndIncludesB var ids = Enumerable.Range(0, 256).Select(index => common + index.ToString("x32", CultureInfo.InvariantCulture)) .Concat([minimum, maximum, common + "f0000000000000000000000000000000"]).ToArray(); await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + context.Native.HierarchicalNamespace = hierarchicalNamespace; Assert.Equal([minimum, maximum], await DrainAsync(context.Catalog.ListAsync( new() { Prefix = includePrefix ? new(day) : default, MinId = new(minimum), MaxId = new(maximum) }, TestContext.Current.CancellationToken))); @@ -807,13 +921,15 @@ public async Task OrderedListAsync_SeeksMinimumBeforeFetchingAnyPageAndIncludesB } [Theory] - [InlineData("AzureBlob")] - [InlineData("S3")] - public async Task OrderedListAsync_PushesPartialDayPrefixWithoutAddingSeparator(string kind) + [InlineData("AzureBlob", false)] + [InlineData("AzureBlob", true)] + [InlineData("S3", false)] + public async Task OrderedListAsync_PushesPartialDayPrefixWithoutAddingSeparator(string kind, bool hierarchicalNamespace) { const string prefix = "jobs/shards/202609"; string[] ids = ["jobs/shards/20260831-a", prefix + "01-a", prefix + "09-b", "jobs/shards/20261001-a"]; await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + context.Native.HierarchicalNamespace = hierarchicalNamespace; Assert.Equal(ids[1..3], await DrainAsync(context.Catalog.ListAsync( new() { Prefix = new(prefix) }, TestContext.Current.CancellationToken))); @@ -854,12 +970,14 @@ public async Task S3ListAsync_OrderedSeekOnlyNarrowsBeyondNativePrefix( } [Theory] - [InlineData("AzureBlob")] - [InlineData("S3")] - public async Task OrderedListAsync_UnicodeBoundsDoNotUseUnsafeNativeOrdering(string kind) + [InlineData("AzureBlob", false)] + [InlineData("AzureBlob", true)] + [InlineData("S3", false)] + public async Task OrderedListAsync_UnicodeBoundsDoNotUseUnsafeNativeOrdering(string kind, bool hierarchicalNamespace) { string[] ids = ["a", "\ud800\udc00", "\ue000", "\uffff"]; await using var context = await CreateAsync(kind, ids, configureS3: options => options.UseOrderedListing = true); + context.Native.HierarchicalNamespace = hierarchicalNamespace; AssertMembership(ids[1..3], await DrainAsync(context.Catalog.ListAsync( new() { MinId = new(ids[1]), MaxId = new(ids[2]) }, TestContext.Current.CancellationToken))); @@ -1084,6 +1202,39 @@ public async Task S3ListAsync_CustomMappingWithBoundsOnlyDoesNotRequirePrefixMap Assert.Null(request.LowerStart); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task S3ListAsync_WhitespaceCommonPrefixPreservesBoundsWithPrefixMapper(bool ordered, bool customMapping) + { + string[] ids = [" 0", " a", " m", " z", " zz"]; + var mappedPrefixes = new List(); + var basePrefix = customMapping ? "current/" : string.Empty; + await using var context = await CreateAsync("S3", [], + keys: ids.Select(id => $"wal/{basePrefix}{id}").ToArray(), configureS3: options => + { + options.UseOrderedListing = ordered; + if (customMapping) + { + options.GetObjectKey = id => basePrefix + id.Value; + options.TryParseJournalId = key => new JournalId(key[basePrefix.Length..]); + } + + options.GetObjectKeyPrefix = prefix => + { + mappedPrefixes.Add(prefix.Value); + return basePrefix + prefix.Value; + }; + }); + + Assert.Equal([" a", " m", " z"], await DrainAsync(context.Catalog.ListAsync( + new() { MinId = new(" a"), MaxId = new(" z") }, TestContext.Current.CancellationToken))); + Assert.Empty(mappedPrefixes); + Assert.All(context.Native.Requests, request => Assert.Equal("wal/", request.Prefix)); + } + [Theory] [InlineData(null)] [InlineData("")] @@ -1384,6 +1535,7 @@ private sealed class NativeState { public List Requests { get; } = []; public bool EmptyFirstPage { get; set; } + public bool HierarchicalNamespace { get; set; } public Exception? Failure { get; set; } public int FailureAtRequest { get; set; } public Action? BeforeResponse { get; set; } @@ -1471,10 +1623,11 @@ public override AsyncPageable GetBlobsAsync(GetBlobsOptions options, C { state.BlobTraits.Add(options.Traits); Assert.Equal(BlobStates.None, options.States); + var comparer = state.HierarchicalNamespace ? HnsBlobNameComparer : StringComparer.Ordinal; return new FakePageable( state, records.Where(item => (options.Prefix is null || item.Name.StartsWith(options.Prefix, StringComparison.Ordinal)) - && (options.StartFrom is null || string.CompareOrdinal(item.Name, options.StartFrom) >= 0)) - .OrderBy(item => item.Name, StringComparer.Ordinal) + && (options.StartFrom is null || comparer.Compare(item.Name, options.StartFrom) >= 0)) + .OrderBy(item => item.Name, comparer) .Select(item => BlobsModelFactory.BlobItem( name: item.Name, deleted: false, properties: item.Properties, metadata: options.Traits.HasFlag(BlobTraits.Metadata) From 783d695324aa468fd8d0243005bcbd199d6507b5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 12:27:19 -0700 Subject: [PATCH 5/9] fix(journaling): preserve raw UTF-16 Redis identities --- src/Redis/Orleans.Journaling.Redis/README.md | 2 +- .../RedisJournalStorage.cs | 83 ++++++++--- .../RedisJournalStorageProvider.cs | 2 +- .../RedisJournalStorageCatalogTests.cs | 115 +++++++++++++- .../Journaling/RedisJournalStorageTests.cs | 141 ++++++++++++++++++ 5 files changed, 316 insertions(+), 27 deletions(-) diff --git a/src/Redis/Orleans.Journaling.Redis/README.md b/src/Redis/Orleans.Journaling.Redis/README.md index 43f4dada9e7..b7bb0c18885 100644 --- a/src/Redis/Orleans.Journaling.Redis/README.md +++ b/src/Redis/Orleans.Journaling.Redis/README.md @@ -64,7 +64,7 @@ Storage errors and cancellation propagate without provider-level retries, restar ## Redis key layout -Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. This layout applies to both default and custom `GetKeyName` mappings. The reversible key name is outside the SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI encoding preserves literal percent signs, separators, Unicode, and Redis glob characters in journal ids; the scan pattern also escapes glob characters in the configured key prefix. +Keys have the form `:journal:{}::metadata` or the same base with a `:data` suffix. This layout applies to both default and custom `GetKeyName` mappings. The reversible key name is outside the SHA-256 hash tag, preserving Redis Cluster colocation of each journal's data and metadata for atomic Lua operations. URI escaping encodes Unicode scalars, while `%uXXXX` encodes each unpaired UTF-16 surrogate using four uppercase hexadecimal digits. Literal percent signs are escaped as `%25`, so raw journal ids round-trip distinctly from escape sequences and replacement characters. The stored `$journal-id` uses the same encoding to preserve the identity through Redis string transport and custom-mapped discovery. Native scan patterns escape glob characters in the configured key prefix and broaden partial surrogate prefixes before applying the ordinal filters locally. **Alpha layout upgrade:** When upgrading from the previous hash-only layout, drain durable jobs before deploying the new version, then recreate journals using the new layout. diff --git a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs index 758e6e1baea..1e84b695cc8 100644 --- a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs +++ b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorage.cs @@ -392,6 +392,7 @@ or tostring(appendLength) ~= state[4] private readonly string _journalFormatKey; private readonly RedisJournalStorageOptions _options; private readonly JournalId _journalId; + private readonly string _encodedJournalId; private string? _contentETag; private long _appendLength; @@ -420,6 +421,7 @@ public RedisJournalStorage( _journalFormatKey = journalFormatKey; _options = options; _journalId = journalId; + _encodedJournalId = EncodeKeyName(journalId.Value); } public bool IsCompactionRequested => _options.CompactionThresholdBytes > 0 && _appendLength >= _options.CompactionThresholdBytes; @@ -435,7 +437,7 @@ public async ValueTask CreateIfNotExistsAsync( var result = await EvaluateArrayAsync( CreateIfNotExistsScript, _journalKeys, - BuildCreateArguments(eTag, _journalFormatKey, _journalId.Value, callerMetadata)).ConfigureAwait(false); + BuildCreateArguments(eTag, _journalFormatKey, _encodedJournalId, callerMetadata)).ConfigureAwait(false); var status = GetStatus(result, nameof(CreateIfNotExistsAsync)); if (status is CollisionStatus or InvalidMetadataStatus) { @@ -488,7 +490,7 @@ public async ValueTask CreateIfNotExistsAsync( BuildUpdateMetadataArguments( expectedETag, newETag, - _journalId.Value, + _encodedJournalId, removeValues, setValues)).ConfigureAwait(false); var status = GetStatus(result, nameof(UpdateMetadataAsync)); @@ -544,7 +546,7 @@ public async ValueTask AppendAsync(ReadOnlySequence value, CancellationTok newETag, payload, _journalFormatKey, - _journalId.Value, + _encodedJournalId, ]).ConfigureAwait(false); var status = GetStatus(result, nameof(AppendAsync)); ThrowForStatus(status, nameof(AppendAsync), expectedContentETag); @@ -568,7 +570,7 @@ public async ValueTask ReplaceAsync(ReadOnlySequence value, CancellationTo newETag, payload, _journalFormatKey, - _journalId.Value, + _encodedJournalId, ]).ConfigureAwait(false); var status = GetStatus(result, nameof(ReplaceAsync)); ThrowForStatus(status, nameof(ReplaceAsync), expectedContentETag); @@ -590,7 +592,7 @@ public async ValueTask DeleteAsync(CancellationToken cancellationToken) var result = await EvaluateArrayAsync( DeleteScript, _journalKeys, - [expectedContentETag, expectedExists ? "1" : "0", _journalId.Value]).ConfigureAwait(false); + [expectedContentETag, expectedExists ? "1" : "0", _encodedJournalId]).ConfigureAwait(false); var status = GetStatus(result, nameof(DeleteAsync)); if (status != AppearedStatus) { @@ -615,7 +617,7 @@ internal static RedisValue GetMetadataKeyPattern(string keyPrefix, string? journ journalIdPrefix = journalIdPrefix[..^1]; } - var encodedPrefix = Uri.EscapeDataString(journalIdPrefix ?? string.Empty); + var encodedPrefix = EncodeKeyName(journalIdPrefix ?? string.Empty); return $"{EscapeRedisPattern(keyPrefix)}:journal:{{*}}:{EscapeRedisPattern(encodedPrefix)}*:metadata"; } @@ -628,7 +630,7 @@ internal static RedisKey GetDataKey(string keyPrefix, string keyName) private static string GetJournalBaseKey(string keyPrefix, string keyName) { var hashTag = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyName))); - return $"{keyPrefix}:journal:{{{hashTag}}}:{Uri.EscapeDataString(keyName)}"; + return $"{keyPrefix}:journal:{{{hashTag}}}:{EncodeKeyName(keyName)}"; } internal static JournalId GetJournalIdFromMetadataKey(string keyPrefix, RedisKey metadataKey) @@ -643,9 +645,8 @@ internal static JournalId GetJournalIdFromMetadataKey(string keyPrefix, RedisKey && key.Length > keyNameOffset + suffix.Length && key.AsSpan(prefix.Length + hashTagLength, 2).SequenceEqual("}:")) { - var keyName = Uri.UnescapeDataString(key[keyNameOffset..^suffix.Length]); - if (TryParseJournalId(keyName, out var journalId) - && GetMetadataKey(keyPrefix, keyName) == metadataKey) + if (TryDecodeJournalId(key[keyNameOffset..^suffix.Length], out var journalId) + && GetMetadataKey(keyPrefix, journalId.Value) == metadataKey) { return journalId; } @@ -787,7 +788,7 @@ private StorageState CreateStorageState(RedisResult[] result, int startIndex, st throw new InvalidOperationException($"Redis journal '{_journalId}' has missing or invalid provider metadata."); } - if (!string.Equals(journalId, _journalId.Value, StringComparison.Ordinal)) + if (!string.Equals(journalId, _encodedJournalId, StringComparison.Ordinal)) { throw new InvalidOperationException( $"Redis journal key mapping collision ({operation}): the configured key for JournalId={_journalId} is already owned by journal '{journalId}'."); @@ -933,18 +934,66 @@ private static void ValidateCallerMetadataPropertyName(string key) private static bool IsProviderMetadataKey(string key) => key.StartsWith("$", StringComparison.Ordinal); - internal static bool TryParseJournalId(string value, out JournalId journalId) + internal static string EncodeKeyName(string value) { - try + StringBuilder? result = null; + var start = 0; + for (var index = 0; index < value.Length; index++) { - journalId = new JournalId(value); - return true; + var character = value[index]; + if (!char.IsSurrogate(character)) + { + continue; + } + + if (char.IsHighSurrogate(character) && index + 1 < value.Length && char.IsLowSurrogate(value[index + 1])) + { + index++; + continue; + } + + // URI escaping handles Unicode scalars; %uXXXX preserves individual unpaired UTF-16 code units. + result ??= new StringBuilder(); + result.Append(Uri.EscapeDataString(value[start..index])); + result.Append("%u").Append(((int)character).ToString("X4", CultureInfo.InvariantCulture)); + start = index + 1; } - catch (ArgumentException) + + return result is null + ? Uri.EscapeDataString(value) + : result.Append(Uri.EscapeDataString(value[start..])).ToString(); + } + + internal static bool TryDecodeJournalId(string value, out JournalId journalId) + { + journalId = default; + StringBuilder? result = null; + var start = 0; + while (value.IndexOf("%u", start, StringComparison.Ordinal) is var index && index >= 0) + { + if (index + 6 > value.Length + || !ushort.TryParse(value.AsSpan(index + 2, 4), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var codeUnit) + || !char.IsSurrogate((char)codeUnit)) + { + return false; + } + + result ??= new StringBuilder(); + result.Append(Uri.UnescapeDataString(value[start..index])); + result.Append((char)codeUnit); + start = index + 6; + } + + var decoded = result is null + ? Uri.UnescapeDataString(value) + : result.Append(Uri.UnescapeDataString(value[start..])).ToString(); + if (string.IsNullOrWhiteSpace(decoded) || !string.Equals(EncodeKeyName(decoded), value, StringComparison.Ordinal)) { - journalId = default; return false; } + + journalId = new JournalId(decoded); + return true; } private static string CreateETag() => Guid.NewGuid().ToString("N"); diff --git a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs index 0fb8b14e82d..aab1696f569 100644 --- a/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs +++ b/src/Redis/Orleans.Journaling.Redis/RedisJournalStorageProvider.cs @@ -173,7 +173,7 @@ public async IAsyncEnumerable ListAsync( value = (RedisValue)result[1]; } - if (!RedisJournalStorage.TryParseJournalId(value.ToString(), out var journalId)) + if (!RedisJournalStorage.TryDecodeJournalId(value.ToString(), out var journalId)) { throw new InvalidOperationException( $"Redis journal metadata '{batch[i]}' contains an invalid '{RedisJournalStorage.JournalIdMetadataKey}' value."); diff --git a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs index cd32ebf78ee..8c7f12b0712 100644 --- a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageCatalogTests.cs @@ -30,7 +30,7 @@ public async Task ListAsync_IncludeMetadataReturnsUnavailableWithoutFetchingMeta var id = JournalId.Create("redis", "projection"); var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, customMapping ? "mapped" : id.Value); var database = Substitute.For(); - database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey).Returns((RedisValue)id.Value); + database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey).Returns((RedisValue)RedisJournalStorage.EncodeKeyName(id.Value)); var provider = await CreateProviderAsync( database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanKeysAsync([key]))); var result = new List(); @@ -76,7 +76,7 @@ public async Task ListAsync_CustomMappingYieldsFirstBatchBeforeScanCompletesAndS .Returns(call => { metadataReads++; - return Task.FromResult((RedisValue)ids[call.ArgAt(0)].Value); + return Task.FromResult((RedisValue)RedisJournalStorage.EncodeKeyName(ids[call.ArgAt(0)].Value)); }); var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanAsync(TestContext.Current.CancellationToken))); var options = new ListOptions { Prefix = nonmatchingId, MinId = firstId, MaxId = new("a") }; @@ -144,7 +144,7 @@ public async Task ListAsync_BoundsAreInclusiveAndDoNotStopUnorderedScan(bool bou for (var index = 0; index < keys.Length; index++) { database.HashGetAsync(keys[index], RedisJournalStorage.JournalIdMetadataKey) - .Returns(Task.FromResult((RedisValue)ids[index].Value)); + .Returns(Task.FromResult((RedisValue)RedisJournalStorage.EncodeKeyName(ids[index].Value))); } var scanned = 0; @@ -190,7 +190,7 @@ public async Task ListAsync_LaterScanFailurePropagatesAfterEarlierIds(bool custo var scannedKeys = 0; var database = Substitute.For(); database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey) - .Returns(Task.FromResult((RedisValue)id.Value)); + .Returns(Task.FromResult((RedisValue)RedisJournalStorage.EncodeKeyName(id.Value))); var provider = await CreateProviderAsync(database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanAsync())); await using var enumerator = provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken); @@ -242,7 +242,7 @@ public async Task ListAsync_CancellationOrEarlyDisposalStopsScan(bool cancel, bo .Returns(_ => { metadataReads++; - return Task.FromResult((RedisValue)id.Value); + return Task.FromResult((RedisValue)RedisJournalStorage.EncodeKeyName(id.Value)); }); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); var provider = await CreateProviderAsync(database, customMapping ? CustomMappingOptions() : new(), CreateServer(ScanAsync(cancellation.Token))); @@ -292,7 +292,7 @@ public async Task ListAsync_DisconnectedLaterPrimaryPropagatesAfterEarlierIds() var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value); var database = Substitute.For(); database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey) - .Returns(Task.FromResult((RedisValue)id.Value)); + .Returns(Task.FromResult((RedisValue)RedisJournalStorage.EncodeKeyName(id.Value))); var disconnected = Substitute.For(); disconnected.IsConnected.Returns(false); var provider = await CreateProviderAsync(database, CreateServer(ScanAsync()), disconnected); @@ -504,6 +504,105 @@ public async Task ListAsync_DefaultMappingBroadensNativePrefixEndingInHighSurrog await AssertNoMetadataReadsAsync(database); } + [Fact] + public void KeyEncoding_RoundTripsRawUtf16WithoutAliases() + { + var cases = new (string Value, string Encoded)[] + { + ("jobs/\uD800", "jobs%2F%uD800"), + ("jobs/\uD801", "jobs%2F%uD801"), + ("jobs/\uDC00", "jobs%2F%uDC00"), + ("jobs/\uD800\uD800", "jobs%2F%uD800%uD800"), + ("jobs/\uDC00\uD800", "jobs%2F%uDC00%uD800"), + ("jobs/\uD800x", "jobs%2F%uD800x"), + ("jobs/\uFFFD", "jobs%2F%EF%BF%BD"), + ("jobs/%uD800", "jobs%2F%25uD800"), + ("jobs/\U0001F600", "jobs%2F%F0%9F%98%80"), + ("jobs/\u96EA", "jobs%2F%E9%9B%AA"), + ("jobs/\0", "jobs%2F%00"), + ("jobs/%uD800/\uD800/\U0001F600/%", "jobs%2F%25uD800%2F%uD800%2F%F0%9F%98%80%2F%25"), + }; + var keys = new HashSet(); + foreach (var (value, encoded) in cases) + { + var metadataKey = RedisJournalStorage.GetMetadataKey(KeyPrefix, value); + var dataKey = RedisJournalStorage.GetDataKey(KeyPrefix, value); + + Assert.EndsWith($":{encoded}:metadata", metadataKey.ToString()); + Assert.Equal(metadataKey.ToString()[..^"metadata".Length] + "data", dataKey.ToString()); + Assert.Equal(new JournalId(value), RedisJournalStorage.GetJournalIdFromMetadataKey(KeyPrefix, metadataKey)); + Assert.True(keys.Add(metadataKey)); + } + } + + [Theory] + [InlineData("%u")] + [InlineData("%uD80")] + [InlineData("%uGGGG")] + [InlineData("%u0041")] + [InlineData("%ud800")] + [InlineData("%uD800%uDC00")] + [InlineData("%FF")] + [InlineData("%20")] + public void JournalIdDecoding_RejectsNonCanonicalEscapes(string encoded) + { + Assert.False(RedisJournalStorage.TryDecodeJournalId(encoded, out var id)); + Assert.Equal(default, id); + } + + [Theory] + [InlineData(0xD800, false)] + [InlineData(0xD800, true)] + [InlineData(0xDC00, false)] + [InlineData(0xDC00, true)] + public async Task ListAsync_DefaultMappingPreservesRawUtf16RangesWithoutMetadataReads(int codeUnit, bool bounded) + { + var prefix = "jobs/" + (char)codeUnit; + var minimum = new JournalId(prefix + "/a"); + var maximum = new JournalId(prefix + "/b"); + var future = new JournalId(prefix + "/z"); + var pairedOrRepeated = new JournalId(prefix + "\uDC00"); + var ids = new[] { future, minimum, new("jobs/\uFFFD/a"), maximum, pairedOrRepeated, new("jobs/%uD800/a") }; + var database = Substitute.For(); + var server = CreateServer(ScanKeysAsync(ids.Select(id => RedisJournalStorage.GetMetadataKey(KeyPrefix, id.Value))), "jobs/"); + var provider = await CreateProviderAsync(database, server); + + var result = await ReadIdsAsync(provider, new() + { + Prefix = new(prefix), + MinId = bounded ? minimum : default, + MaxId = bounded ? maximum : default, + }); + + Assert.Equal(bounded ? [minimum, maximum] : new[] { future, minimum, maximum, pairedOrRepeated }, result); + _ = server.Received(1).KeysAsync(0, "catalog-tests:journal:{*}:jobs%2F*:metadata", pageSize: 250); + await AssertNoMetadataReadsAsync(database); + } + + [Fact] + public async Task ListAsync_CustomMappingPreservesRawUtf16MetadataFromRedis() + { + var ids = new JournalId[] { new("jobs/\uD800"), new("jobs/%uD800"), new("jobs/\uDC00") }; + var encodedIds = new[] { "jobs%2F%uD800", "jobs%2F%25uD800", "jobs%2F%uDC00" }; + var keys = ids.Select((_, index) => RedisJournalStorage.GetMetadataKey(KeyPrefix, $"opaque-{index}")).ToArray(); + var database = Substitute.For(); + for (var index = 0; index < keys.Length; index++) + { + database.HashGetAsync(keys[index], RedisJournalStorage.JournalIdMetadataKey) + .Returns((RedisValue)Encoding.UTF8.GetBytes(encodedIds[index])); + } + + var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanKeysAsync(keys))); + + var result = await ReadIdsAsync(provider, new() { Prefix = new("jobs/") }); + + Assert.Equal(ids, result); + foreach (var key in keys) + { + await database.Received(1).HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey); + } + } + [Theory] [InlineData("jobs/", "z", "a", false)] [InlineData("jobs/", "z", "a", true)] @@ -597,7 +696,7 @@ public async Task ListAsync_CustomMappingReadsBatchesConcurrently() for (var i = 0; i < completions.Length; i++) { - completions[i].SetResult(ids[i].Value); + completions[i].SetResult(RedisJournalStorage.EncodeKeyName(ids[i].Value)); } Assert.True(await moveNext); @@ -641,7 +740,7 @@ public async Task ListAsync_CustomMappingMissingMetadataChecksDeletionOrReportsC var key = RedisJournalStorage.GetMetadataKey(KeyPrefix, "opaque"); var database = Substitute.For(); database.HashGetAsync(key, RedisJournalStorage.JournalIdMetadataKey).Returns(Task.FromResult(RedisValue.Null)); - var response = status == 1 ? new RedisValue[] { status, value } : [status]; + var response = status == 1 ? new RedisValue[] { status, RedisJournalStorage.EncodeKeyName(value!) } : [status]; database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(RedisResult.Create(response))); var provider = await CreateProviderAsync(database, CustomMappingOptions(), CreateServer(ScanKeysAsync([key]))); diff --git a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs index 61acc384d53..f71fc891038 100644 --- a/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs +++ b/test/Extensions/Orleans.Redis.Tests/Journaling/RedisJournalStorageTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; using Microsoft.Extensions.Options; +using NSubstitute; using Orleans.Configuration; using Orleans.Journaling; using Orleans.Runtime; @@ -18,6 +19,146 @@ namespace Tester.Redis.Journaling; [TestCategory("BVT")] public sealed class RedisJournalStorageTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StorageOperations_PreserveRawUtf16IdentityAcrossRedisTransport(bool customMapping) + { + var id = new JournalId("jobs/\uD800/%uD800/\uDC00/\U0001F600"); + const string encodedId = "jobs%2F%uD800%2F%25uD800%2F%uDC00%2F%F0%9F%98%80"; + var options = new RedisJournalStorageOptions(); + var database = Substitute.For(); + var keyName = customMapping ? "mapped/\uD800" : id.Value; + var calls = new List<(RedisKey[] Keys, RedisValue[] Values)>(); + var responses = new Queue( + [ + MetadataResponse(), + MetadataResponse(), + RedisResult.Create(new RedisValue[] { 1, 2 }), + RedisResult.Create((RedisValue[])[1, new byte[] { 1, 2 }, .. MetadataValues()]), + MetadataResponse(), + RedisResult.Create(new RedisValue[] { 1, 1 }), + RedisResult.Create(new RedisValue[] { 1 }), + ]); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + calls.Add((call.ArgAt(1), call.ArgAt(2))); + return responses.Dequeue(); + }); + var storage = new RedisJournalStorage(database, "raw-utf16", keyName, "json", options, id); + + Assert.True(await storage.CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken)); + Assert.Equal("json", (await storage.GetMetadataAsync(TestContext.Current.CancellationToken))!.Format); + await storage.AppendAsync(new ReadOnlySequence([1, 2]), TestContext.Current.CancellationToken); + + var reader = new RedisJournalStorage(database, "raw-utf16", keyName, "json", options, id); + var consumer = new CapturingJournalStorageConsumer(); + await reader.ReadAsync(consumer, TestContext.Current.CancellationToken); + Assert.Equal([1, 2], consumer.Bytes); + Assert.True(consumer.IsCompleted); + Assert.NotNull(await reader.UpdateMetadataAsync( + new Dictionary { ["owner"] = "test" }, + cancellationToken: TestContext.Current.CancellationToken)); + await reader.ReplaceAsync(new ReadOnlySequence([3]), TestContext.Current.CancellationToken); + await reader.DeleteAsync(TestContext.Current.CancellationToken); + + Assert.Empty(responses); + foreach (var (callIndex, argumentIndex) in new[] { (0, 2), (2, 5), (4, 3), (5, 5), (6, 2) }) + { + Assert.Equal(encodedId, calls[callIndex].Values[argumentIndex].ToString()); + Assert.Equal(Encoding.UTF8.GetBytes(encodedId), (byte[]?)calls[callIndex].Values[argumentIndex]); + } + + Assert.All(calls, call => Assert.Contains(RedisJournalStorage.GetMetadataKey("raw-utf16", keyName), call.Keys)); + + static RedisValue[] MetadataValues() => + [ + RedisJournalStorage.ETagMetadataKey, "etag", + RedisJournalStorage.ContentETagMetadataKey, "content-etag", + RedisJournalStorage.AppendLengthMetadataKey, "0", + RedisJournalStorage.FormatMetadataKey, "json", + RedisJournalStorage.JournalIdMetadataKey, Encoding.UTF8.GetBytes(encodedId), + RedisJournalStorage.SchemaVersionMetadataKey, RedisJournalStorage.CurrentSchemaVersion, + ]; + + static RedisResult MetadataResponse() => RedisResult.Create((RedisValue[])[1, .. MetadataValues()]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RawUtf16Identities_RoundTripThroughStorageAndCatalog(bool customMapping) + { + TestUtils.CheckForRedis(); + await using var context = await RedisJournalStorageTestContext.CreateAsync( + TestContext.Current.CancellationToken, + options => + { + if (customMapping) + { + options.GetKeyName = static id => "mapped/" + id.Value; + } + }); + var ids = new JournalId[] + { + new("jobs/\uD800"), new("jobs/\uD801"), new("jobs/\uDC00"), + new("jobs/\uFFFD"), new("jobs/%uD800"), new("jobs/\U0001F600"), + }; + for (var index = 0; index < ids.Length; index++) + { + var storage = context.Provider.CreateStorage(ids[index]); + switch (index % 3) + { + case 0: + Assert.True(await storage.CreateIfNotExistsAsync(cancellationToken: TestContext.Current.CancellationToken)); + await storage.AppendAsync(new ReadOnlySequence([(byte)index]), TestContext.Current.CancellationToken); + break; + case 1: + await storage.AppendAsync(new ReadOnlySequence([(byte)index]), TestContext.Current.CancellationToken); + break; + case 2: + await storage.ReplaceAsync(new ReadOnlySequence([(byte)index]), TestContext.Current.CancellationToken); + break; + } + } + + var listed = await ToListAsync(context.Provider.ListAsync(new() { Prefix = new("jobs/") }, + TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); + Assert.Equal(ids.OrderBy(id => id.Value, StringComparer.Ordinal), listed.OrderBy(id => id.Value, StringComparer.Ordinal)); + + var bounded = await ToListAsync(context.Provider.ListAsync(new() + { + Prefix = new("jobs/\uD800"), + MinId = ids[0], + MaxId = ids[0], + }, TestContext.Current.CancellationToken), TestContext.Current.CancellationToken); + Assert.Equal([ids[0]], bounded); + + for (var index = 0; index < ids.Length; index++) + { + var storage = context.Provider.CreateStorage(ids[index]); + var consumer = new CapturingJournalStorageConsumer(); + await storage.ReadAsync(consumer, TestContext.Current.CancellationToken); + Assert.Equal([(byte)index], consumer.Bytes); + Assert.True(consumer.IsCompleted); + var metadata = await storage.UpdateMetadataAsync( + new Dictionary { ["owner"] = "test" }, + expectedETag: consumer.Metadata!.ETag, + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("test", metadata!.Properties["owner"]); + await storage.ReplaceAsync(new ReadOnlySequence([42]), TestContext.Current.CancellationToken); + var replaced = new CapturingJournalStorageConsumer(); + await context.Provider.CreateStorage(ids[index]).ReadAsync(replaced, TestContext.Current.CancellationToken); + Assert.Equal([42], replaced.Bytes); + await storage.DeleteAsync(TestContext.Current.CancellationToken); + Assert.Null(await context.Provider.CreateStorage(ids[index]).GetMetadataAsync(TestContext.Current.CancellationToken)); + } + + Assert.Empty(await ToListAsync(context.Provider.ListAsync(cancellationToken: TestContext.Current.CancellationToken), + TestContext.Current.CancellationToken)); + } + [Theory] [InlineData("")] [InlineData(" ")] From f62c4cb0f794da50b84a081e5d6e78ad8e7878ff Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 14 Sep 2026 22:53:29 -0700 Subject: [PATCH 6/9] feat(durable-jobs): discover shards by start time Split the time-addressable discovery implementation from dotnet/orleans#11210, preserving the final source behavior and contributor fixes at 7c8dae07d451cda3ebc949fc02ce23c15476d169. --- .../Hosting/DurableJobsOptions.cs | 24 + src/Orleans.DurableJobs/JobShardId.cs | 11 +- src/Orleans.DurableJobs/JobShardManager.cs | 13 + .../JournaledJobShardManager.cs | 130 +++-- .../LocalDurableJobManager.cs | 66 ++- src/Orleans.DurableJobs/README.md | 62 +- .../Orleans.DurableJobs.cs | 13 +- .../DurableJobs/DurableJobsOptionsTests.cs | 68 +++ .../LocalDurableJobManagerTests.cs | 481 +++++++++++++++- .../JournaledJobShardDiscoveryTests.cs | 538 ++++++++++++++++++ .../JournaledJobShardManagerTests.cs | 185 +++++- .../JournaledJobShardStateTests.cs | 52 ++ 12 files changed, 1539 insertions(+), 104 deletions(-) create mode 100644 test/Orleans.Core.Tests/DurableJobs/DurableJobsOptionsTests.cs create mode 100644 test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs diff --git a/src/Orleans.DurableJobs/Hosting/DurableJobsOptions.cs b/src/Orleans.DurableJobs/Hosting/DurableJobsOptions.cs index 1ee468289d6..fcce49e69ef 100644 --- a/src/Orleans.DurableJobs/Hosting/DurableJobsOptions.cs +++ b/src/Orleans.DurableJobs/Hosting/DurableJobsOptions.cs @@ -29,6 +29,21 @@ public sealed class DurableJobsOptions /// public TimeSpan ShardActivationBufferPeriod { get; set; } = TimeSpan.FromMinutes(5); + /// + /// Gets or sets how far ahead of the current time shard discovery loads eligible shards. + /// A value of zero discovers shards whose start time is at or before the current time. + /// The resulting discovery horizon is capped at . + /// Must be non-negative. Default: 10 minutes. + /// + public TimeSpan ShardLoadLookaheadPeriod { get; set; } = TimeSpan.FromMinutes(10); + + /// + /// Gets or sets the interval between periodic shard discovery and writable-shard cleanup checks. + /// Membership changes also trigger a fresh discovery sweep. + /// Must be between 1 and 4294967294 milliseconds, inclusive. Default: 5 minutes. + /// + public TimeSpan ShardCheckInterval { get; set; } = TimeSpan.FromMinutes(5); + /// /// Gets or sets the number of writable shards to use for each shard time bucket. /// Increasing this value distributes jobs with the same due-time bucket across multiple shard journals. @@ -202,6 +217,15 @@ public void ValidateConfiguration() { throw new OrleansConfigurationException("DurableJobsOptions.ShardDuration must be greater than zero."); } + if (options.ShardLoadLookaheadPeriod < TimeSpan.Zero) + { + throw new OrleansConfigurationException("DurableJobsOptions.ShardLoadLookaheadPeriod must be non-negative."); + } + if (options.ShardCheckInterval < TimeSpan.FromMilliseconds(1) || + options.ShardCheckInterval > TimeSpan.FromMilliseconds(uint.MaxValue - 1)) + { + throw new OrleansConfigurationException("DurableJobsOptions.ShardCheckInterval must be between 1 and 4294967294 milliseconds, inclusive."); + } if (options.ShardStripeCount <= 0) { throw new OrleansConfigurationException("DurableJobsOptions.ShardStripeCount must be greater than zero."); diff --git a/src/Orleans.DurableJobs/JobShardId.cs b/src/Orleans.DurableJobs/JobShardId.cs index a6b6e14480a..57ea9458a6e 100644 --- a/src/Orleans.DurableJobs/JobShardId.cs +++ b/src/Orleans.DurableJobs/JobShardId.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using Orleans.Journaling; namespace Orleans.DurableJobs; @@ -7,6 +8,7 @@ internal readonly record struct JobShardId { private const string RootSegment = "jobs"; private const string ShardsSegment = "shards"; + private const string StartTimeFormat = "yyyyMMdd'T'HHmmssfffffff'Z'"; public JobShardId(string value) { @@ -18,7 +20,14 @@ public JobShardId(string value) public static JournalId StoragePrefix => JournalId.Create(RootSegment, ShardsSegment); - public static JobShardId New() => new(Guid.NewGuid().ToString("N")); + public static JobShardId New(DateTimeOffset startTime) + => new($"{FormatStartTime(startTime)}-{Guid.NewGuid():N}"); + + public static JournalId GetMaxJournalId(DateTimeOffset startTime) + => JournalId.Create(RootSegment, ShardsSegment, $"{FormatStartTime(startTime)}~"); + + private static string FormatStartTime(DateTimeOffset startTime) + => startTime.UtcDateTime.ToString(StartTimeFormat, CultureInfo.InvariantCulture); public static JobShardId Parse(string value) => new(value); diff --git a/src/Orleans.DurableJobs/JobShardManager.cs b/src/Orleans.DurableJobs/JobShardManager.cs index 5a1a9408768..21efe747874 100644 --- a/src/Orleans.DurableJobs/JobShardManager.cs +++ b/src/Orleans.DurableJobs/JobShardManager.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Orleans.Runtime; @@ -19,6 +20,18 @@ public abstract class JobShardManager /// protected SiloAddress SiloAddress { get; } + internal virtual async IAsyncEnumerable DiscoverJobShardsAsync( + DateTimeOffset maxDueTime, + int maxNewClaims, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var shard in await AssignJobShardsAsync(maxDueTime, maxNewClaims, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return shard; + } + } + /// /// Initializes a new instance of the class. /// diff --git a/src/Orleans.DurableJobs/JournaledJobShardManager.cs b/src/Orleans.DurableJobs/JournaledJobShardManager.cs index d7dbd929887..aeab8738474 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardManager.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardManager.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -40,7 +41,6 @@ internal sealed class JournaledJobShardManager : JobShardManager // (via UnregisterShardAsync). Mis-cache from split-brain is bounded by storage-layer ETag // conflicts triggering InconsistentStateException → the journaling layer's recovery path. private readonly ConcurrentDictionary _ownedShards = new(StringComparer.Ordinal); - public JournaledJobShardManager( ILocalSiloDetails localSiloDetails, IJournaledStateManagerFactory stateManagerFactory, @@ -82,73 +82,107 @@ private static SiloAddress GetSiloAddress(ILocalSiloDetails localSiloDetails) public override async Task> AssignJobShardsAsync(DateTimeOffset maxDueTime, int maxNewClaims, CancellationToken cancellationToken) { var result = new List(); - var newClaimCount = 0; - var membershipSnapshot = _membershipService.CurrentSnapshot; + await foreach (var shard in DiscoverJobShardsAsync(maxDueTime, maxNewClaims, cancellationToken)) + { + result.Add(shard); + } + + return result; + } - await foreach (var entry in _catalog.ListAsync(new() { Prefix = new(JobShardId.StoragePrefix.Value + "/") }, cancellationToken)) + internal override async IAsyncEnumerable DiscoverJobShardsAsync( + DateTimeOffset maxDueTime, + int maxNewClaims, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var storageEntries = new SortedDictionary(Comparer.Create( + static (left, right) => StringComparer.Ordinal.Compare(left.Value, right.Value))); + var options = new ListOptions { - var descriptor = await GetDescriptorAsync(entry.Id, cancellationToken); - if (descriptor is null || descriptor.Poisoned || descriptor.StartTime > maxDueTime) - { - continue; - } + Prefix = new JournalId(JobShardId.StoragePrefix.Value + "/"), + MaxId = JobShardId.GetMaxJournalId(maxDueTime), + IncludeMetadata = true + }; + await foreach (var entry in _catalog.ListAsync(options, cancellationToken)) + { + storageEntries.TryAdd(entry.Id, entry); + } - if (descriptor.MembershipVersion > membershipSnapshot.Version) + // Providers can return identities in any order. Names order the selected shards by UTC start time. + var newClaimCount = 0; + foreach (var entry in storageEntries.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + var (shard, claimed) = await TryAssignShardAsync(entry, maxDueTime, newClaimCount < maxNewClaims, cancellationToken); + if (claimed) { - // Refresh membership to at least that version. - await _membershipService.Refresh(descriptor.MembershipVersion, cancellationToken); - membershipSnapshot = _membershipService.CurrentSnapshot; + newClaimCount++; } - - if (descriptor.Owner is { } owner && owner.Equals(SiloAddress)) + if (shard is not null) { - result.Add(await GetOrOpenShardAsync(descriptor, cancellationToken)); - continue; + yield return shard; } + } + } - // Determine if this is an adopted shard (taken from dead owner) vs orphaned (gracefully released). - var isAdopted = false; - if (descriptor.Owner is { } previousOwner) - { - var ownerStatus = membershipSnapshot.GetSiloStatus(previousOwner); - if (ownerStatus is not SiloStatus.Dead and not SiloStatus.None) - { - // Owner is still active and it's not me, skip this shard. - continue; - } + private async ValueTask<(IJobShard? Shard, bool Claimed)> TryAssignShardAsync( + JournalCatalogEntry entry, DateTimeOffset maxDueTime, bool canClaim, CancellationToken cancellationToken) + { + var descriptor = entry.Metadata is { ETag: not null } metadata + ? ShardCatalogProperties.From(entry.Id, metadata) + : await GetDescriptorAsync(entry.Id, cancellationToken); + if (descriptor is null || descriptor.Poisoned || descriptor.StartTime > maxDueTime) + { + return default; + } - isAdopted = ownerStatus == SiloStatus.Dead; - } + var membershipSnapshot = _membershipService.CurrentSnapshot; + if (descriptor.MembershipVersion > membershipSnapshot.Version) + { + await _membershipService.Refresh(descriptor.MembershipVersion, cancellationToken); + membershipSnapshot = _membershipService.CurrentSnapshot; + } - // Respect the slow-start budget: skip claiming if we've exhausted the budget. - // This must be checked before incrementing the adopted count to avoid - // inflating the count when the shard isn't actually claimed. - if (newClaimCount >= maxNewClaims) - { - continue; - } + cancellationToken.ThrowIfCancellationRequested(); + if (descriptor.Owner is { } owner && owner.Equals(SiloAddress)) + { + return (await GetOrOpenShardAsync(descriptor, cancellationToken), false); + } - // Try to claim orphaned or adopted shard. - var claimedShard = await TryClaimShardAsync(descriptor, isAdopted, cancellationToken); - if (claimedShard is null) + var isAdopted = false; + if (descriptor.Owner is { } previousOwner) + { + var ownerStatus = membershipSnapshot.GetSiloStatus(previousOwner); + if (ownerStatus is not SiloStatus.Dead and not SiloStatus.None) { - // Either poisoned shard or someone else took ownership. - continue; + return default; } - _jobShardCache[claimedShard.Id] = claimedShard; - result.Add(claimedShard); - newClaimCount++; + isAdopted = ownerStatus == SiloStatus.Dead; } - return result; + // Exhausting the claim budget still advances discovery to later locally owned shards. + if (!canClaim) + { + return default; + } + + var claimedShard = await TryClaimShardAsync(descriptor, isAdopted, cancellationToken); + if (claimedShard is null) + { + return default; + } + + _jobShardCache[claimedShard.Id] = claimedShard; + return (claimedShard, true); } public override async Task CreateShardAsync(DateTimeOffset minDueTime, DateTimeOffset maxDueTime, IDictionary metadata, CancellationToken cancellationToken) { while (true) { - var shardId = JobShardId.New(); + var shardId = JobShardId.New(minDueTime); var storageId = shardId.ToJournalId(); var initialProperties = CreateInitialProperties(minDueTime, maxDueTime, metadata); var storage = _storageProvider.CreateStorage(storageId); @@ -442,8 +476,10 @@ private IDurableValueCommandCodec CreateOperationC IEnumerable? remove, CancellationToken cancellationToken) { + var expectedETag = descriptor.Properties.ETag + ?? throw new InvalidOperationException($"DurableJobs shard '{descriptor.ShardId}' requires a storage metadata ETag for conditional ownership updates."); var storage = _storageProvider.CreateStorage(descriptor.StorageId); - return await storage.UpdateMetadataAsync(set, remove, descriptor.Properties.ETag, cancellationToken); + return await storage.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); } private Dictionary CreateInitialProperties(DateTimeOffset minDueTime, DateTimeOffset maxDueTime, IDictionary? metadata) diff --git a/src/Orleans.DurableJobs/LocalDurableJobManager.cs b/src/Orleans.DurableJobs/LocalDurableJobManager.cs index 4f34d48b28a..b5bc991de86 100644 --- a/src/Orleans.DurableJobs/LocalDurableJobManager.cs +++ b/src/Orleans.DurableJobs/LocalDurableJobManager.cs @@ -430,32 +430,36 @@ private async Task PeriodicShardCheck() { await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext); - using var timer = new PeriodicTimer(TimeSpan.FromMinutes(10), _timeProvider); + using var timer = new PeriodicTimer(_options.ShardCheckInterval, _timeProvider); - Task timerTask = Task.CompletedTask; + var timerTask = timer.WaitForNextTickAsync(_cts.Token).AsTask(); + var signalTask = _shardCheckSignal.WaitAsync(_cts.Token); while (!_cts.Token.IsCancellationRequested) { try { - // Wait for either periodic timer OR signal from membership changes - if (timerTask.IsCompleted) + var completedTask = await Task.WhenAny(timerTask, signalTask); + await completedTask; + // Retain the other pending wait: it can complete during discovery. + if (ReferenceEquals(completedTask, timerTask)) { timerTask = timer.WaitForNextTickAsync(_cts.Token).AsTask(); } - - var signalTask = _shardCheckSignal.WaitAsync(_cts.Token); - await Task.WhenAny(timerTask, signalTask); + else + { + signalTask = _shardCheckSignal.WaitAsync(_cts.Token); + } await ProcessShardCheckCycleAsync(_cts.Token); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (_cts.IsCancellationRequested) { break; } catch (Exception ex) { LogErrorInPeriodicCheck(_logger, ex); - await Task.Delay(TimeSpan.FromSeconds(5), _cts.Token).SuppressThrowing(); + await Task.Delay(TimeSpan.FromSeconds(5), _timeProvider, _cts.Token).SuppressThrowing(); } } } @@ -483,30 +487,39 @@ internal async Task ProcessShardCheckCycleAsync(CancellationToken cancellationTo } } - // Compute the slow-start budget for this cycle - var budget = ComputeClaimBudget(); - - // Query ShardManager for assigned shards (source of truth) - var shards = await _shardManager.AssignJobShardsAsync(now.AddHours(1), budget, cancellationToken); + await ProcessShardDiscoveryAsync(cancellationToken); + } - // Count newly claimed shards (those not already in our cache) + private async Task ProcessShardDiscoveryAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var now = _timeProvider.GetUtcNow(); + var maxDueTime = new DateTimeOffset(now.UtcDateTime.AddClamped(_options.ShardLoadLookaheadPeriod)); + // Compute the slow-start budget for this sweep. + var budget = ComputeClaimBudget(); var newClaimsThisCycle = 0; - if (shards.Count > 0) + var assignedCount = 0; + await foreach (var shard in _shardManager.DiscoverJobShardsAsync(maxDueTime, budget, cancellationToken)) { - LogAssignedShards(_logger, shards.Count); - foreach (var shard in shards) + // Take responsibility for yielded resources before observing cancellation. + if (_shardCache.TryAdd(shard.Id, shard)) { - if (_shardCache.TryAdd(shard.Id, shard)) - { - newClaimsThisCycle++; - } + newClaimsThisCycle++; + _totalClaimedShards++; + } - if (!_runningShards.ContainsKey(shard.Id)) - { - TryActivateShard(shard); - } + cancellationToken.ThrowIfCancellationRequested(); + assignedCount++; + if (!_runningShards.ContainsKey(shard.Id)) + { + TryActivateShard(shard); } } + + if (assignedCount > 0) + { + LogAssignedShards(_logger, assignedCount); + } else { LogNoShardsToAssign(_logger); @@ -514,7 +527,6 @@ internal async Task ProcessShardCheckCycleAsync(CancellationToken cancellationTo if (newClaimsThisCycle > 0) { - _totalClaimedShards += newClaimsThisCycle; LogOrphanedShardsClaimed(_logger, newClaimsThisCycle, _totalClaimedShards); } } diff --git a/src/Orleans.DurableJobs/README.md b/src/Orleans.DurableJobs/README.md index 66678475b15..230b6501705 100644 --- a/src/Orleans.DurableJobs/README.md +++ b/src/Orleans.DurableJobs/README.md @@ -81,6 +81,10 @@ builder.UseOrleans(siloBuilder => { // Duration of each job shard (jobs are partitioned by time) options.ShardDuration = TimeSpan.FromMinutes(5); + + // Load eligible shards within this horizon and check at this interval + options.ShardLoadLookaheadPeriod = TimeSpan.FromMinutes(10); + options.ShardCheckInterval = TimeSpan.FromMinutes(5); // Maximum number of jobs that can execute concurrently on each silo options.MaxConcurrentJobsPerSilo = 100; @@ -101,6 +105,56 @@ builder.UseOrleans(siloBuilder => }); ``` +## Shard discovery and lookahead + +Each silo discovers shards whose start time is within `DurableJobsOptions.ShardLoadLookaheadPeriod` +of its current Durable Jobs time-provider clock. The default lookahead is ten minutes. +A discovered shard starts processing once its start time enters `ShardActivationBufferPeriod`. +`DurableJobsOptions.ShardCheckInterval` controls periodic discovery and writable-shard cleanup +checks, with a default of five minutes. Membership changes also trigger checks. +The lookahead accepts non-negative durations; zero selects shards whose start time is at or +before the current time. The discovery horizon is capped at `DateTimeOffset.MaxValue`. +The check interval accepts durations from 1 to 4294967294 milliseconds. + +Shard journals use names such as +`jobs/shards/20260909T1200000000000Z-`. The fixed-width UTC start time +precedes the unique suffix, so ordinal name order is shard-start-time order. Each sweep +lists the raw `jobs/shards/` prefix with an inclusive `ListOptions.MaxId` bound covering the lookahead +horizon. The range includes every earlier start time, including jobs overdue after a long +outage. Future shard identities are filtered by the catalog before candidate metadata reads. + +Each periodic or membership check starts a fresh, locally scoped sweep. Discovery requests +catalog metadata, orders and deduplicates the selected entries, then uses each supplied +ownership snapshot with an ETag or reads current metadata when that snapshot is unavailable. +Claims run oldest first and require the snapshot's ETag for conditional updates, so a +concurrent ownership change rejects a stale claim. Providers used for Durable Jobs supply +metadata ETags and enforce conditional updates; a missing ETag surfaces as a discovery error. +Assigned shards are delivered as they are opened, allowing execution to proceed while later +candidates are evaluated. The claim budget limits new claims; +locally owned shards remain eligible after that budget is exhausted. + +Catalog providers apply raw-prefix and range constraints using their storage capabilities. +The timestamp representation also supports narrower day/hour prefixes and inclusive `MinId` +and `MaxId` intervals for callers selecting a specific time window. Recovery starts at the +shard namespace's beginning so that all overdue jobs remain eligible. +Azure Table's default mapping uses indexed key ranges. Azure Blob and ordered general-purpose +S3 listings can seek lower bounds and stop at upper bounds. S3 Express and Redis filter time +bounds during their provider-defined traversal. Discovery orders the selected names itself +to provide consistent oldest-first processing across providers. Storage listing work and request +latency remain provider-dependent. Blob, Table, and Volatile catalogs supply metadata snapshots +alongside identities; S3 and Redis require separate candidate metadata reads. + +The sweep owns its enumeration and selected identity set until completion. Storage errors +propagate to the runtime's error reporting, and a later check starts a fresh sweep. Shards +already delivered to the local manager are tracked before cancellation is observed and +continue through their execution lifecycle. +Cancellation flows through listing, metadata, and journal operations. + +Shorter lookahead periods reduce early loading of recovered shards. Shorter check intervals +increase sweep frequency and reduce the wait for newly inserted or newly eligible shards. +The public `JobShardManager.AssignJobShardsAsync` method collects the same ordered discovery +stream into its full-result list. + ## Shutdown lifecycle Scheduling, cancellation requests, and activation use the shared @@ -121,10 +175,10 @@ requests, awaiting execution, and releasing shards. Successful scheduling and cancellation writes retain their result, and successful shard creations remain owned even when cancellation races with their completion. Shutdown then -awaits the active shard check and every admitted shard's execution and cleanup. It unregisters cached shards which -remained inactive using the shutdown token, then disposes them. The journaled provider -releases populated shards for another silo to claim and deletes empty shards, including -creations which completed after request cancellation. +awaits the active sweep and every admitted shard's execution and cleanup. It unregisters +cached shards which remained inactive using the shutdown token, then disposes them. The +journaled provider releases populated shards for another silo to claim and deletes empty +shards, including creations which completed after request cancellation. ## Usage Examples diff --git a/src/api/Orleans.DurableJobs/Orleans.DurableJobs.cs b/src/api/Orleans.DurableJobs/Orleans.DurableJobs.cs index 5a52f588388..fd99ed1f2ce 100644 --- a/src/api/Orleans.DurableJobs/Orleans.DurableJobs.cs +++ b/src/api/Orleans.DurableJobs/Orleans.DurableJobs.cs @@ -41,7 +41,7 @@ public enum DurableJobMutationResult { Applied = 0, JobNotFound = 1, - OwnershipLost = 2, + OwnershipLost = 2 } [GenerateSerializer] @@ -95,7 +95,6 @@ public static partial class DurableJobTimeProviderNames public partial interface IDurableJobFeatureHandler { bool CanHandle(string jobName); - System.Threading.Tasks.ValueTask ExecuteJobAsync(IJobRunContext context, System.Threading.CancellationToken attemptCancellationToken); } @@ -136,8 +135,8 @@ public partial interface IJobShard : System.IAsyncDisposable System.Threading.Tasks.Task RemoveJobAsync(string jobId, System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.Task RescheduleJobAsync(IJobRunContext jobContext, System.DateTimeOffset newDueTime, System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.Task RetryJobLaterAsync(IJobRunContext jobContext, System.DateTimeOffset newDueTime, System.Threading.CancellationToken cancellationToken); - System.Threading.Tasks.Task TryStartAttemptAsync(IJobRunContext jobContext, System.Threading.CancellationToken cancellationToken); System.Threading.Tasks.Task TryScheduleJobAsync(ScheduleJobRequest request, System.Threading.CancellationToken cancellationToken); + System.Threading.Tasks.Task TryStartAttemptAsync(IJobRunContext jobContext, System.Threading.CancellationToken cancellationToken); } public partial interface ILocalDurableJobManager @@ -179,9 +178,9 @@ protected void EnqueueJob(DurableJob job, int dequeueCount) { } public System.Threading.Tasks.Task RetryJobLaterAsync(IJobRunContext jobContext, System.DateTimeOffset newDueTime, System.Threading.CancellationToken cancellationToken) { throw null; } - public System.Threading.Tasks.Task TryStartAttemptAsync(IJobRunContext jobContext, System.Threading.CancellationToken cancellationToken) { throw null; } - public System.Threading.Tasks.Task TryScheduleJobAsync(ScheduleJobRequest request, System.Threading.CancellationToken cancellationToken) { throw null; } + + public System.Threading.Tasks.Task TryStartAttemptAsync(IJobRunContext jobContext, System.Threading.CancellationToken cancellationToken) { throw null; } } public abstract partial class JobShardManager @@ -240,6 +239,8 @@ public sealed partial class DurableJobsOptions public System.TimeSpan ShardBatchLingerDelay { get { throw null; } set { } } + public System.TimeSpan ShardCheckInterval { get { throw null; } set { } } + public int ShardClaimInitialBudget { get { throw null; } set { } } public int ShardClaimMaxBudget { get { throw null; } set { } } @@ -248,6 +249,8 @@ public sealed partial class DurableJobsOptions public System.TimeSpan ShardDuration { get { throw null; } set { } } + public System.TimeSpan ShardLoadLookaheadPeriod { get { throw null; } set { } } + public int ShardStripeCount { get { throw null; } set { } } public System.Func ShouldRetry { get { throw null; } set { } } diff --git a/test/Orleans.Core.Tests/DurableJobs/DurableJobsOptionsTests.cs b/test/Orleans.Core.Tests/DurableJobs/DurableJobsOptionsTests.cs new file mode 100644 index 00000000000..ec951d0f130 --- /dev/null +++ b/test/Orleans.Core.Tests/DurableJobs/DurableJobsOptionsTests.cs @@ -0,0 +1,68 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Orleans.Hosting; +using Xunit; + +namespace NonSilo.Tests.DurableJobs; + +[TestCategory("BVT"), TestCategory("DurableJobs")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("DurableJobs")] +public class DurableJobsOptionsTests +{ + [Fact] + public void DiscoveryTimingDefaults() + { + var options = new DurableJobsOptions(); + + Assert.Equal(TimeSpan.FromMinutes(10), options.ShardLoadLookaheadPeriod); + Assert.Equal(TimeSpan.FromMinutes(5), options.ShardCheckInterval); + CreateValidator(options).ValidateConfiguration(); + } + + [Fact] + public void ValidateConfiguration_NegativeLookahead_Throws() + { + var options = new DurableJobsOptions { ShardLoadLookaheadPeriod = TimeSpan.FromTicks(-1) }; + + var exception = Assert.Throws(CreateValidator(options).ValidateConfiguration); + + Assert.Contains(nameof(DurableJobsOptions.ShardLoadLookaheadPeriod), exception.Message); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + [InlineData(0.9999)] + [InlineData(4294967295)] + public void ValidateConfiguration_InvalidCheckInterval_Throws(double milliseconds) + { + var options = new DurableJobsOptions { ShardCheckInterval = TimeSpan.FromMilliseconds(milliseconds) }; + + var exception = Assert.Throws(CreateValidator(options).ValidateConfiguration); + + Assert.Contains(nameof(DurableJobsOptions.ShardCheckInterval), exception.Message); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(23, 120000)] + [InlineData(10, 4294967294)] + public void ValidateConfiguration_ValidDiscoveryTimings_AcceptsValues(int lookaheadMinutes, double intervalMilliseconds) + { + var options = new DurableJobsOptions + { + ShardLoadLookaheadPeriod = TimeSpan.FromMinutes(lookaheadMinutes), + ShardCheckInterval = TimeSpan.FromMilliseconds(intervalMilliseconds) + }; + + CreateValidator(options).ValidateConfiguration(); + using var timer = new PeriodicTimer(options.ShardCheckInterval); + Assert.Equal(options.ShardCheckInterval, timer.Period); + Assert.Equal(TimeSpan.FromMinutes(lookaheadMinutes), options.ShardLoadLookaheadPeriod); + } + + private static DurableJobsOptionsValidator CreateValidator(DurableJobsOptions options) => + new(NullLogger.Instance, Options.Create(options)); +} diff --git a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs index f74265eaf4d..3766e7f2875 100644 --- a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs @@ -180,10 +180,366 @@ async IAsyncEnumerable ConsumeAsync([EnumeratorCancellation] Can } } + [Fact] + public async Task PeriodicDiscovery_ActivatesFirstYieldBeforeSweepCompletesAndDisposesOnStop() + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var shard = new BlockingQueueShard("first-yield", timeProvider.GetUtcNow(), timeProvider.GetUtcNow().AddHours(1)); + shard.AllowDispose.SetResult(); + var tailStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var discoveryCanceled = false; + var disposalCount = 0; + var calls = 0; + shardManager.DiscoverShards = (_, _, token) => DiscoverShards(token); + + async IAsyncEnumerable DiscoverShards([EnumeratorCancellation] CancellationToken token) + { + Interlocked.Increment(ref calls); + try + { + yield return shard; + tailStarted.SetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + } + finally + { + discoveryCanceled = token.IsCancellationRequested; + Interlocked.Increment(ref disposalCount); + } + } + + var manager = CreateManager(shardManager, timeProvider, CreateOptions()); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); + manager.Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + try + { + accessor.SignalShardCheck(); + await shard.ConsumeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await tailStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Volatile.Read(ref calls)); + Assert.Equal(0, Volatile.Read(ref disposalCount)); + Assert.True(accessor.TryGetRunningShardTask(shard.Id, out var running)); + Assert.NotNull(running); + Assert.False(running.IsCompleted); + } + finally + { + await lifecycle.OnStop(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.True(discoveryCanceled); + Assert.Equal(1, Volatile.Read(ref disposalCount)); + Assert.Equal(1, shard.DisposeCallCount); + Assert.Equal(1, Volatile.Read(ref calls)); + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + } + [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Stop_DuringAssignmentReadinessCheckRejectsActivationAndAwaitsCleanup(bool useSiloLifecycle) + [InlineData(null)] + [InlineData(2)] + public async Task PeriodicDiscovery_RetainsMembershipSignalDuringSweepAndRestartsOnTimer(int? checkIntervalMinutes) + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var firstSweep = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var finishSweep = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var membershipSweep = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var periodicSweep = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var options = CreateOptions(); + if (checkIntervalMinutes is { } minutes) + { + options.ShardCheckInterval = TimeSpan.FromMinutes(minutes); + } + else + { + Assert.Equal(TimeSpan.FromMinutes(5), options.ShardCheckInterval); + } + + var calls = 0; + var disposedSweeps = 0; + shardManager.DiscoverShards = (_, _, token) => DiscoverShards(token); + + async IAsyncEnumerable DiscoverShards([EnumeratorCancellation] CancellationToken token) + { + try + { + switch (Interlocked.Increment(ref calls)) + { + case 1: + firstSweep.SetResult(); + await finishSweep.Task.WaitAsync(token); + break; + case 2: + membershipSweep.SetResult(); + break; + case 3: + periodicSweep.SetResult(); + break; + } + + yield break; + } + finally + { + Interlocked.Increment(ref disposedSweeps); + } + } + + var manager = CreateManager(shardManager, timeProvider, options); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); + manager.Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + try + { + accessor.SignalShardCheck(); + await firstSweep.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Volatile.Read(ref calls)); + Assert.Equal(0, Volatile.Read(ref disposedSweeps)); + accessor.SignalShardCheck(); + finishSweep.SetResult(); + await membershipSweep.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await manager.QueueTask(() => Task.CompletedTask).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(2, Volatile.Read(ref calls)); + Assert.Equal(2, Volatile.Read(ref disposedSweeps)); + + timeProvider.Advance(options.ShardCheckInterval - TimeSpan.FromTicks(1)); + await manager.QueueTask(() => Task.CompletedTask).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(periodicSweep.Task.IsCompleted); + Assert.Equal(2, Volatile.Read(ref calls)); + + timeProvider.Advance(TimeSpan.FromTicks(1)); + await periodicSweep.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(3, Volatile.Read(ref calls)); + } + finally + { + await lifecycle.OnStop(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + [Theory] + [InlineData(null)] + [InlineData(0)] + [InlineData(23)] + public async Task Discovery_UsesConfiguredLookahead(int? lookaheadMinutes) + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var options = CreateOptions(); + if (lookaheadMinutes is { } minutes) + { + options.ShardLoadLookaheadPeriod = TimeSpan.FromMinutes(minutes); + } + + var manager = CreateManager(shardManager, timeProvider, options); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var expectedLookahead = TimeSpan.FromMinutes(lookaheadMinutes ?? 10); + + await accessor.ProcessShardCheckCycleAsync(TestContext.Current.CancellationToken); + Assert.Equal(timeProvider.GetUtcNow().Add(expectedLookahead), shardManager.LastMaxDueTime); + + timeProvider.Advance(TimeSpan.FromMinutes(1)); + await accessor.ProcessShardCheckCycleAsync(TestContext.Current.CancellationToken); + Assert.Equal(timeProvider.GetUtcNow().Add(expectedLookahead), shardManager.LastMaxDueTime); + } + + [Fact] + public async Task Discovery_MaximumLookaheadIncludesAllRepresentableDates() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var options = CreateOptions(); + options.ShardLoadLookaheadPeriod = TimeSpan.MaxValue; + new DurableJobsOptionsValidator(NullLogger.Instance, Options.Create(options)).ValidateConfiguration(); + var manager = CreateManager(shardManager, timeProvider, options); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + + await accessor.ProcessShardCheckCycleAsync(TestContext.Current.CancellationToken); + + Assert.Equal(DateTimeOffset.MaxValue, shardManager.LastMaxDueTime); + Assert.Equal(TimeSpan.Zero, shardManager.LastMaxDueTime.Offset); + Assert.Equal(TimeSpan.MaxValue, options.ShardLoadLookaheadPeriod); + } + + [Theory] + [InlineData(2, 0, 2)] + [InlineData(2, 1, 1)] + [InlineData(2, 2, 0)] + [InlineData(2, 3, 0)] + [InlineData(2, long.MaxValue, 0)] + [InlineData(0, 0, 0)] + [InlineData(0, 1, 0)] + public async Task Discovery_NearMaximumTimeClampsLookahead(long remainingTicks, long lookaheadTicks, long expectedRemainingTicks) + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.MaxValue.AddTicks(-remainingTicks)); + var shardManager = new TestJobShardManager(); + var options = CreateOptions(); + options.ShardLoadLookaheadPeriod = TimeSpan.FromTicks(lookaheadTicks); + var manager = CreateManager(shardManager, timeProvider, options); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + + await accessor.ProcessShardCheckCycleAsync(TestContext.Current.CancellationToken); + + Assert.Equal(DateTimeOffset.MaxValue.AddTicks(-expectedRemainingTicks), shardManager.LastMaxDueTime); + Assert.Equal(TimeSpan.Zero, shardManager.LastMaxDueTime.Offset); + Assert.Equal(TimeSpan.FromTicks(lookaheadTicks), options.ShardLoadLookaheadPeriod); + } + + [Fact] + public async Task Discovery_RevisitedFutureShardActivatesAfterClockAdvances() + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var start = timeProvider.GetUtcNow().AddMinutes(10); + var shard = new CompletingShard("future", start, start.AddHours(1)); + shardManager.AssignedShards.Add(shard); + var manager = CreateManager(shardManager, timeProvider, CreateOptions()); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + + await accessor.ProcessShardCheckCycleAsync(cancellationToken); + Assert.True(accessor.HasCachedShard(shard.Id)); + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + Assert.Equal(timeProvider.GetUtcNow().AddMinutes(10), shardManager.LastMaxDueTime); + + timeProvider.Advance(TimeSpan.FromMinutes(10)); + await accessor.ProcessShardCheckCycleAsync(cancellationToken); + await shard.ConsumeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(accessor.TryGetRunningShardTask(shard.Id, out var running)); + Assert.Equal(timeProvider.GetUtcNow().AddMinutes(10), shardManager.LastMaxDueTime); + await shard.MarkAsCompleteAsync(cancellationToken); + await running!.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Contains(shard, shardManager.UnregisteredShards); + } + + [Theory] + [InlineData(0)] + [InlineData(10)] + public async Task Discovery_CancellationAfterYieldTracksShardUntilShutdownCompletes(int startDelayMinutes) + { + var cancellationToken = TestContext.Current.CancellationToken; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var start = timeProvider.GetUtcNow().AddMinutes(startDelayMinutes); + var shard = new BlockingQueueShard("canceled-yield", start, start.AddHours(1)); + var discoveryDisposed = false; + var shardManager = new TestJobShardManager + { + DiscoverShards = (_, _, _) => DiscoverShards() + }; + + async IAsyncEnumerable DiscoverShards() + { + await Task.CompletedTask; + try + { + cancellation.Cancel(); + yield return shard; + } + finally + { + discoveryDisposed = true; + } + } + + var manager = CreateManager(shardManager, timeProvider, CreateOptions()); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); + manager.Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + Task? stop = null; + try + { + var exception = await Assert.ThrowsAnyAsync( + () => accessor.ProcessShardCheckCycleAsync(cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.True(discoveryDisposed); + Assert.True(accessor.HasCachedShard(shard.Id)); + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + Assert.False(shard.ConsumeStarted.Task.IsCompleted); + Assert.False(shard.DisposeStarted.Task.IsCompleted); + + stop = lifecycle.OnStop(cancellationToken); + await shard.DisposeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(stop.IsCompleted); + } + finally + { + shard.AllowDispose.TrySetResult(); + await (stop ?? lifecycle.OnStop(cancellationToken)).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.Equal(1, shard.DisposeCallCount); + Assert.Empty(shardManager.UnregisteredShards); + Assert.False(accessor.HasCachedShard(shard.Id)); + Assert.False(shard.ConsumeStarted.Task.IsCompleted); + } + + [Fact] + public async Task Discovery_CanceledRepeatedYieldKeepsRunningShardUntilShutdown() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var start = timeProvider.GetUtcNow(); + var shard = new BlockingQueueShard("running-yield", start, start.AddHours(1)); + var shardManager = new TestJobShardManager { DiscoverShards = (_, _, _) => DiscoverShards() }; + + async IAsyncEnumerable DiscoverShards() + { + await Task.CompletedTask; + cancellation.Cancel(); + yield return shard; + } + + var manager = CreateManager(shardManager, timeProvider, CreateOptions()); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); + manager.Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + Task? stop = null; + try + { + accessor.AddWritableShard(start, shard); + accessor.TryActivateShard(shard); + await shard.ConsumeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(accessor.TryGetRunningShardTask(shard.Id, out var running)); + + await Assert.ThrowsAnyAsync( + () => accessor.ProcessShardCheckCycleAsync(cancellation.Token)); + Assert.True(accessor.HasCachedShard(shard.Id)); + Assert.True(accessor.TryGetRunningShardTask(shard.Id, out var afterDiscovery)); + Assert.Same(running, afterDiscovery); + Assert.False(shard.DisposeStarted.Task.IsCompleted); + + stop = lifecycle.OnStop(cancellationToken); + await shard.DisposeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(stop.IsCompleted); + } + finally + { + shard.AllowDispose.TrySetResult(); + await (stop ?? lifecycle.OnStop(cancellationToken)).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.Equal(1, shard.DisposeCallCount); + Assert.False(accessor.HasCachedShard(shard.Id)); + Assert.False(accessor.HasWritableShard(start)); + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Stop_DuringDiscoveryReadinessCheckRejectsActivationAndAwaitsCleanup(bool failDiscoveryDisposal, bool useSiloLifecycle) { var cancellationToken = TestContext.Current.CancellationToken; var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); @@ -209,7 +565,7 @@ public async Task Stop_DuringAssignmentReadinessCheckRejectsActivationAndAwaitsC discoveredShard.Id.Returns(shard.Id); discoveredShard.StartTime.Returns(_ => { - // The assigned shard is cached and checking readiness when shutdown closes admission. + // The yielded shard has passed its cancellation check, but has not been activated yet. stop ??= observer.OnStop(cancellationToken); // OnStop returns its task so this check can finish while shutdown awaits the loop. return shard.StartTime; @@ -217,29 +573,68 @@ public async Task Stop_DuringAssignmentReadinessCheckRejectsActivationAndAwaitsC discoveredShard.EndTime.Returns(shard.EndTime); discoveredShard.ConsumeDurableJobsAsync().Returns(_ => shard.ConsumeDurableJobsAsync()); discoveredShard.DisposeAsync().Returns(_ => shard.DisposeAsync()); - shardManager.AssignedShards.Add(discoveredShard); + var undeliveredShard = CreateSubstituteShard("canceled-after-stop", shard.StartTime, shard.EndTime); + var discoveryDisposed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var disposalCount = 0; + shardManager.DiscoverShards = (_, _, _) => DiscoverShards(); + + async IAsyncEnumerable DiscoverShards() + { + await Task.CompletedTask; + try + { + yield return discoveredShard; + // Cancellation at delivery leaves the iterator suspended, forcing cleanup through DisposeAsync. + yield return undeliveredShard; + } + finally + { + Interlocked.Increment(ref disposalCount); + // Let the readiness check finish before observing the closed activation gate. + discoveryDisposed.SetResult(manager.QueueTask(() => + { + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + Assert.False(shard.ConsumeStarted.Task.IsCompleted); + Assert.NotNull(stop); + Assert.False(stop.IsCompleted); + return Task.CompletedTask; + })); + if (failDiscoveryDisposal) + { + throw new InvalidOperationException("Discovery disposal failed"); + } + } + } await observer.OnStart(cancellationToken); try { accessor.SignalShardCheck(); await shard.DisposeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - await manager.QueueTask(() => Task.CompletedTask).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.NotNull(stop); - Assert.False(stop.IsCompleted); - Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); - Assert.False(shard.ConsumeStarted.Task.IsCompleted); + var shutdownAssertion = await discoveryDisposed.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await shutdownAssertion.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); Assert.Equal(0, shard.DisposeCallCount); + Assert.True(accessor.HasCachedShard(undeliveredShard.Id)); + Assert.False(accessor.TryGetRunningShardTask(undeliveredShard.Id, out _)); } finally { shard.AllowDispose.TrySetResult(); - await (stop ?? observer.OnStop(cancellationToken)).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + if (stop is not null) + { + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + Assert.Equal(1, Volatile.Read(ref disposalCount)); Assert.Equal(1, shard.DisposeCallCount); - Assert.Same(discoveredShard, Assert.Single(shardManager.UnregisteredShards)); + Assert.Equal(2, shardManager.UnregisteredShards.Count); + Assert.Contains(discoveredShard, shardManager.UnregisteredShards); + Assert.Contains(undeliveredShard, shardManager.UnregisteredShards); Assert.False(accessor.HasCachedShard(shard.Id)); + Assert.False(accessor.HasCachedShard(undeliveredShard.Id)); + await undeliveredShard.Received(1).DisposeAsync(); Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); Assert.DoesNotContain(lifecycleLogger.Entries, entry => entry.Level == LogLevel.Error); } @@ -465,6 +860,59 @@ public async Task TryActivateShard_WhenAnotherActivationWinsReadinessRace_RunsSh AssertSchedulingCacheEmpty(accessor); } + [Fact] + public async Task Discovery_WhenLaterCandidateFails_LeavesYieldedPrefixRunning() + { + var cancellationToken = TestContext.Current.CancellationToken; + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var shardManager = new TestJobShardManager(); + var shard = new BlockingQueueShard("successful-prefix", timeProvider.GetUtcNow(), timeProvider.GetUtcNow().AddHours(1)); + shard.AllowDispose.SetResult(); + var failure = new InvalidOperationException("Later candidate failed"); + var disposalCount = 0; + shardManager.DiscoverShards = (_, _, token) => DiscoverShards(token); + + async IAsyncEnumerable DiscoverShards([EnumeratorCancellation] CancellationToken token) + { + try + { + yield return shard; + await shard.ConsumeStarted.Task.WaitAsync(token); + throw failure; + } + finally + { + Interlocked.Increment(ref disposalCount); + } + } + + var manager = CreateManager(shardManager, timeProvider, CreateOptions()); + var accessor = new LocalDurableJobManager.TestAccessor(manager); + var lifecycle = new SiloLifecycleSubject(NullLogger.Instance); + manager.Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + try + { + var exception = await Assert.ThrowsAsync( + () => accessor.ProcessShardCheckCycleAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Same(failure, exception); + Assert.Equal(1, Volatile.Read(ref disposalCount)); + Assert.True(accessor.HasCachedShard(shard.Id)); + Assert.True(accessor.TryGetRunningShardTask(shard.Id, out var running)); + Assert.NotNull(running); + Assert.False(running.IsCompleted); + Assert.Equal(0, shard.DisposeCallCount); + Assert.Empty(shardManager.UnregisteredShards); + } + finally + { + await lifecycle.OnStop(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.Equal(1, shard.DisposeCallCount); + Assert.False(accessor.TryGetRunningShardTask(shard.Id, out _)); + } + [Fact] public async Task Stop_WhenActiveShardWaitsForQueueChange_CompletesAfterCleanupWithoutLifecycleError() { @@ -550,7 +998,7 @@ public async Task ProcessShardCheckCycleAsync_MarksExpiredWritableShardComplete( Assert.False(accessor.HasWritableShard(shardKey)); await shard.Received(1).MarkAsCompleteAsync(Arg.Any()); - Assert.Equal(timeProvider.GetUtcNow().AddHours(1), shardManager.LastMaxDueTime); + Assert.Equal(timeProvider.GetUtcNow().AddMinutes(10), shardManager.LastMaxDueTime); } [Fact] @@ -2697,6 +3145,13 @@ private static async IAsyncEnumerable ConsumeAsync() private sealed class TestJobShardManager() : JobShardManager(SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5000), 0)) { + public Func>? DiscoverShards { get; set; } + + internal override IAsyncEnumerable DiscoverJobShardsAsync(DateTimeOffset maxDueTime, int maxNewClaims, CancellationToken cancellationToken) + => DiscoverShards is { } discover + ? discover(maxDueTime, maxNewClaims, cancellationToken) + : base.DiscoverJobShardsAsync(maxDueTime, maxNewClaims, cancellationToken); + public List AssignedShards { get; } = []; public List UnregisteredShards { get; } = []; diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs new file mode 100644 index 00000000000..1ebe58acf7c --- /dev/null +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -0,0 +1,538 @@ +using System.Diagnostics; +using System.Net; +using Orleans.DurableJobs; +using Orleans.Journaling; +using Orleans.Runtime; +using Xunit; + +namespace Tester.DurableJobs; + +public partial class JournaledJobShardManagerTests +{ + [Fact] + public async Task Discovery_UsesProjectedMetadataAndReadsOnlyEntriesWithoutMetadata() + { + await using var fixture = new DiscoveryFixture(); + var projected = await fixture.AddShardAsync("projected", fixture.Now.AddYears(-1)); + var missing = await fixture.AddShardAsync("missing", fixture.Now); + var metadata = await fixture.Storage.CreateStorage(projected).GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(metadata); + fixture.Catalog.Metadata.Add(projected, metadata); + fixture.Catalog.Ids.AddRange([missing, projected, projected]); + fixture.Storage.MetadataReads.Clear(); + + AssertAssignedIds([projected, missing], await fixture.DiscoverAsync(maxNewClaims: 2)); + Assert.Equal(new[] { missing }, fixture.Storage.MetadataReads); + Assert.Equal(3, fixture.Catalog.YieldedIds); + Assert.Equal(1, fixture.Catalog.ListCalls); + } + + [Fact] + public async Task Discovery_ProjectedMetadataWithoutETagReadsCurrentMetadataForConditionalClaim() + { + await using var fixture = new DiscoveryFixture(); + var id = await fixture.AddShardAsync("no-projected-etag", fixture.Now); + var storage = fixture.Storage.CreateStorage(id); + var snapshot = await storage.GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(snapshot); + Assert.NotNull(snapshot.ETag); + fixture.Catalog.Metadata.Add(id, new JournalMetadata(snapshot.Format, properties: snapshot.Properties)); + fixture.Catalog.Ids.Add(id); + fixture.Storage.MetadataReads.Clear(); + + AssertAssignedIds([id], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { id }, fixture.Storage.MetadataReads); + Assert.Equal((id, snapshot.ETag), Assert.Single(fixture.Storage.MetadataUpdates)); + var claimed = await storage.GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(claimed); + Assert.Equal(fixture.Silo.ToParsableString(), claimed.Properties["DurableJobsOwner"]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Discovery_MetadataWithoutETagRejectsUnconditionalClaim(bool includeProjectedMetadata) + { + await using var fixture = new DiscoveryFixture(); + var id = await fixture.AddShardAsync("no-storage-etag", fixture.Now); + var storage = fixture.Storage.CreateStorage(id); + var original = await storage.GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(original); + if (includeProjectedMetadata) + { + fixture.Catalog.Metadata.Add(id, new JournalMetadata(original.Format, properties: original.Properties)); + } + + fixture.Catalog.Ids.Add(id); + fixture.Storage.OmitMetadataETags = true; + fixture.Storage.MetadataReads.Clear(); + + var exception = await Assert.ThrowsAsync(() => fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Contains("requires a storage metadata ETag", exception.Message); + Assert.Equal(new[] { id }, fixture.Storage.MetadataReads); + Assert.Empty(fixture.Storage.MetadataUpdates); + fixture.Storage.OmitMetadataETags = false; + var current = await storage.GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(current); + Assert.Equal(original.ETag, current.ETag); + Assert.False(current.Properties.ContainsKey("DurableJobsOwner")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Discovery_StaleProjectedMetadataCannotOverwriteConcurrentOwnership(bool includeETag) + { + await using var fixture = new DiscoveryFixture(); + var id = await fixture.AddShardAsync("stale", fixture.Now); + var storage = fixture.Storage.CreateStorage(id); + var cancellationToken = TestContext.Current.CancellationToken; + var snapshot = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(snapshot); + fixture.Catalog.Metadata.Add(id, includeETag ? snapshot : new JournalMetadata(snapshot.Format, properties: snapshot.Properties)); + fixture.Catalog.Ids.Add(id); + + var other = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5101), 0); + fixture.Membership.SetSiloStatus(other, SiloStatus.Active); + var current = await storage.UpdateMetadataAsync( + new Dictionary { ["DurableJobsOwner"] = other.ToParsableString() }, + expectedETag: snapshot.ETag, cancellationToken: cancellationToken); + Assert.NotNull(current); + fixture.Storage.MetadataReads.Clear(); + fixture.Storage.MetadataUpdates.Clear(); + + Assert.Empty(await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(includeETag ? Array.Empty() : [id], fixture.Storage.MetadataReads); + if (includeETag) + { + Assert.Equal((id, snapshot.ETag), Assert.Single(fixture.Storage.MetadataUpdates)); + } + else + { + Assert.Empty(fixture.Storage.MetadataUpdates); + } + + var after = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(after); + Assert.Equal(current.ETag, after.ETag); + Assert.Equal(other.ToParsableString(), after.Properties["DurableJobsOwner"]); + } + + [Fact] + public async Task Discovery_UnorderedCatalogClaimsOldestFirstIncludingYearsOverdue() + { + await using var fixture = new DiscoveryFixture(); + var oldest = await fixture.AddShardAsync("oldest", fixture.Now.AddYears(-8)); + var middle = await fixture.AddShardAsync("middle", fixture.Now.AddDays(-1)); + var youngest = await fixture.AddShardAsync("youngest", fixture.Now); + fixture.Catalog.Ids.AddRange([youngest, oldest, middle]); + + AssertAssignedIds([oldest, middle], await fixture.DiscoverAsync(maxNewClaims: 2)); + Assert.Equal(new[] { oldest, middle, youngest }, fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.ListCalls); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + + AssertAssignedIds([oldest, middle, youngest], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { oldest, middle, youngest, oldest, middle, youngest }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Theory] + [InlineData(0)] + [InlineData(5)] + [InlineData(-7)] + public async Task Discovery_InclusiveUtcTickBoundExcludesFutureMetadata(int offsetHours) + { + await using var fixture = new DiscoveryFixture(); + var horizon = fixture.Horizon.AddTicks(1234567).ToOffset(TimeSpan.FromHours(offsetHours)); + var previous = await fixture.AddShardAsync("previous", horizon.AddTicks(-1)); + var boundaryA = await fixture.AddShardAsync("boundary-a", horizon); + var boundaryZ = await fixture.AddShardAsync("boundary-z", horizon.ToUniversalTime()); + var nextTick = await fixture.AddShardAsync("next-tick", horizon.AddTicks(1)); + var farFuture = await fixture.AddShardAsync("far-future", horizon.AddYears(10)); + fixture.Catalog.Ids.AddRange([farFuture, nextTick, boundaryZ, previous, boundaryA]); + fixture.Storage.BeforeMetadataRead = (id, _) => + { + Assert.NotEqual(nextTick, id); + Assert.NotEqual(farFuture, id); + return ValueTask.CompletedTask; + }; + + AssertAssignedIds([previous, boundaryA, boundaryZ], await fixture.DiscoverAsync(horizon: horizon)); + Assert.Equal(new[] { previous, boundaryA, boundaryZ }, fixture.Storage.MetadataReads); + Assert.Equal(3, fixture.Catalog.YieldedIds); + Assert.Equal(4, fixture.Catalog.MoveNextCalls); + var request = Assert.Single(fixture.Catalog.Requests); + Assert.Equal(JobShardId.StoragePrefix.Value + "/", request.Prefix.Value); + Assert.Equal(JobShardId.GetMaxJournalId(horizon.ToUniversalTime()), request.MaxId); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_EmptyAndFreshSweepsObserveEarlierInsertionsAndLaterHorizon() + { + await using var fixture = new DiscoveryFixture(); + Assert.Empty(await fixture.DiscoverAsync()); + Assert.Empty(fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.MoveNextCalls); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + + var current = await fixture.AddShardAsync("current", fixture.Now); + var future = await fixture.AddShardAsync("future", fixture.Horizon.AddTicks(1)); + fixture.Catalog.Ids.AddRange([future, current]); + AssertAssignedIds([current], await fixture.DiscoverAsync()); + + var inserted = await fixture.AddShardAsync("inserted", fixture.Now.AddYears(-3)); + fixture.Catalog.Ids.Add(inserted); + AssertAssignedIds([inserted, current], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { current, inserted, current }, fixture.Storage.MetadataReads); + + var laterHorizon = fixture.Horizon.AddTicks(1); + AssertAssignedIds([inserted, current, future], await fixture.DiscoverAsync(horizon: laterHorizon)); + Assert.Equal(new[] { current, inserted, current, inserted, current, future }, fixture.Storage.MetadataReads); + Assert.Equal(4, fixture.Catalog.ListCalls); + Assert.Equal(4, fixture.Catalog.DisposeCalls); + Assert.Equal( + new[] + { + JobShardId.GetMaxJournalId(fixture.Horizon), + JobShardId.GetMaxJournalId(fixture.Horizon), + JobShardId.GetMaxJournalId(fixture.Horizon), + JobShardId.GetMaxJournalId(laterHorizon) + }, + fixture.Catalog.Requests.Select(request => request.MaxId)); + } + + [Fact] + public async Task Discovery_ClaimBudgetCountsOnlyClaimsAndStillReturnsLocalShards() + { + await using var fixture = new DiscoveryFixture(); + var firstLocal = await fixture.AddShardAsync("first-local", fixture.Now.AddHours(-3), fixture.Silo); + var olderOrphan = await fixture.AddShardAsync("older-orphan", fixture.Now.AddHours(-2)); + var youngerOrphan = await fixture.AddShardAsync("younger-orphan", fixture.Now); + var lastLocal = await fixture.AddShardAsync("last-local", fixture.Now.AddMinutes(30), fixture.Silo); + fixture.Catalog.Ids.AddRange([lastLocal, youngerOrphan, olderOrphan, firstLocal]); + + AssertAssignedIds([firstLocal, lastLocal], await fixture.DiscoverAsync(maxNewClaims: 0)); + Assert.Equal(new[] { firstLocal, olderOrphan, youngerOrphan, lastLocal }, fixture.Storage.MetadataReads); + + AssertAssignedIds([firstLocal, olderOrphan, lastLocal], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal( + new[] { firstLocal, olderOrphan, youngerOrphan, lastLocal, firstLocal, olderOrphan, youngerOrphan, lastLocal }, + fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + var unclaimed = await fixture.Storage.CreateStorage(youngerOrphan).GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(unclaimed); + Assert.False(unclaimed.Properties.ContainsKey("DurableJobsOwner")); + } + + [Fact] + public async Task Discovery_MetadataFailurePreservesYieldedAssignmentAndRetriesFresh() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-2)); + var failing = await fixture.AddShardAsync("failing", fixture.Now.AddMinutes(-1)); + var tail = await fixture.AddShardAsync("tail", fixture.Now); + fixture.Catalog.Ids.AddRange([tail, failing, first]); + var failure = new InvalidOperationException("Metadata unavailable"); + fixture.Storage.BeforeMetadataRead = (id, _) => id == failing ? throw failure : ValueTask.CompletedTask; + + IJobShard retained; + await using (var discovery = fixture.DiscoverStreamAsync(TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await discovery.MoveNextAsync()); + retained = discovery.Current; + AssertAssignedIds([first], [retained]); + Assert.Equal(new[] { first }, fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + Assert.Same(failure, await Assert.ThrowsAsync(() => discovery.MoveNextAsync().AsTask())); + } + + Assert.Equal(new[] { first, failing }, fixture.Storage.MetadataReads); + Assert.Equal(0, await retained.GetJobCountAsync()); + fixture.Storage.BeforeMetadataRead = null; + var retry = await fixture.DiscoverAsync(maxNewClaims: 2); + AssertAssignedIds([first, failing, tail], retry); + Assert.Same(retained, retry[0]); + Assert.Equal(new[] { first, failing, first, failing, tail }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_CancellationDuringListingDisposesEnumeratorAndRetriesFresh() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now); + fixture.Catalog.Ids.Add(first); + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + await Assert.ThrowsAnyAsync(() => fixture.DiscoverWithCancellationAsync(canceled.Token)); + Assert.Equal(0, fixture.Catalog.ListCalls); + + using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var moveNextStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Catalog.BeforeMoveNext = async (_, token) => + { + Assert.Equal(lifetime.Token, token); + moveNextStarted.SetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + }; + var discovery = fixture.DiscoverWithCancellationAsync(lifetime.Token); + try + { + await moveNextStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + finally + { + lifetime.Cancel(); + await Assert.ThrowsAnyAsync(() => discovery); + } + + Assert.Empty(fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + fixture.Catalog.BeforeMoveNext = null; + AssertAssignedIds([first], await fixture.DiscoverAsync()); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_CancellationBetweenAssignmentsStopsBeforeNextMetadataRead() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-1)); + var next = await fixture.AddShardAsync("next", fixture.Now); + fixture.Catalog.Ids.AddRange([next, first]); + using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + await using (var discovery = fixture.DiscoverStreamAsync(lifetime.Token).GetAsyncEnumerator(lifetime.Token)) + { + Assert.True(await discovery.MoveNextAsync()); + AssertAssignedIds([first], [discovery.Current]); + lifetime.Cancel(); + await Assert.ThrowsAnyAsync(() => discovery.MoveNextAsync().AsTask()); + } + + Assert.Equal(new[] { first }, fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + AssertAssignedIds([first, next], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { first, first, next }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_StoppingConsumerDoesNotRetainEnumerationOrTraversalPosition() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-1)); + var tail = await fixture.AddShardAsync("tail", fixture.Now); + fixture.Catalog.Ids.AddRange([tail, first]); + await using (var discovery = fixture.DiscoverStreamAsync(TestContext.Current.CancellationToken).GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + Assert.True(await discovery.MoveNextAsync()); + AssertAssignedIds([first], [discovery.Current]); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + } + + Assert.Equal(new[] { first }, fixture.Storage.MetadataReads); + var inserted = await fixture.AddShardAsync("inserted", fixture.Now.AddYears(-1)); + fixture.Catalog.Ids.Add(inserted); + AssertAssignedIds([inserted, first], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { first, inserted, first, tail }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_AwaitsCatalogDisposalBeforeReadingMetadata() + { + await using var fixture = new DiscoveryFixture(); + var due = await fixture.AddShardAsync("due", fixture.Now); + fixture.Catalog.Ids.Add(due); + var disposalStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowDisposal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.Catalog.OnDispose = async () => + { + disposalStarted.SetResult(); + await allowDisposal.Task.WaitAsync(TestContext.Current.CancellationToken); + }; + + var discovery = fixture.DiscoverAsync(); + try + { + await disposalStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(discovery.IsCompleted); + Assert.Empty(fixture.Storage.MetadataReads); + } + finally + { + allowDisposal.TrySetResult(); + await discovery; + } + + AssertAssignedIds([due], await discovery); + Assert.Equal(new[] { due }, fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_ListingFailureSurfacesBeforeMetadataAndRetriesFresh() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-1)); + var tail = await fixture.AddShardAsync("tail", fixture.Now); + fixture.Catalog.Ids.AddRange([first, tail]); + var failure = new InvalidOperationException("Listing unavailable"); + fixture.Catalog.BeforeMoveNext = (index, _) => index == 1 ? throw failure : ValueTask.CompletedTask; + + Assert.Same(failure, await Assert.ThrowsAsync(() => fixture.DiscoverAsync())); + Assert.Empty(fixture.Storage.MetadataReads); + Assert.Equal(1, fixture.Catalog.YieldedIds); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + + var inserted = await fixture.AddShardAsync("inserted", fixture.Now.AddYears(-1)); + fixture.Catalog.Ids.Add(inserted); + fixture.Catalog.BeforeMoveNext = null; + AssertAssignedIds([inserted, first, tail], await fixture.DiscoverAsync()); + Assert.Equal(new[] { inserted, first, tail }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_MembershipChangesUseCurrentStatusForEachCandidate() + { + await using var fixture = new DiscoveryFixture(); + var owner = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5101), 0); + fixture.Membership.SetSiloStatus(owner, SiloStatus.Active); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-1), owner); + var second = await fixture.AddShardAsync("second", fixture.Now, owner); + fixture.Catalog.Ids.AddRange([second, first]); + fixture.Storage.BeforeMetadataRead = (id, _) => + { + fixture.Membership.SetSiloStatus(owner, id == first ? SiloStatus.Dead : SiloStatus.Active); + return ValueTask.CompletedTask; + }; + + AssertAssignedIds([first], await fixture.DiscoverAsync()); + Assert.Equal(new[] { first, second }, fixture.Storage.MetadataReads); + + fixture.Storage.BeforeMetadataRead = null; + fixture.Membership.SetSiloStatus(owner, SiloStatus.Dead); + AssertAssignedIds([first, second], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { first, second, first, second }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + var metadata = await fixture.Storage.CreateStorage(second).GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.Equal(fixture.Silo.ToParsableString(), metadata!.Properties["DurableJobsOwner"]); + Assert.Equal("1", metadata.Properties["DurableJobsAdoptedCount"]); + } + + [Fact] + public async Task Discovery_DuplicateIdentitiesReadMetadataAndAssignOnlyOnce() + { + await using var fixture = new DiscoveryFixture(); + var due = await fixture.AddShardAsync("due", fixture.Now); + fixture.Catalog.Ids.AddRange([due, due, due]); + + AssertAssignedIds([due], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { due }, fixture.Storage.MetadataReads); + Assert.Equal(3, fixture.Catalog.YieldedIds); + Assert.Equal(1, fixture.Catalog.ListCalls); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + } + + [Fact] + public async Task Discovery_VolatileCatalogHonorsTimestampBoundAndProjectsMetadata() + { + await using var fixture = new DiscoveryFixture(useStorageCatalog: true); + var due = await fixture.AddShardAsync("due", fixture.Now); + await fixture.AddShardAsync("future", fixture.Horizon.AddTicks(1)); + + AssertAssignedIds([due], await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Empty(fixture.Storage.MetadataReads); + } + + [Fact] + public async Task Assignment_PublicApiCollectsFreshBoundedOldestFirstSweep() + { + await using var fixture = new DiscoveryFixture(); + var first = await fixture.AddShardAsync("first", fixture.Now.AddMinutes(-1), fixture.Silo); + var second = await fixture.AddShardAsync("second", fixture.Now, fixture.Silo); + var future = await fixture.AddShardAsync("future", fixture.Horizon.AddTicks(1), fixture.Silo); + fixture.Catalog.Ids.AddRange([future, second, first]); + + var initial = await fixture.AssignAsync(); + AssertAssignedIds([first, second], initial); + var inserted = await fixture.AddShardAsync("inserted", fixture.Now.AddYears(-1), fixture.Silo); + fixture.Catalog.Ids.Add(inserted); + var next = await fixture.AssignAsync(); + AssertAssignedIds([inserted, first, second], next); + Assert.Same(initial[0], next[1]); + Assert.Same(initial[1], next[2]); + Assert.Equal(new[] { first, second, inserted, first, second }, fixture.Storage.MetadataReads); + Assert.Equal(2, fixture.Catalog.ListCalls); + Assert.Equal(2, fixture.Catalog.DisposeCalls); + Assert.All(fixture.Catalog.Requests, request => Assert.Equal(JobShardId.GetMaxJournalId(fixture.Horizon), request.MaxId)); + } + + [Theory] + [InlineData(1, 0)] + [InlineData(30, 0)] + [InlineData(1, 3)] + [InlineData(30, 3)] + public async Task Discovery_DefunctPredecessorWorkloadBoundsCandidateMetadataWork(int futureBuckets, int maxNewClaims) + { + const int PredecessorCount = 256; + await using var fixture = new DiscoveryFixture(); + var dueIds = new List(); + for (var predecessor = 0; predecessor < PredecessorCount; predecessor++) + { + var owner = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5200 + predecessor), 0); + fixture.Membership.SetSiloStatus(owner, SiloStatus.Dead); + var due = await fixture.AddShardAsync( + $"due-{predecessor}", fixture.Now.AddYears(-10).AddHours(predecessor), owner); + dueIds.Add(due); + fixture.Catalog.Ids.Add(due); + for (var bucket = 0; bucket < futureBuckets; bucket++) + { + fixture.Catalog.Ids.Add(await fixture.AddShardAsync( + $"future-{predecessor}-{bucket}", + fixture.Horizon.AddYears(1).AddDays(bucket).AddTicks(predecessor), + owner)); + } + } + + fixture.Catalog.Ids.Reverse(); + var stopwatch = Stopwatch.StartNew(); + var assigned = await fixture.DiscoverAsync(maxNewClaims); + stopwatch.Stop(); + + AssertAssignedIds(dueIds.Take(maxNewClaims), assigned); + Assert.Equal(dueIds, fixture.Storage.MetadataReads); + Assert.Equal(PredecessorCount, fixture.Storage.MetadataReads.Count); + Assert.Equal(PredecessorCount, fixture.Catalog.YieldedIds); + Assert.Equal(PredecessorCount + 1, fixture.Catalog.MoveNextCalls); + Assert.Equal(1, fixture.Catalog.ListCalls); + Assert.Equal(1, fixture.Catalog.DisposeCalls); + var request = Assert.Single(fixture.Catalog.Requests); + Assert.Equal(JobShardId.StoragePrefix.Value + "/", request.Prefix.Value); + Assert.Equal(JobShardId.GetMaxJournalId(fixture.Horizon), request.MaxId); + var unboundedIdentityCount = PredecessorCount * (futureBuckets + 1); + Assert.Equal(unboundedIdentityCount, fixture.Catalog.Ids.Count); + TestContext.Current.TestOutputHelper?.WriteLine( + $"Workload: predecessor hosts={PredecessorCount}, due shards per predecessor=1, total due shards={dueIds.Count}, " + + $"future time buckets={futureBuckets}, future shards per bucket={PredecessorCount}, " + + $"total future shards={fixture.Catalog.Ids.Count - dueIds.Count}, total catalog shards={fixture.Catalog.Ids.Count}. " + + $"Per recovering silo (one silo: {fixture.Silo}): hypothetical unbounded candidate identities={unboundedIdentityCount}, " + + $"actual bounded candidate identities={fixture.Catalog.YieldedIds}, actual metadata reads={fixture.Storage.MetadataReads.Count}, " + + $"actual new claims={assigned.Count}, claim budget={maxNewClaims}, catalog enumerations={fixture.Catalog.ListCalls}, " + + $"elapsed sweep={stopwatch.Elapsed.TotalMilliseconds:F3} ms (diagnostic only; no timing threshold). " + + "Candidate and metadata counts measure consumer work; they do not measure or imply reduced provider-internal LIST scanning."); + } + + private static void AssertAssignedIds(IEnumerable expected, IEnumerable actual) + => Assert.Equal(expected.Select(id => JobShardId.FromJournalId(id).Value), actual.Select(shard => shard.Id)); +} diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs index 217c490241a..a0ebacda678 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs @@ -3,6 +3,7 @@ using System.Collections.Immutable; using System.Diagnostics.Metrics; using System.Net; +using System.Runtime.CompilerServices; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -592,7 +593,7 @@ public async Task SlowStart_LimitsOrphanedShardClaims() } } - private static ServiceProvider CreateServices(IJournalStorageProvider storageProvider, TimeProvider? timeProvider = null) + private static ServiceProvider CreateServices(IJournalStorageProvider storageProvider, TimeProvider? timeProvider = null, IJournalStorageCatalog? catalog = null) { var builder = new TestSiloBuilder(); builder.AddJournalStorage(); @@ -601,7 +602,7 @@ private static ServiceProvider CreateServices(IJournalStorageProvider storagePro builder.Services.AddSingleton(timeProvider ?? TimeProvider.System); builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (sp, _) => sp.GetRequiredService()); builder.Services.AddSingleton(storageProvider); - builder.Services.AddSingleton((IJournalStorageCatalog)storageProvider); + builder.Services.AddSingleton(catalog ?? (IJournalStorageCatalog)storageProvider); return builder.Services.BuildServiceProvider(); } @@ -622,6 +623,10 @@ private static MeterListener CreateStorageBatchSizeListener(ConcurrentBag private sealed class CountingJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog { + public ConcurrentQueue MetadataReads { get; } = new(); + public ConcurrentQueue<(JournalId Id, string? ExpectedETag)> MetadataUpdates { get; } = new(); + public Func? BeforeMetadataRead { get; set; } + public bool OmitMetadataETags { get; set; } private readonly VolatileJournalStorageProvider _inner = new(); private readonly Func? _onAppend; private readonly object _appendGate = new(); @@ -673,7 +678,7 @@ public void AllowAppends() public IJournalStorage CreateStorage(JournalId journalId) { OpenedJournalIds.Add(journalId); - return new CountingJournalStorage(this, _inner.CreateStorage(journalId)); + return new CountingJournalStorage(this, journalId, _inner.CreateStorage(journalId)); } public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) @@ -700,22 +705,36 @@ private async ValueTask OnAppendAsync(CancellationToken cancellationToken) } } - private sealed class CountingJournalStorage(CountingJournalStorageProvider owner, IJournalStorage inner) : IJournalStorage + private sealed class CountingJournalStorage(CountingJournalStorageProvider owner, JournalId journalId, IJournalStorage inner) : IJournalStorage { public bool IsCompactionRequested => inner.IsCompactionRequested; public ValueTask CreateIfNotExistsAsync(IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => inner.CreateIfNotExistsAsync(metadata, cancellationToken); - public ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) - => inner.GetMetadataAsync(cancellationToken); + public async ValueTask GetMetadataAsync(CancellationToken cancellationToken = default) + { + owner.MetadataReads.Enqueue(journalId); + if (owner.BeforeMetadataRead is { } beforeRead) + { + await beforeRead(journalId, cancellationToken); + } + + var metadata = await inner.GetMetadataAsync(cancellationToken); + return owner.OmitMetadataETags && metadata is not null + ? new JournalMetadata(metadata.Format, properties: metadata.Properties) + : metadata; + } public ValueTask UpdateMetadataAsync( IReadOnlyDictionary? set = null, IEnumerable? remove = null, string? expectedETag = null, CancellationToken cancellationToken = default) - => inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + { + owner.MetadataUpdates.Enqueue((journalId, expectedETag)); + return inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + } public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) => inner.ReadAsync(consumer, cancellationToken); @@ -734,6 +753,158 @@ public ValueTask DeleteAsync(CancellationToken cancellationToken) } } + private sealed class DiscoveryFixture : IAsyncDisposable + { + private readonly ServiceProvider _services; + private readonly HashSet _opened = []; + + public DiscoveryFixture(bool useStorageCatalog = false) + { + Catalog = new ScriptedCatalog(); + _services = CreateServices(Storage, catalog: useStorageCatalog ? Storage : Catalog); + Membership.SetSiloStatus(Silo, SiloStatus.Active); + Manager = CreateManager(_services, Membership, Silo); + } + + public DateTimeOffset Now { get; } = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + public DateTimeOffset Horizon => Now.AddHours(1); + public SiloAddress Silo { get; } = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5100), 0); + public CountingJournalStorageProvider Storage { get; } = new(delayAppends: false); + public ScriptedCatalog Catalog { get; } + public TestClusterMembershipService Membership { get; } = new(); + public JournaledJobShardManager Manager { get; } + + public async Task AddShardAsync(string name, DateTimeOffset start, SiloAddress? owner = null, bool poisoned = false) + { + var timestampedId = JobShardId.New(start); + var id = new JobShardId(timestampedId.Value[..^32] + name).ToJournalId(); + var properties = new Dictionary + { + ["DurableJobsMinDueTime"] = start.ToString("O"), + ["DurableJobsMaxDueTime"] = start.AddHours(1).ToString("O"), + ["DurableJobsPoisoned"] = poisoned.ToString(), + ["DurableJobsClosed"] = bool.TrueString + }; + if (owner is not null) + { + properties["DurableJobsOwner"] = owner.ToParsableString(); + } + + await Storage.CreateStorage(id).CreateIfNotExistsAsync(properties, TestContext.Current.CancellationToken); + return id; + } + + public Task> DiscoverAsync(int maxNewClaims = int.MaxValue, DateTimeOffset? horizon = null) + => DiscoverWithCancellationAsync(TestContext.Current.CancellationToken, maxNewClaims, horizon); + + public async Task> DiscoverWithCancellationAsync(CancellationToken cancellationToken, int maxNewClaims = int.MaxValue, DateTimeOffset? horizon = null) + { + var result = new List(); + await foreach (var shard in DiscoverStreamAsync(cancellationToken, maxNewClaims, horizon)) + { + result.Add(shard); + } + + return result; + } + + public async IAsyncEnumerable DiscoverStreamAsync( + [EnumeratorCancellation] CancellationToken cancellationToken, + int maxNewClaims = int.MaxValue, + DateTimeOffset? horizon = null) + { + await foreach (var shard in Manager.DiscoverJobShardsAsync(horizon ?? Horizon, maxNewClaims, cancellationToken)) + { + _opened.Add(shard); + yield return shard; + } + } + + public async Task> AssignAsync() + { + var result = await Manager.AssignJobShardsAsync(Horizon, 0, TestContext.Current.CancellationToken); + _opened.UnionWith(result); + return result; + } + + public async ValueTask DisposeAsync() + { + try + { + await Task.WhenAll(_opened.Select(shard => shard.DisposeAsync().AsTask())); + } + finally + { + await _services.DisposeAsync(); + } + } + } + + private sealed class ScriptedCatalog : IJournalStorageCatalog + { + public List Ids { get; } = []; + public Dictionary Metadata { get; } = []; + public Func? BeforeMoveNext { get; set; } + public Func? OnDispose { get; set; } + public int ListCalls { get; private set; } + public int MoveNextCalls { get; private set; } + public int DisposeCalls { get; private set; } + public int YieldedIds { get; private set; } + public List<(JournalId Prefix, JournalId MaxId)> Requests { get; } = []; + + public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) + { + Assert.NotNull(options); + Assert.Equal(JobShardId.StoragePrefix.Value + "/", options.Prefix.Value); + Assert.False(options.MaxId.IsDefault); + Assert.True(options.IncludeMetadata); + ListCalls++; + var prefix = options.Prefix; + var maxId = options.MaxId; + Requests.Add((prefix, maxId)); + return Enumerate(prefix, maxId, cancellationToken); + } + + private async IAsyncEnumerable Enumerate( + JournalId prefix, + JournalId maxId, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var snapshot = Ids.Where(id => id.Value.StartsWith(prefix.Value, StringComparison.Ordinal) + && StringComparer.Ordinal.Compare(id.Value, maxId.Value) <= 0).ToArray(); + try + { + for (var index = 0; ; index++) + { + MoveNextCalls++; + cancellationToken.ThrowIfCancellationRequested(); + if (BeforeMoveNext is { } beforeMoveNext) + { + await beforeMoveNext(index, cancellationToken); + } + + if (index == snapshot.Length) + { + yield break; + } + + YieldedIds++; + var id = snapshot[index]; + Metadata.TryGetValue(id, out var metadata); + yield return new JournalCatalogEntry(id, metadata); + } + } + finally + { + DisposeCalls++; + if (OnDispose is { } onDispose) + { + await onDispose(); + } + } + } + } + private static JournaledJobShardManager CreateManager( IServiceProvider services, TestClusterMembershipService membership, diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardStateTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardStateTests.cs index 616a295c3d7..557d3c4a026 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardStateTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardStateTests.cs @@ -173,6 +173,58 @@ public void JobShardId_MapsToJournalStorageIdentityWithoutExposingRawIds() Assert.Equal(shardId, JobShardId.FromJournalId(storageId)); } + [Fact] + public void JobShardId_New_EncodesUtcStartTimeBeforeUniqueSuffix() + { + var start = new DateTimeOffset(2026, 9, 9, 13, 0, 0, TimeSpan.FromHours(5.5)).AddTicks(1234567); + const string expectedPrefix = "20260909T0730001234567Z-"; + + var first = JobShardId.New(start); + var second = JobShardId.New(start.ToUniversalTime()); + + Assert.StartsWith(expectedPrefix, first.Value); + Assert.StartsWith(expectedPrefix, second.Value); + Assert.True(Guid.TryParseExact(first.Value[expectedPrefix.Length..], "N", out _)); + Assert.NotEqual(first, second); + Assert.Equal($"jobs/shards/{first.Value}", first.ToJournalId().Value); + Assert.Equal(first, JobShardId.FromJournalId(first.ToJournalId())); + Assert.Equal(first, JobShardId.Parse(first.Value)); + } + + [Fact] + public void JobShardId_NameOrderMatchesStartTimeAcrossDateBoundaries() + { + DateTimeOffset[] starts = + [ + DateTimeOffset.MinValue, + new(2025, 12, 31, 23, 59, 59, TimeSpan.Zero), + new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero).AddTicks(1), + DateTimeOffset.MaxValue + ]; + var ids = starts.Select(JobShardId.New).ToArray(); + + Assert.Equal(ids, ids.AsEnumerable().Reverse().OrderBy(id => id.ToJournalId().Value, StringComparer.Ordinal)); + } + + [Fact] + public void JobShardId_DiscoveryBoundIncludesExactStartTimeAndExcludesNextTick() + { + var horizon = new DateTimeOffset(2026, 9, 9, 12, 0, 0, TimeSpan.Zero).AddTicks(1234567); + var before = JobShardId.New(horizon.AddTicks(-1)).ToJournalId(); + var exact = JobShardId.New(horizon).ToJournalId(); + var after = JobShardId.New(horizon.AddTicks(1)).ToJournalId(); + var bound = JobShardId.GetMaxJournalId(horizon); + + Assert.True(JobShardId.StoragePrefix.IsPrefixOf(bound)); + Assert.True(StringComparer.Ordinal.Compare(before.Value, bound.Value) < 0); + Assert.True(StringComparer.Ordinal.Compare(exact.Value, bound.Value) < 0); + Assert.True(StringComparer.Ordinal.Compare(after.Value, bound.Value) > 0); + Assert.True(StringComparer.Ordinal.Compare( + JobShardId.New(DateTimeOffset.MaxValue).ToJournalId().Value, + JobShardId.GetMaxJournalId(DateTimeOffset.MaxValue).Value) < 0); + } + [Fact] public void Apply_DefaultKind_Throws() { From 2ba6384382e54e153839b4c57022b476d9472fd2 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 02:28:06 -0700 Subject: [PATCH 7/9] fix(durable-jobs): dispose duplicate shard claims Preserve the existing cached shard when discovery races with unregister cleanup, await disposal of a losing instance, and retain claim budget accounting. Add a deterministic release/claim race regression covering cache identity, disposal, and subsequent job recovery. --- .../JournaledJobShardManager.cs | 10 +- .../JournaledJobShardDiscoveryTests.cs | 92 +++++++++++++++++++ .../JournaledJobShardManagerTests.cs | 16 +++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/Orleans.DurableJobs/JournaledJobShardManager.cs b/src/Orleans.DurableJobs/JournaledJobShardManager.cs index aeab8738474..22d27548b6a 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardManager.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardManager.cs @@ -174,8 +174,14 @@ internal override async IAsyncEnumerable DiscoverJobShardsAsync( return default; } - _jobShardCache[claimedShard.Id] = claimedShard; - return (claimedShard, true); + if (_jobShardCache.TryAdd(claimedShard.Id, claimedShard)) + { + return (claimedShard, true); + } + + // Unregister can still own the previous instance after releasing storage ownership. + await claimedShard.DisposeAsync(); + return (null, true); } public override async Task CreateShardAsync(DateTimeOffset minDueTime, DateTimeOffset maxDueTime, IDictionary metadata, CancellationToken cancellationToken) diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 1ebe58acf7c..63940c2d356 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -1,5 +1,7 @@ using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Net; +using Microsoft.Extensions.DependencyInjection; using Orleans.DurableJobs; using Orleans.Journaling; using Orleans.Runtime; @@ -444,6 +446,66 @@ public async Task Discovery_DuplicateIdentitiesReadMetadataAndAssignOnlyOnce() Assert.Equal(1, fixture.Catalog.DisposeCalls); } + [Fact] + public async Task Discovery_ConcurrentUnregisterDisposesDuplicateClaimAndPreservesCachedShard() + { + var cancellationToken = TestContext.Current.CancellationToken; + var storage = new CountingJournalStorageProvider(delayAppends: false); + await using var services = CreateServices(storage); + var factory = new TrackingJournaledStateManagerFactory(services.GetRequiredService()); + var membership = new TestClusterMembershipService(); + var silo = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5100), 0); + membership.SetSiloStatus(silo, SiloStatus.Active); + var manager = CreateManager(services, membership, silo, stateManagerFactory: factory); + var start = DateTimeOffset.UtcNow.AddMinutes(-1); + await using var shard = await manager.CreateShardAsync(start, start.AddHours(1), new Dictionary(), cancellationToken); + await ScheduleJobAsync(shard, "retained-job", cancellationToken); + var storageId = ((JournaledJobShard)shard).StorageId; + var releaseStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + storage.AfterMetadataUpdate = async (id, metadata, token) => + { + if (id == storageId && metadata is not null && !metadata.Properties.ContainsKey("DurableJobsOwner")) + { + releaseStarted.TrySetResult(); + await allowRelease.Task.WaitAsync(token); + } + }; + + var unregister = manager.UnregisterShardAsync(shard, cancellationToken); + List discovered = []; + try + { + await releaseStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(unregister.IsCompleted); + + discovered = await manager.AssignJobShardsAsync(start.AddHours(1), 1, cancellationToken); + Assert.Empty(discovered); + Assert.Equal(2, factory.Managers.Count); + Assert.Equal(0, factory.Managers[0].DisposeCalls); + Assert.Equal(1, factory.Managers[1].DisposeCalls); + Assert.Same(shard, Assert.Single(await manager.AssignJobShardsAsync(start.AddHours(1), 0, cancellationToken))); + } + finally + { + allowRelease.TrySetResult(); + await unregister; + foreach (var instance in discovered) + { + await instance.DisposeAsync(); + } + } + + Assert.Equal(1, factory.Managers[0].DisposeCalls); + storage.AfterMetadataUpdate = null; + await using var recovered = Assert.Single(await manager.AssignJobShardsAsync(start.AddHours(1), 0, cancellationToken)); + Assert.NotSame(shard, recovered); + Assert.Equal(3, factory.Managers.Count); + Assert.Equal(1, await recovered.GetJobCountAsync()); + await DrainAndUnregisterAsync(manager, recovered, cancellationToken); + Assert.All(factory.Managers, stateManager => Assert.Equal(1, stateManager.DisposeCalls)); + } + [Fact] public async Task Discovery_VolatileCatalogHonorsTimestampBoundAndProjectsMetadata() { @@ -535,4 +597,34 @@ public async Task Discovery_DefunctPredecessorWorkloadBoundsCandidateMetadataWor private static void AssertAssignedIds(IEnumerable expected, IEnumerable actual) => Assert.Equal(expected.Select(id => JobShardId.FromJournalId(id).Value), actual.Select(shard => shard.Id)); + + private sealed class TrackingJournaledStateManagerFactory(IJournaledStateManagerFactory inner) : IJournaledStateManagerFactory + { + public List Managers { get; } = []; + + public IJournaledStateManager Create(JournalId journalId) + { + var manager = new TrackingJournaledStateManager(inner.Create(journalId)); + Managers.Add(manager); + return manager; + } + } + + private sealed class TrackingJournaledStateManager(IJournaledStateManager inner) : IJournaledStateManager + { + public int DisposeCalls { get; private set; } + public long PendingWriteByteCount => inner.PendingWriteByteCount; + public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); + public void RegisterState(string name, IJournaledState state) => inner.RegisterState(name, state); + public bool TryGetState(string name, [NotNullWhen(true)] out IJournaledState? state) => inner.TryGetState(name, out state); + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => inner.WriteStateAsync(cancellationToken); + public ValueTask RevertPendingChangesAsync(CancellationToken cancellationToken) => inner.RevertPendingChangesAsync(cancellationToken); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => inner.DeleteStateAsync(cancellationToken); + + public async ValueTask DisposeAsync() + { + await inner.DisposeAsync(); + DisposeCalls++; + } + } } diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs index a0ebacda678..9b04dfcfd9a 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs @@ -626,6 +626,7 @@ private sealed class CountingJournalStorageProvider : IJournalStorageProvider, I public ConcurrentQueue MetadataReads { get; } = new(); public ConcurrentQueue<(JournalId Id, string? ExpectedETag)> MetadataUpdates { get; } = new(); public Func? BeforeMetadataRead { get; set; } + public Func? AfterMetadataUpdate { get; set; } public bool OmitMetadataETags { get; set; } private readonly VolatileJournalStorageProvider _inner = new(); private readonly Func? _onAppend; @@ -726,14 +727,20 @@ public ValueTask CreateIfNotExistsAsync(IReadOnlyDictionary UpdateMetadataAsync( + public async ValueTask UpdateMetadataAsync( IReadOnlyDictionary? set = null, IEnumerable? remove = null, string? expectedETag = null, CancellationToken cancellationToken = default) { owner.MetadataUpdates.Enqueue((journalId, expectedETag)); - return inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + var metadata = await inner.UpdateMetadataAsync(set, remove, expectedETag, cancellationToken); + if (owner.AfterMetadataUpdate is { } afterUpdate) + { + await afterUpdate(journalId, metadata, cancellationToken); + } + + return metadata; } public ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) @@ -909,10 +916,11 @@ private static JournaledJobShardManager CreateManager( IServiceProvider services, TestClusterMembershipService membership, SiloAddress siloAddress, - DurableJobsOptions? options = null) + DurableJobsOptions? options = null, + IJournaledStateManagerFactory? stateManagerFactory = null) => new( new TestLocalSiloDetails(siloAddress), - services.GetRequiredService(), + stateManagerFactory ?? services.GetRequiredService(), services.GetRequiredService(), services.GetRequiredService(), membership, From f164e73526d5cbb75c9677874c4ace1bb2e1b1f9 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 07:15:32 -0700 Subject: [PATCH 8/9] fix(durable-jobs): revalidate uncached local shard snapshots Read current metadata when a projected local-owner snapshot has no cached shard, then apply the normal eligibility and conditional ownership decisions. Preserve cached instances and add deterministic coverage for release, deletion, reassignment, poisoned state, and cache reuse. --- .../JournaledJobShardManager.cs | 11 +- src/Orleans.DurableJobs/README.md | 3 + .../JournaledJobShardDiscoveryTests.cs | 116 ++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/src/Orleans.DurableJobs/JournaledJobShardManager.cs b/src/Orleans.DurableJobs/JournaledJobShardManager.cs index 22d27548b6a..eca24a9cdbc 100644 --- a/src/Orleans.DurableJobs/JournaledJobShardManager.cs +++ b/src/Orleans.DurableJobs/JournaledJobShardManager.cs @@ -132,6 +132,15 @@ internal override async IAsyncEnumerable DiscoverJobShardsAsync( var descriptor = entry.Metadata is { ETag: not null } metadata ? ShardCatalogProperties.From(entry.Id, metadata) : await GetDescriptorAsync(entry.Id, cancellationToken); + JournaledJobShard? cachedShard = null; + if (descriptor?.Owner is { } snapshotOwner && snapshotOwner.Equals(SiloAddress) + && !_jobShardCache.TryGetValue(descriptor.ShardId.Value, out cachedShard) + && entry.Metadata is { ETag: not null }) + { + // A listed local owner can have released the shard since the snapshot was taken. + descriptor = await GetDescriptorAsync(entry.Id, cancellationToken); + } + if (descriptor is null || descriptor.Poisoned || descriptor.StartTime > maxDueTime) { return default; @@ -147,7 +156,7 @@ internal override async IAsyncEnumerable DiscoverJobShardsAsync( cancellationToken.ThrowIfCancellationRequested(); if (descriptor.Owner is { } owner && owner.Equals(SiloAddress)) { - return (await GetOrOpenShardAsync(descriptor, cancellationToken), false); + return (cachedShard ?? await GetOrOpenShardAsync(descriptor, cancellationToken), false); } var isAdopted = false; diff --git a/src/Orleans.DurableJobs/README.md b/src/Orleans.DurableJobs/README.md index 230b6501705..8a53c99fe90 100644 --- a/src/Orleans.DurableJobs/README.md +++ b/src/Orleans.DurableJobs/README.md @@ -126,6 +126,9 @@ outage. Future shard identities are filtered by the catalog before candidate met Each periodic or membership check starts a fresh, locally scoped sweep. Discovery requests catalog metadata, orders and deduplicates the selected entries, then uses each supplied ownership snapshot with an ETag or reads current metadata when that snapshot is unavailable. +For a snapshot naming the local silo as owner, discovery reuses the cached shard or reads +current metadata on a cache miss. The refreshed descriptor determines eligibility, ownership, +and any required conditional claim before a new instance is opened. Claims run oldest first and require the snapshot's ETag for conditional updates, so a concurrent ownership change rejects a stale claim. Providers used for Durable Jobs supply metadata ETags and enforce conditional updates; a missing ETag surfaces as a discovery error. diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs index 63940c2d356..074495abc97 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardDiscoveryTests.cs @@ -120,6 +120,122 @@ public async Task Discovery_StaleProjectedMetadataCannotOverwriteConcurrentOwner Assert.Equal(other.ToParsableString(), after.Properties["DurableJobsOwner"]); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Discovery_ProjectedLocalOwnerRevalidatesAfterUnregister(bool deleteEmptyShard) + { + await using var fixture = new DiscoveryFixture(); + var cancellationToken = TestContext.Current.CancellationToken; + await using var shard = await fixture.Manager.CreateShardAsync( + fixture.Now, fixture.Horizon, new Dictionary(), cancellationToken); + if (!deleteEmptyShard) + { + Assert.NotNull(await shard.TryScheduleJobAsync(new() + { + Target = GrainId.Create("type", "target"), + JobName = "retained-job", + DueTime = fixture.Now + }, cancellationToken)); + } + + var id = ((JournaledJobShard)shard).StorageId; + var storage = fixture.Storage.CreateStorage(id); + var snapshot = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(snapshot); + fixture.Catalog.Metadata.Add(id, snapshot); + fixture.Catalog.Ids.Add(id); + fixture.Catalog.OnDispose = async () => await fixture.Manager.UnregisterShardAsync(shard, cancellationToken); + fixture.Storage.MetadataReads.Clear(); + fixture.Storage.MetadataUpdates.Clear(); + + Assert.Empty(await fixture.DiscoverAsync(maxNewClaims: 0)); + Assert.Equal(new[] { id, id }, fixture.Storage.MetadataReads); + Assert.Equal(deleteEmptyShard ? 0 : 1, fixture.Storage.MetadataUpdates.Count); + var current = await storage.GetMetadataAsync(cancellationToken); + if (deleteEmptyShard) + { + Assert.Null(current); + } + else + { + Assert.NotNull(current); + Assert.False(current.Properties.ContainsKey("DurableJobsOwner")); + fixture.Catalog.OnDispose = null; + fixture.Storage.MetadataReads.Clear(); + fixture.Storage.MetadataUpdates.Clear(); + + var recovered = Assert.Single(await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.NotSame(shard, recovered); + Assert.True(recovered.IsAddingCompleted); + Assert.Equal(1, await recovered.GetJobCountAsync()); + Assert.Equal(new[] { id }, fixture.Storage.MetadataReads); + Assert.Equal((id, current.ETag), Assert.Single(fixture.Storage.MetadataUpdates)); + Assert.Same(recovered, Assert.Single(await fixture.DiscoverAsync(maxNewClaims: 0))); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Discovery_ProjectedLocalOwnerRevalidatesCurrentEligibility(bool poisoned) + { + await using var fixture = new DiscoveryFixture(); + var cancellationToken = TestContext.Current.CancellationToken; + var id = await fixture.AddShardAsync("formerly-local", fixture.Now, fixture.Silo); + var storage = fixture.Storage.CreateStorage(id); + var snapshot = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(snapshot); + fixture.Catalog.Metadata.Add(id, snapshot); + fixture.Catalog.Ids.Add(id); + var other = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5101), 0); + fixture.Membership.SetSiloStatus(other, SiloStatus.Active); + IJournalMetadata? current = null; + fixture.Catalog.OnDispose = async () => + { + current = await storage.UpdateMetadataAsync( + poisoned + ? new Dictionary { ["DurableJobsPoisoned"] = bool.TrueString } + : new Dictionary { ["DurableJobsOwner"] = other.ToParsableString() }, + expectedETag: snapshot.ETag, cancellationToken: cancellationToken); + Assert.NotNull(current); + }; + fixture.Storage.MetadataReads.Clear(); + fixture.Storage.MetadataUpdates.Clear(); + + Assert.Empty(await fixture.DiscoverAsync(maxNewClaims: 1)); + Assert.Equal(new[] { id }, fixture.Storage.MetadataReads); + Assert.Equal((id, snapshot.ETag), Assert.Single(fixture.Storage.MetadataUpdates)); + var after = await storage.GetMetadataAsync(cancellationToken); + Assert.NotNull(current); + Assert.NotNull(after); + Assert.Equal(current.ETag, after.ETag); + Assert.Equal(poisoned ? fixture.Silo.ToParsableString() : other.ToParsableString(), after.Properties["DurableJobsOwner"]); + Assert.Equal(poisoned.ToString(), after.Properties["DurableJobsPoisoned"]); + } + + [Fact] + public async Task Discovery_ProjectedLocalOwnerReadsCurrentMetadataOnCacheMissAndReusesCachedInstance() + { + await using var fixture = new DiscoveryFixture(); + var id = await fixture.AddShardAsync("local", fixture.Now, fixture.Silo); + var snapshot = await fixture.Storage.CreateStorage(id).GetMetadataAsync(TestContext.Current.CancellationToken); + Assert.NotNull(snapshot); + fixture.Catalog.Metadata.Add(id, snapshot); + fixture.Catalog.Ids.Add(id); + fixture.Storage.MetadataReads.Clear(); + + var shard = Assert.Single(await fixture.DiscoverAsync(maxNewClaims: 0)); + AssertAssignedIds([id], [shard]); + Assert.Equal(new[] { id }, fixture.Storage.MetadataReads); + Assert.Empty(fixture.Storage.MetadataUpdates); + + fixture.Storage.MetadataReads.Clear(); + Assert.Same(shard, Assert.Single(await fixture.DiscoverAsync(maxNewClaims: 0))); + Assert.Empty(fixture.Storage.MetadataReads); + Assert.Empty(fixture.Storage.MetadataUpdates); + } + [Fact] public async Task Discovery_UnorderedCatalogClaimsOldestFirstIncludingYearsOverdue() { From f5d1840bf9efa93cad1e796ddef4a0430c7afd40 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 15 Sep 2026 12:31:18 -0700 Subject: [PATCH 9/9] test(durable-jobs): align discovery with current lifecycle Assert inactive yielded shards are unregistered on shutdown and verify the descendant-only catalog prefix while cached projected shards require no storage opens. --- .../DurableJobs/LocalDurableJobManagerTests.cs | 2 +- .../DurableJobs/JournaledJobShardManagerTests.cs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs index 3766e7f2875..83926f463e0 100644 --- a/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs +++ b/test/Orleans.Core.Tests/DurableJobs/LocalDurableJobManagerTests.cs @@ -476,7 +476,7 @@ async IAsyncEnumerable DiscoverShards() } Assert.Equal(1, shard.DisposeCallCount); - Assert.Empty(shardManager.UnregisteredShards); + Assert.Same(shard, Assert.Single(shardManager.UnregisteredShards)); Assert.False(accessor.HasCachedShard(shard.Id)); Assert.False(shard.ConsumeStarted.Task.IsCompleted); } diff --git a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs index 9b04dfcfd9a..3fc10692aa5 100644 --- a/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs +++ b/test/Orleans.DurableJobs.Tests/DurableJobs/JournaledJobShardManagerTests.cs @@ -137,7 +137,8 @@ public async Task AssignJobShardsAsync_OpensOnlyShardNamespaceDescendants() var assigned = await manager.AssignJobShardsAsync(start.AddHours(1), int.MaxValue, cancellationToken); Assert.Same(shard, Assert.Single(assigned)); - Assert.Equal(((JournaledJobShard)shard).StorageId, Assert.Single(storageProvider.OpenedJournalIds)); + Assert.Equal(JobShardId.StoragePrefix.Value + "/", storageProvider.LastListPrefix.Value); + Assert.Empty(storageProvider.OpenedJournalIds); await manager.UnregisterShardAsync(shard, cancellationToken); } @@ -656,6 +657,7 @@ public Task AppendStarted public int AppendCount => Volatile.Read(ref _appendCount); public ConcurrentBag OpenedJournalIds { get; } = new(); + public JournalId LastListPrefix { get; private set; } public void BlockAppends() { @@ -683,7 +685,10 @@ public IJournalStorage CreateStorage(JournalId journalId) } public IAsyncEnumerable ListAsync(ListOptions? options = null, CancellationToken cancellationToken = default) - => _inner.ListAsync(options, cancellationToken); + { + LastListPrefix = options?.Prefix ?? default; + return _inner.ListAsync(options, cancellationToken); + } private async ValueTask OnAppendAsync(CancellationToken cancellationToken) {