diff --git a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md index 25e8022b98b..13130384a93 100644 --- a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md +++ b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md @@ -86,6 +86,18 @@ Request latency covers the interval until the caller's callback completes, inclu | `orleans-scheduler-long-running-turns` | C | Turns, implicit | - | Grain micro-turns whose synchronous execution exceeded . | | `orleans-system-targets` | UDC | System targets, implicit | `type` | Current Orleans system-target instances by type. | +## Cluster manifests + +| Instrument | Type | Unit | Attributes | Description | +|---|---|---|---|---| +| `orleans-manifest-cache-lookups` | C | Lookups, implicit | `result`, `source` | Content-cache lookups with `hit` or `miss` results, sourced from a `silo` hash response or a `peer` summary. | +| `orleans-manifest-fallbacks` | C | Retrievals, implicit | `reason` | Hash retrievals falling back to the direct manifest RPC: `error`, `missing`, or `mismatch`. | +| `orleans-manifest-peer-probes` | C | Attempts, implicit | `status` | Local peer attempts completed with `success`, `timeout`, `canceled`, or `error`; `skipped` counts attempts denied by local admission. | +| `orleans-manifest-peer-repairs` | C | Silo entries, implicit | - | Missing silo manifest entries supplied by successfully published peer repairs. | +| `orleans-manifest-retrieval-duration` | H | `ms` | `mode`, `status` | Local silo-manifest retrieval duration in `direct` or `hash` mode, ending in `success`, `error`, or `canceled`. | + +The categories are fixed and carry no silo addresses, manifest hashes, or grain type names. Probe cancellation includes optional work superseded by complete direct results. See [cluster manifest retrieval](../../implementation/cluster-manifest-retrieval.md) for publication, fallback, and rollout semantics. + ## Grain directory and consistent rings | Instrument | Type | Unit | Attributes | Description | diff --git a/docs/site/src/content/docs/implementation/cluster-manifest-retrieval.md b/docs/site/src/content/docs/implementation/cluster-manifest-retrieval.md new file mode 100644 index 00000000000..4180885790f --- /dev/null +++ b/docs/site/src/content/docs/implementation/cluster-manifest-retrieval.md @@ -0,0 +1,62 @@ +--- +title: Cluster manifest retrieval +description: How Orleans discovers silo manifests, reuses content hashes, and repairs missing metadata during membership changes. +ms.date: 09/10/2026 +ms.topic: concept-article +--- + +# Cluster manifest retrieval + +Each silo publishes an immutable grain manifest describing its grain types, interfaces, and properties. The cluster manifest provider assembles these manifests into the metadata used for type resolution and version-aware placement. + +## Membership and publication + +Cluster manifests have a major version corresponding to the membership version and a minor version which advances as manifests arrive. When membership advances, the provider synchronously prunes non-active silos and includes the local silo when it is active. Local grain metadata also remains available while the local silo is starting. + +The provider retrieves missing active silos' manifests and publishes successful results. Unsuccessful fetches are retried after five seconds or when a newer membership snapshot arrives. Cancellation reaches the remote call and stops local waiting. + +## Content-addressed retrieval + + defaults to `true`, enabling hash-based retrieval and peer repair. Set it to `false` in to request each missing active silo's manifest directly. Configure the option through the silo builder before starting the silo. The provider captures the value at construction; a restart applies configuration changes. + +With this option enabled, the provider asks each missing silo for the content hash of its local manifest. A matching entry in the local hash cache supplies the manifest immediately. Otherwise, the provider requests the manifest by hash, verifies its content, and adds it to the cache. + +Hashes use incremental SHA-256 over a fixed canonical traversal: an encoding version, sorted grain entries, then sorted interface entries. Each section and property collection starts with its count. Identifiers have a byte-length prefix; property keys and values have UTF-16 code-unit-length prefixes. A length of `-1` represents null or a default identifier, while `0` represents an empty value. Counts and lengths are signed 32-bit big-endian integers, and UTF-16 code units are written in big-endian order. This layout preserves exact identifier bytes and string contents, including invalid UTF-16, and gives equivalent content the same hash regardless of dictionary insertion order. Hashes of immutable manifest instances are memoized with weak keys, allowing the manifests to be collected when their owners release them. + +Each publication creates a fresh cache containing its live manifests and local metadata. In-flight retrievals retain their original cache instance. A stale retrieval can populate that instance, while newer publications retain their own cache. The provider exposes the new manifest version before its new cache. Retrievals capture the cache before checking the version, so an older update which captures the newer cache also observes that its membership snapshot has been superseded. + +## Peer repair and bounded waiting + +When more than one active silo's manifest is missing, the enabled provider also probes up to three peers selected from a rotating ordered membership list. Direct retrieval starts alongside these probes. + +Each probe has a one-second deadline shared by its hash-summary and manifest-update requests. Cached hashes can satisfy missing entries directly; otherwise the provider requests a complete update from the peer and verifies each candidate against the summary's hash. Successfully repaired entries are published immediately, including partial repairs. Direct requests for those entries are removed from the required completion set, allowing the remaining successful fetches to advance the manifest independently. + +When direct retrieval supplies every missing manifest first, the provider cancels the optional peer attempts and publishes the direct results immediately. The successful direct path therefore completes independently of a slow peer-summary request. + +The provider admits at most three concurrent local probe attempts. Completion, timeout, and caller cancellation release the attempt's slot immediately, allowing later retries to proceed. The one-second deadline cancels the request token and ends local waiting; Orleans signals cancellation to the peer using the ordinary RPC path. Direct retrieval continues alongside these attempts. Late responses leave the completed attempt's result unchanged, and late failures are observed and logged. Retry selection rotates through active peers, with retries paced by the five-second delay or a newer membership snapshot. + +## Compatibility, rollout, and rollback + +Silos serve hash requests on demand, including when they are configured for direct retrieval. Providers using the default content-addressed mode compute and reuse content hashes as manifests become available. + +During a rolling deployment, enabled silos try hash retrieval and fall back to the established direct RPC when a peer rejects the newer method or its hash request fails. An invalid or unavailable hash-addressed body also triggers direct retrieval. Independent remote cancellation follows that compatibility path; cancellation of the local request propagates to the caller. Peer-repair failures leave direct retrieval responsible for filling the missing entries. + +Upgraded silos adopt content-addressed retrieval at startup while existing silos continue using their deployed implementation. Unsupported hash requests can produce transient exception logs and extra requests during the upgrade; successful direct fallback supplies the required metadata. Individual silo-manifest RPCs retain the configured system response timeout, while optional peer-summary/update attempts have a one-second deadline. + +Observe manifest retrieval and peer-probe diagnostics during joins and restarts. Debug logs distinguish hash fallback, peer timeout, occupied probe slots, and late failures. Warnings identify failed direct fetches. To select direct retrieval, set the option to `false` and restart the affected silos; they continue answering enabled peers' hash requests on demand. + +## Measuring retrieval + +The `Microsoft.Orleans` meter exposes [manifest retrieval instruments](../host/monitoring/metrics-catalog.md#cluster-manifests). Compare a canary with silos using direct retrieval during equivalent joins and restarts: + +| Signal | Interpretation | +|---|---| +| `orleans-manifest-cache-lookups`, split by `result` and `source` | The hit fraction shows how often known content satisfies direct hash requests or peer-summary lookups. | +| `orleans-manifest-fallbacks`, split by `reason` | Counts hash retrievals which proceed through the direct RPC after an error, missing body, or content mismatch. | +| `orleans-manifest-peer-probes`, split by `status` | Shows completed local attempts, timeouts, cancellation, errors, and admission skips. Cancellation also includes optional probes superseded by successful direct retrieval. | +| `orleans-manifest-peer-repairs` | Counts missing silo entries supplied by successfully published peer repairs. Repeated summaries contribute each repaired entry once per publication. | +| `orleans-manifest-retrieval-duration`, split by `mode` and `status` | Measures each local silo-manifest retrieval in milliseconds, including cache lookup, fallback, and terminal cancellation or failure. | + +Attributes use fixed categories, keeping time-series cardinality stable across silo restarts. The direct mode records retrieval duration while the hash and peer counters reflect content-addressed retrieval activity. Use these metrics alongside serialized message sizes and silo CPU/memory to evaluate the benefit for the service's manifest sizes and mix of application versions. + +See [cluster membership](cluster-management.md) for membership transitions and [rolling version skew](rolling-version-skew.md) for how manifest metadata drives interface-version selection. diff --git a/docs/site/src/content/docs/toc.yml b/docs/site/src/content/docs/toc.yml index 508f8578f84..5372e6b840b 100644 --- a/docs/site/src/content/docs/toc.yml +++ b/docs/site/src/content/docs/toc.yml @@ -343,6 +343,8 @@ items: href: implementation/grain-directory.md - name: Cluster membership href: implementation/cluster-management.md + - name: Cluster manifest retrieval + href: implementation/cluster-manifest-retrieval.md - name: Runtime lifecycle href: implementation/orleans-lifecycle.md - name: Scheduling and turn execution diff --git a/src/Orleans.Core/Diagnostics/Metrics/ClusterManifestInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/ClusterManifestInstruments.cs new file mode 100644 index 00000000000..5e21f38e8cf --- /dev/null +++ b/src/Orleans.Core/Diagnostics/Metrics/ClusterManifestInstruments.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace Orleans.Runtime; + +internal sealed class ClusterManifestInstruments(OrleansInstruments instruments) +{ + private readonly Counter _cacheLookups = instruments.Meter.CreateCounter(InstrumentNames.MANIFEST_CACHE_LOOKUPS); + private readonly Counter _fallbacks = instruments.Meter.CreateCounter(InstrumentNames.MANIFEST_FALLBACKS); + private readonly Counter _peerProbes = instruments.Meter.CreateCounter(InstrumentNames.MANIFEST_PEER_PROBES); + private readonly Counter _peerRepairs = instruments.Meter.CreateCounter(InstrumentNames.MANIFEST_PEER_REPAIRS); + private readonly Histogram _retrievalDuration = instruments.Meter.CreateHistogram(InstrumentNames.MANIFEST_RETRIEVAL_DURATION, "ms"); + + public bool RetrievalDurationEnabled => _retrievalDuration.Enabled; + + public void OnCacheLookup(bool hit, string source) => _cacheLookups.Add( + 1, + new KeyValuePair("result", hit ? "hit" : "miss"), + new KeyValuePair("source", source)); + + public void OnFallback(string reason) => _fallbacks.Add(1, new KeyValuePair("reason", reason)); + + public void OnPeerProbe(string status) => _peerProbes.Add(1, new KeyValuePair("status", status)); + + public void OnPeerRepair(int count) => _peerRepairs.Add(count); + + public void OnRetrievalCompleted(TimeSpan elapsed, string mode, string status) + { + if (_retrievalDuration.Enabled) + { + _retrievalDuration.Record( + elapsed.TotalMilliseconds, + new KeyValuePair("mode", mode), + new KeyValuePair("status", status)); + } + } +} diff --git a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs index 91f8f0cf8d4..20e51cc78e6 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs @@ -38,6 +38,13 @@ internal static class InstrumentNames // Runtime public const string SCHEDULER_NUM_LONG_RUNNING_TURNS = "orleans-scheduler-long-running-turns"; + // Cluster manifests + public const string MANIFEST_CACHE_LOOKUPS = "orleans-manifest-cache-lookups"; + public const string MANIFEST_FALLBACKS = "orleans-manifest-fallbacks"; + public const string MANIFEST_PEER_PROBES = "orleans-manifest-peer-probes"; + public const string MANIFEST_PEER_REPAIRS = "orleans-manifest-peer-repairs"; + public const string MANIFEST_RETRIEVAL_DURATION = "orleans-manifest-retrieval-duration"; + // Catalog public const string CATALOG_ACTIVATION_COUNT = "orleans-catalog-activations"; public const string CATALOG_ACTIVATION_WORKING_SET = "orleans-catalog-activation-working-set"; diff --git a/src/Orleans.Core/Manifest/IClusterManifestSystemTarget.cs b/src/Orleans.Core/Manifest/IClusterManifestSystemTarget.cs index 4ecf6f8d0c6..b75277a66c3 100644 --- a/src/Orleans.Core/Manifest/IClusterManifestSystemTarget.cs +++ b/src/Orleans.Core/Manifest/IClusterManifestSystemTarget.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Collections.Immutable; using System.Threading; using System.Threading.Tasks; @@ -13,6 +14,7 @@ internal interface IClusterManifestSystemTarget : ISystemTarget /// /// Gets the current cluster manifest. /// + /// The token used to cancel the request. /// The current cluster manifest. [Alias("40D39F85")] ValueTask GetClusterManifest(CancellationToken cancellationToken = default); @@ -20,9 +22,90 @@ internal interface IClusterManifestSystemTarget : ISystemTarget /// /// Gets an updated cluster manifest if newer than the provided . /// + /// The last observed manifest version. + /// The token used to cancel the request. /// The current cluster manifest, or if it is not newer than the provided version. [Alias("4EFCA109")] ValueTask GetClusterManifestUpdate(MajorMinorVersion previousVersion, CancellationToken cancellationToken = default); + + /// + /// Gets a hash summary for the current cluster manifest. + /// + /// The token used to cancel the request. + /// The current cluster manifest hash summary. + [Alias("25AE6E4A")] + ValueTask GetClusterManifestHashSummary(CancellationToken cancellationToken); + + /// + /// Gets the hash of the local silo manifest. + /// + /// The token used to cancel the request. + /// The hash of the local silo manifest. + [Alias("3D9B7FE6")] + ValueTask GetSiloManifestHash(CancellationToken cancellationToken); + + /// + /// Gets the local silo manifest if the provided hash matches it. + /// + /// The expected manifest hash. + /// The token used to cancel the request. + /// The local silo manifest, or if the hash does not match. + [Alias("93B8854F")] + ValueTask GetSiloManifestByHash(ManifestHash hash, CancellationToken cancellationToken); + } + + /// + /// Identifies a manifest by its canonical content hash. + /// + [GenerateSerializer, Immutable] + internal readonly struct ManifestHash : System.IEquatable + { + public ManifestHash(string value) + { + Value = value; + } + + [Id(0)] + public string Value { get; } + + public bool Equals(ManifestHash other) => string.Equals(Value, other.Value, System.StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is ManifestHash other && Equals(other); + + public override int GetHashCode() => System.StringComparer.Ordinal.GetHashCode(Value ?? string.Empty); + + public override string ToString() => Value ?? string.Empty; + + public static bool operator ==(ManifestHash left, ManifestHash right) => left.Equals(right); + + public static bool operator !=(ManifestHash left, ManifestHash right) => !left.Equals(right); + } + + /// + /// Represents a hash summary for a cluster manifest. + /// + [GenerateSerializer, Immutable] + internal sealed class ClusterManifestHashSummary + { + public ClusterManifestHashSummary( + MajorMinorVersion version, + Dictionary siloManifestHashes) + { + Version = version; + SiloManifestHashes = siloManifestHashes.ToImmutableDictionary(); + } + + /// + /// Gets the cluster manifest version. + /// + [Id(0)] + public MajorMinorVersion Version { get; } + + /// + /// Gets the manifest hash for each silo. + /// + [Id(1)] + public ImmutableDictionary SiloManifestHashes { get; } } /// diff --git a/src/Orleans.Core/OrleansContracts.txt b/src/Orleans.Core/OrleansContracts.txt index 8c85d3f7bcc..2536412af67 100644 --- a/src/Orleans.Core/OrleansContracts.txt +++ b/src/Orleans.Core/OrleansContracts.txt @@ -49,7 +49,10 @@ interface [GrainInterfaceType("Orleans.Placement.Repartitioning.IActivationRepar interface [GrainInterfaceType("Orleans.Runtime.IClusterManifestSystemTarget")] Orleans.Runtime.IClusterManifestSystemTarget [Version(0)] 40D39F85: GetClusterManifest(System.Threading.CancellationToken) -> ValueTask + 25AE6E4A: GetClusterManifestHashSummary(System.Threading.CancellationToken) -> ValueTask 4EFCA109: GetClusterManifestUpdate(Orleans.Metadata.MajorMinorVersion, System.Threading.CancellationToken) -> ValueTask + 93B8854F: GetSiloManifestByHash(Orleans.Runtime.ManifestHash, System.Threading.CancellationToken) -> ValueTask + 3D9B7FE6: GetSiloManifestHash(System.Threading.CancellationToken) -> ValueTask interface [GrainInterfaceType("Orleans.Runtime.IDeploymentLoadPublisher")] Orleans.Runtime.IDeploymentLoadPublisher [Version(0)] C5255F0C: UpdateRuntimeStatistics(Orleans.Runtime.SiloAddress, Orleans.Runtime.SiloRuntimeStatistics, System.Threading.CancellationToken) -> Task diff --git a/src/Orleans.Runtime/Configuration/Options/ClusterManifestOptions.cs b/src/Orleans.Runtime/Configuration/Options/ClusterManifestOptions.cs new file mode 100644 index 00000000000..fa0b70d731f --- /dev/null +++ b/src/Orleans.Runtime/Configuration/Options/ClusterManifestOptions.cs @@ -0,0 +1,22 @@ +namespace Orleans.Configuration; + +/// +/// Configures how a silo retrieves grain manifests from the cluster. +/// +public sealed class ClusterManifestOptions +{ + /// + /// Gets or sets a value indicating whether the silo retrieves manifests by content hash + /// and uses peer summaries to repair missing manifests. + /// + /// + /// by default. Set to to retrieve each active silo's manifest directly. + /// + /// + /// The runtime captures this setting when the silo's manifest provider is constructed. + /// Restart the silo to apply a changed value. Silos serve hash requests from enabled peers + /// on demand, including when their own retrieval is configured to use the direct path. + /// Peer repair uses up to three concurrent local attempts, each with a one-second deadline. + /// + public bool EnableContentAddressedRetrieval { get; set; } = true; +} diff --git a/src/Orleans.Runtime/GrainTypeManager/ClusterManifestSystemTarget.cs b/src/Orleans.Runtime/GrainTypeManager/ClusterManifestSystemTarget.cs index 52fddde1e54..e536a2c795d 100644 --- a/src/Orleans.Runtime/GrainTypeManager/ClusterManifestSystemTarget.cs +++ b/src/Orleans.Runtime/GrainTypeManager/ClusterManifestSystemTarget.cs @@ -1,6 +1,8 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Orleans.Metadata; +using Orleans.Runtime.Metadata; namespace Orleans.Runtime { @@ -12,6 +14,8 @@ internal sealed class ClusterManifestSystemTarget : SystemTarget, IClusterManife private readonly ClusterManifestUpdate? _noUpdate = default; private MembershipVersion _cachedMembershipVersion; private ClusterManifestUpdate? _cachedUpdate; + private MajorMinorVersion _cachedHashSummaryVersion; + private ClusterManifestHashSummary? _cachedHashSummary; public ClusterManifestSystemTarget( IClusterMembershipService clusterMembershipService, @@ -31,6 +35,41 @@ public ValueTask GetClusterManifest(CancellationToken cancellat return new(_clusterManifestProvider.Current); } + public ValueTask GetClusterManifestHashSummary(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var manifest = _clusterManifestProvider.Current; + + // Reuse the summary while the version is unchanged. Individual content hashes are cached by + // immutable manifest identity across versions and across silos with identical manifests. + if (_cachedHashSummary is null || manifest.Version != _cachedHashSummaryVersion) + { + var hashes = new Dictionary(); + foreach (var siloManifest in manifest.Silos) + { + cancellationToken.ThrowIfCancellationRequested(); + hashes[siloManifest.Key] = ManifestHashCalculator.ComputeHash(siloManifest.Value); + } + + _cachedHashSummary = new ClusterManifestHashSummary(manifest.Version, hashes); + _cachedHashSummaryVersion = manifest.Version; + } + + return new(_cachedHashSummary); + } + + public ValueTask GetSiloManifestHash(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(ManifestHashCalculator.ComputeHash(_siloManifest)); + } + + public ValueTask GetSiloManifestByHash(ManifestHash hash, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(hash == ManifestHashCalculator.ComputeHash(_siloManifest) ? _siloManifest : null); + } + public ValueTask GetClusterManifestUpdate( MajorMinorVersion version, CancellationToken cancellationToken = default) diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 6435d28f8c6..5595642641c 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -73,6 +73,7 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); @@ -337,6 +338,7 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.ConfigureFormatter(); services.ConfigureFormatter(); services.ConfigureFormatter(); + services.ConfigureFormatter(); services.ConfigureFormatter(); services.ConfigureFormatter(); services.ConfigureFormatter(); diff --git a/src/Orleans.Runtime/Manifest/ClusterManifestProvider.cs b/src/Orleans.Runtime/Manifest/ClusterManifestProvider.cs index 626d19a7c76..e6ff21e72fc 100644 --- a/src/Orleans.Runtime/Manifest/ClusterManifestProvider.cs +++ b/src/Orleans.Runtime/Manifest/ClusterManifestProvider.cs @@ -1,12 +1,16 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; using Orleans.Core.Diagnostics; using Orleans.Internal; using Orleans.Metadata; @@ -16,11 +20,17 @@ namespace Orleans.Runtime.Metadata { internal partial class ClusterManifestProvider : IClusterManifestProvider, IAsyncDisposable, IDisposable, ILifecycleParticipant { + private const int MaxConcurrentPeerManifestProbes = 3; + private static readonly TimeSpan PeerManifestProbeTimeout = TimeSpan.FromSeconds(1); private readonly SiloAddress _localSiloAddress; private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IClusterMembershipService _clusterMembershipService; private readonly IFatalErrorHandler _fatalErrorHandler; + private readonly TimeProvider _timeProvider; + private readonly ClusterManifestInstruments _instruments; + private readonly bool _enableContentAddressedRetrieval; + private readonly SemaphoreSlim? _peerProbeSlots; private readonly CancellationTokenSource _shutdownCts = new(); private readonly AsyncEnumerable _updates; #if NET9_0_OR_GREATER @@ -31,6 +41,8 @@ internal partial class ClusterManifestProvider : IClusterManifestProvider, IAsyn private ClusterManifest _current; private IInternalGrainFactory? _grainFactory; private Task? _runTask; + private int _peerProbeRound; + private ConcurrentDictionary? _manifestCache; public ClusterManifestProvider( ILocalSiloDetails localSiloDetails, @@ -38,21 +50,32 @@ public ClusterManifestProvider( IClusterMembershipService clusterMembershipService, IFatalErrorHandler fatalErrorHandler, ILogger logger, - IServiceProvider services) + IServiceProvider services, + TimeProvider timeProvider, + IOptions options, + ClusterManifestInstruments instruments) { _localSiloAddress = localSiloDetails.SiloAddress; _logger = logger; _services = services; _clusterMembershipService = clusterMembershipService; _fatalErrorHandler = fatalErrorHandler; + _timeProvider = timeProvider; + _instruments = instruments; + _enableContentAddressedRetrieval = options.Value.EnableContentAddressedRetrieval; LocalGrainManifest = siloManifestProvider.SiloManifest; + if (_enableContentAddressedRetrieval) + { + _peerProbeSlots = new(MaxConcurrentPeerManifestProbes, MaxConcurrentPeerManifestProbes); + } + _current = CreateClusterManifest( MajorMinorVersion.MinValue, ImmutableDictionary.Empty); _updates = new AsyncEnumerable( initialValue: _current, updateValidator: (previous, proposed) => proposed.Version > previous.Version, - onPublished: update => Interlocked.Exchange(ref _current, update)); + onPublished: OnManifestPublished); } public ClusterManifest Current => EnsureValidManifestForCurrentMembership(_clusterMembershipService.CurrentSnapshot); @@ -92,13 +115,12 @@ private ClusterManifest EnsureValidManifestForCurrentMembership(ClusterMembershi } } - private async Task ProcessMembershipUpdates() + private async Task ProcessMembershipUpdates(CancellationToken cancellationToken) { try { LogDebugStartingToProcessMembershipUpdates(); - var cancellationToken = _shutdownCts.Token; await using var membershipUpdates = _clusterMembershipService.MembershipUpdates.GetAsyncEnumerator(cancellationToken); var nextUpdateTask = membershipUpdates.MoveNextAsync().AsTask(); ClusterMembershipSnapshot? membershipSnapshot = null; @@ -116,7 +138,7 @@ private async Task ProcessMembershipUpdates() nextUpdateTask = membershipUpdates.MoveNextAsync().AsTask(); } - if (await UpdateManifest(membershipSnapshot)) + if (await UpdateManifest(membershipSnapshot, cancellationToken)) { membershipSnapshot = null; continue; @@ -140,7 +162,7 @@ private async Task ProcessMembershipUpdates() } } } - catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { // Ignore during shutdown. } @@ -154,8 +176,11 @@ private async Task ProcessMembershipUpdates() } } - private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembership) + private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembership, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + // An update overtaken by a newer publication keeps populating its original cache. + var cache = Volatile.Read(ref _manifestCache); var existingManifest = EnsureValidManifestForCurrentMembership(clusterMembership); if (existingManifest.Version.Major > clusterMembership.Version.Value) { @@ -166,7 +191,7 @@ private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembers var modified = false; // Fill missing entries. - var tasks = new List>(); + var missingSilos = new List(); foreach (var entry in clusterMembership.Members) { var member = entry.Value; @@ -183,16 +208,81 @@ private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembers continue; } - tasks.Add(GetManifest(siloAddress)); + missingSilos.Add(siloAddress); } - async Task<(SiloAddress Key, GrainManifest? Value, Exception? Exception)> GetManifest(SiloAddress siloAddress) + using var peerRepairCancellation = cache is not null && missingSilos.Count > 1 + ? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken) + : null; + var peerRepairTask = peerRepairCancellation is not null + ? TryFillMissingManifestsFromPeers(clusterMembership, builder, missingSilos, cache!, peerRepairCancellation.Token) + : Task.FromResult(false); + + var tasks = new Dictionary>(); + foreach (var siloAddress in missingSilos) { + tasks.Add(siloAddress, GetManifest(siloAddress, cancellationToken)); + } + + var directFetches = Task.WhenAll(tasks.Values); + var peerRepaired = false; + if (!peerRepairTask.IsCompleted + && await Task.WhenAny(peerRepairTask, directFetches) == directFetches + && (await directFetches).All(static result => result.Value is not null && result.Exception is null)) + { + peerRepairCancellation!.Cancel(); try { - // Get the manifest from the silo. - var remoteManifestProvider = _grainFactory!.GetSystemTarget(Constants.ManifestProviderType, siloAddress); - var manifest = await remoteManifestProvider.GetSiloManifest(_shutdownCts.Token); + peerRepaired = await peerRepairTask; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // Direct retrieval supplied every missing manifest. + } + } + else + { + peerRepaired = await peerRepairTask; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (peerRepaired) + { + modified = true; + var repairedSilos = builder.ToImmutable(); + var repairedManifest = CreateClusterManifest( + new MajorMinorVersion(clusterMembership.Version.Value, existingManifest.Version.Minor + 1), + repairedSilos); + if (!TryPublishManifest(repairedManifest)) + { + return false; + } + + _instruments.OnPeerRepair(missingSilos.Count(builder.ContainsKey)); + existingManifest = repairedManifest; + modified = false; + foreach (var siloAddress in missingSilos) + { + if (builder.ContainsKey(siloAddress)) + { + tasks.Remove(siloAddress); + } + } + + // Each fetch observes its own failures, so redundant requests can finish independently. + if (tasks.Count == 0) + { + return true; + } + } + + async Task<(SiloAddress Key, GrainManifest? Value, Exception? Exception)> GetManifest( + SiloAddress siloAddress, + CancellationToken requestCancellationToken) + { + try + { + var manifest = await GetSiloManifest(siloAddress, cache, requestCancellationToken); return (siloAddress, manifest, null); } catch (Exception exception) @@ -202,8 +292,9 @@ private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembers } var fetchSuccess = true; - await Task.WhenAll(tasks); - foreach (var task in tasks) + await Task.WhenAll(tasks.Values); + cancellationToken.ThrowIfCancellationRequested(); + foreach (var task in tasks.Values) { var result = await task; if (result.Exception is Exception exception) @@ -232,14 +323,216 @@ private async Task UpdateManifest(ClusterMembershipSnapshot clusterMembers var version = new MajorMinorVersion(clusterMembership.Version.Value, existingManifest.Version.Minor + 1); if (modified) { - var manifest = CreateClusterManifest(version, builder.ToImmutable()); + var silos = builder.ToImmutable(); + var manifest = CreateClusterManifest(version, silos); var publishSuccess = TryPublishManifest(manifest); return publishSuccess && fetchSuccess; } - return fetchSuccess; } + private void OnManifestPublished(ClusterManifest manifest) + { + if (!_enableContentAddressedRetrieval) + { + Interlocked.Exchange(ref _current, manifest); + return; + } + + // Each publication seeds a new read-through cache with its live manifests. Fetches populate + // their captured cache instance, keeping late completions isolated in their original epoch. + var cache = new ConcurrentDictionary(); + foreach (var grainManifest in manifest.AllGrainManifests) + { + cache[ManifestHashCalculator.ComputeHash(grainManifest)] = grainManifest; + } + + cache[ManifestHashCalculator.ComputeHash(LocalGrainManifest)] = LocalGrainManifest; + // Readers capture the cache before checking the manifest version, so expose the version first. + Interlocked.Exchange(ref _current, manifest); + Interlocked.Exchange(ref _manifestCache, cache); + } + + private async Task TryFillMissingManifestsFromPeers( + ClusterMembershipSnapshot clusterMembership, + ImmutableDictionary.Builder builder, + List missingSilos, + ConcurrentDictionary cache, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var missing = new HashSet(missingSilos); + var modified = false; + var peers = clusterMembership.Members.Values + .Where(static member => member.Status == SiloStatus.Active) + .Select(static member => member.SiloAddress) + .Where(peer => !peer.Equals(_localSiloAddress)) + .OrderBy(static silo => silo) + .ToArray(); + if (peers.Length == 0) + { + return false; + } + + var round = Interlocked.Increment(ref _peerProbeRound); + var start = (int)((uint)(_localSiloAddress.GetConsistentHashCode() + round) % (uint)peers.Length); + var probeCount = Math.Min(MaxConcurrentPeerManifestProbes, peers.Length); + var probes = new Task[probeCount]; + for (var i = 0; i < probeCount; i++) + { + probes[i] = ProbePeerForManifests(peers[(start + i) % peers.Length], missingSilos, cache, cancellationToken); + } + + var results = await Task.WhenAll(probes); + cancellationToken.ThrowIfCancellationRequested(); + foreach (var result in results) + { + if (result is null || missing.Count == 0) + { + continue; + } + + FillFromCachedHashes(result.Summary, missing, builder, cache, ref modified); + if (result.Update?.SiloManifests is { } manifests) + { + foreach (var silo in missing.ToArray()) + { + if (!result.Summary.SiloManifestHashes.TryGetValue(silo, out var expectedHash) + || !manifests.TryGetValue(silo, out var manifest) + || ManifestHashCalculator.ComputeHash(manifest) != expectedHash) + { + continue; + } + + cache[expectedHash] = manifest; + builder[silo] = manifest; + missing.Remove(silo); + modified = true; + } + } + } + + return modified; + } + + private async Task ProbePeerForManifests( + SiloAddress peer, + IReadOnlyCollection missingSilos, + ConcurrentDictionary cache, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var slots = _peerProbeSlots!; + if (!slots.Wait(0, cancellationToken)) + { + _instruments.OnPeerProbe("skipped"); + LogDebugClusterManifestPeerProbeAtCapacity(peer); + return null; + } + + using var timeoutCancellation = new CancellationTokenSource(PeerManifestProbeTimeout, _timeProvider); + using var probeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellation.Token); + var probeToken = probeCancellation.Token; + var status = "error"; + Task? probeTask = null; + try + { + var remoteManifestProvider = _grainFactory!.GetSystemTarget(Constants.ManifestProviderType, peer); + var summaryTask = remoteManifestProvider.GetClusterManifestHashSummary(probeToken).AsTask(); + probeTask = summaryTask; + var summary = await summaryTask + .WaitAsync(probeToken); + probeTask = null; + probeToken.ThrowIfCancellationRequested(); + if (missingSilos.All(silo => + summary.SiloManifestHashes.TryGetValue(silo, out var hash) + && cache.ContainsKey(hash))) + { + status = "success"; + return new(summary, Update: null); + } + + probeToken.ThrowIfCancellationRequested(); + + // No per-peer manifest body is retained, so request a complete update instead of synthesizing a + // baseline from the local provider's version. + var updateTask = remoteManifestProvider.GetClusterManifestUpdate(MajorMinorVersion.MinValue, probeToken).AsTask(); + probeTask = updateTask; + var update = await updateTask + .WaitAsync(probeToken); + probeTask = null; + probeToken.ThrowIfCancellationRequested(); + status = "success"; + return new(summary, update); + } + catch (TimeoutException) + { + status = "timeout"; + LogDebugClusterManifestPeerProbeTimedOut(peer, PeerManifestProbeTimeout); + return null; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + status = "canceled"; + throw; + } + catch (OperationCanceledException) when (timeoutCancellation.IsCancellationRequested) + { + status = "timeout"; + LogDebugClusterManifestPeerProbeTimedOut(peer, PeerManifestProbeTimeout); + return null; + } + catch (Exception exception) + { + LogDebugErrorRetrievingClusterManifestFromPeer(exception, peer); + return null; + } + finally + { + slots.Release(); + _instruments.OnPeerProbe(status); + if (probeTask is { IsCompleted: false }) + { + probeTask.ContinueWith( + task => LogDebugLateClusterManifestPeerProbeFailure(task.Exception!, peer), + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default).Ignore(); + } + else + { + _ = probeTask?.Exception; + } + } + } + + private sealed record PeerManifestProbeResult( + ClusterManifestHashSummary Summary, + ClusterManifestUpdate? Update); + + private void FillFromCachedHashes( + ClusterManifestHashSummary summary, + HashSet missing, + ImmutableDictionary.Builder builder, + ConcurrentDictionary cache, + ref bool modified) + { + foreach (var silo in missing.ToArray()) + { + if (summary.SiloManifestHashes.TryGetValue(silo, out var hash)) + { + var hit = cache.TryGetValue(hash, out var cached); + _instruments.OnCacheLookup(hit, "peer"); + if (hit) + { + builder[silo] = cached!; + missing.Remove(silo); + modified = true; + } + } + } + } + private ClusterManifest CreateClusterManifest( MajorMinorVersion version, ImmutableDictionary silos) @@ -277,17 +570,111 @@ private static ImmutableDictionary RemoveNonActiveSi return builder?.ToImmutable() ?? silos; } + private Task GetSiloManifest( + SiloAddress siloAddress, + ConcurrentDictionary? cache, + CancellationToken cancellationToken) => + _instruments.RetrievalDurationEnabled + ? GetSiloManifestWithMetrics(siloAddress, cache, cancellationToken) + : GetSiloManifestCore(siloAddress, cache, cancellationToken); + + private async Task GetSiloManifestWithMetrics( + SiloAddress siloAddress, + ConcurrentDictionary? cache, + CancellationToken cancellationToken) + { + var started = _timeProvider.GetTimestamp(); + var status = "error"; + try + { + var manifest = await GetSiloManifestCore(siloAddress, cache, cancellationToken); + status = "success"; + return manifest; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + status = "canceled"; + throw; + } + finally + { + _instruments.OnRetrievalCompleted(_timeProvider.GetElapsedTime(started), cache is null ? "direct" : "hash", status); + } + } + + private async Task GetSiloManifestCore( + SiloAddress siloAddress, + ConcurrentDictionary? cache, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (cache is not null) + { + try + { + var remoteManifestProvider = _grainFactory!.GetSystemTarget(Constants.ManifestProviderType, siloAddress); + var hash = await AwaitManifestRequest(remoteManifestProvider.GetSiloManifestHash(cancellationToken).AsTask(), cancellationToken); + var hit = cache.TryGetValue(hash, out var cached); + _instruments.OnCacheLookup(hit, "silo"); + if (hit) + { + return cached!; + } + + var manifest = await AwaitManifestRequest(remoteManifestProvider.GetSiloManifestByHash(hash, cancellationToken).AsTask(), cancellationToken); + if (manifest is not null && ManifestHashCalculator.ComputeHash(manifest) == hash) + { + cache[hash] = manifest; + return manifest; + } + + _instruments.OnFallback(manifest is null ? "missing" : "mismatch"); + } + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + _instruments.OnFallback("error"); + LogDebugErrorRetrievingSiloManifestByHash(exception, siloAddress); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + var legacyManifestProvider = _grainFactory!.GetSystemTarget(Constants.ManifestProviderType, siloAddress); + var legacyManifest = await AwaitManifestRequest(legacyManifestProvider.GetSiloManifest(cancellationToken).AsTask(), cancellationToken); + if (cache is not null) + { + cache[ManifestHashCalculator.ComputeHash(legacyManifest)] = legacyManifest; + } + + return legacyManifest; + } + + private async Task AwaitManifestRequest(Task request, CancellationToken cancellationToken) + { + try + { + return await request.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + request.Ignore(); + throw; + } + } + [MemberNotNull(nameof(_runTask))] private Task StartAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); Debug.Assert(_grainFactory is not null); - _runTask = Task.Run(ProcessMembershipUpdates, CancellationToken.None); + var shutdownToken = _shutdownCts.Token; + _runTask = Task.Run(() => ProcessMembershipUpdates(shutdownToken), shutdownToken); return Task.CompletedTask; } [MemberNotNull(nameof(_grainFactory))] private Task Initialize(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); _grainFactory = _services.GetRequiredService(); return Task.CompletedTask; } @@ -318,15 +705,7 @@ public void Participate(ISiloLifecycle lifecycle) static Task NoOpStop(CancellationToken _) => Task.CompletedTask; } - public async ValueTask DisposeAsync() - { - if (_shutdownCts.IsCancellationRequested) - { - return; - } - - await StopAsync(CancellationToken.None); - } + public ValueTask DisposeAsync() => new(StopAsync(CancellationToken.None)); public void Dispose() { @@ -339,6 +718,36 @@ public void Dispose() )] private partial void LogWarningErrorRetrievingSiloManifest(Exception exception, SiloAddress siloAddress); + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Error retrieving silo manifest by hash for silo {SiloAddress}. Falling back to direct manifest fetch." + )] + private partial void LogDebugErrorRetrievingSiloManifestByHash(Exception exception, SiloAddress siloAddress); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Error retrieving cluster manifest from peer {SiloAddress}. Falling back to direct manifest fetch." + )] + private partial void LogDebugErrorRetrievingClusterManifestFromPeer(Exception exception, SiloAddress siloAddress); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Cluster manifest peer probe to {SiloAddress} exceeded {Timeout}. Direct manifest fetch continues." + )] + private partial void LogDebugClusterManifestPeerProbeTimedOut(SiloAddress siloAddress, TimeSpan timeout); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Skipping cluster manifest peer probe for {SiloAddress} while three local probe attempts are active. Direct manifest fetch continues." + )] + private partial void LogDebugClusterManifestPeerProbeAtCapacity(SiloAddress siloAddress); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Cluster manifest peer probe task for {SiloAddress} faulted after the caller stopped waiting." + )] + private partial void LogDebugLateClusterManifestPeerProbeFailure(Exception exception, SiloAddress siloAddress); + [LoggerMessage( Level = LogLevel.Debug, Message = "Starting to process membership updates" diff --git a/src/Orleans.Runtime/Manifest/ManifestHashCalculator.cs b/src/Orleans.Runtime/Manifest/ManifestHashCalculator.cs new file mode 100644 index 00000000000..97388df5216 --- /dev/null +++ b/src/Orleans.Runtime/Manifest/ManifestHashCalculator.cs @@ -0,0 +1,83 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using Orleans.Metadata; + +namespace Orleans.Runtime.Metadata; + +internal static class ManifestHashCalculator +{ + private const int EncodingVersion = 2; + private static readonly ConditionalWeakTable> Hashes = new(); + + public static ManifestHash ComputeHash(GrainManifest manifest) => + Hashes.GetValue(manifest, static value => new(ComputeHashCore(value))).Value; + + private static ManifestHash ComputeHashCore(GrainManifest manifest) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + AppendInt32(hash, EncodingVersion); + + // Fixed section/field order, collection counts, and value lengths delimit the canonical input. + AppendInt32(hash, manifest.Grains.Count); + foreach (var grain in manifest.Grains.OrderBy(static entry => entry.Key)) + { + AppendIdentifier(hash, GrainType.UnsafeGetArray(grain.Key)); + AppendProperties(hash, grain.Value.Properties); + } + + AppendInt32(hash, manifest.Interfaces.Count); + foreach (var grainInterface in manifest.Interfaces.OrderBy(static entry => entry.Key.Value)) + { + AppendIdentifier(hash, IdSpan.UnsafeGetArray(grainInterface.Key.Value)); + AppendProperties(hash, grainInterface.Value.Properties); + } + + return new ManifestHash(Convert.ToHexString(hash.GetHashAndReset())); + } + + private static void AppendProperties(IncrementalHash hash, ImmutableDictionary properties) + { + AppendInt32(hash, properties.Count); + foreach (var property in properties.OrderBy(static entry => entry.Key, StringComparer.Ordinal)) + { + AppendString(hash, property.Key); + AppendString(hash, property.Value); + } + } + + private static void AppendString(IncrementalHash hash, string? value) + { + AppendInt32(hash, value?.Length ?? -1); + if (value is null) + { + return; + } + + Span codeUnit = stackalloc byte[sizeof(char)]; + foreach (var character in value) + { + BinaryPrimitives.WriteUInt16BigEndian(codeUnit, character); + hash.AppendData(codeUnit); + } + } + + private static void AppendIdentifier(IncrementalHash hash, byte[]? value) + { + AppendInt32(hash, value?.Length ?? -1); + if (value is not null) + { + hash.AppendData(value); + } + } + + private static void AppendInt32(IncrementalHash hash, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32BigEndian(bytes, value); + hash.AppendData(bytes); + } +} diff --git a/src/api/Orleans.Runtime/Orleans.Runtime.cs b/src/api/Orleans.Runtime/Orleans.Runtime.cs index 8023cf6f362..54ec16ebc1c 100644 --- a/src/api/Orleans.Runtime/Orleans.Runtime.cs +++ b/src/api/Orleans.Runtime/Orleans.Runtime.cs @@ -107,6 +107,11 @@ public sealed partial class ActivationRepartitionerOptions public System.TimeSpan RecoveryPeriod { get { throw null; } set { } } } + public sealed partial class ClusterManifestOptions + { + public bool EnableContentAddressedRetrieval { get { throw null; } set { } } + } + public partial class ConsistentRingOptions { public const int DEFAULT_NUM_VIRTUAL_RING_BUCKETS = 30; diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.DirectRetrieval.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.DirectRetrieval.cs new file mode 100644 index 00000000000..92b23f616b8 --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.DirectRetrieval.cs @@ -0,0 +1,513 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using TestExtensions; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + private static readonly FieldInfo MemoizedHashesField = typeof(ManifestHashCalculator) + .GetField("Hashes", BindingFlags.Static | BindingFlags.NonPublic)!; + + [Fact] + public async Task ClientProvider_UpdateCancellation_DoesNotFetchLegacyManifest() + { + var provider = (ClientClusterManifestProvider)RuntimeHelpers.GetUninitializedObject( + typeof(ClientClusterManifestProvider)); + var remoteProvider = Substitute.For(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.Cancel(); + remoteProvider + .GetClusterManifestUpdate(default, cancellation.Token) + .Returns(_ => new ValueTask( + Task.FromCanceled(cancellation.Token))); + var method = typeof(ClientClusterManifestProvider).GetMethod( + "GetClusterManifestUpdate", + BindingFlags.Instance | BindingFlags.NonPublic)!; + var task = (Task)method.Invoke( + provider, + [remoteProvider, default(MajorMinorVersion), cancellation.Token])!; + + await Assert.ThrowsAnyAsync(() => task); + + var call = Assert.Single(remoteProvider.ReceivedCalls()); + Assert.Equal(nameof(IClusterManifestSystemTarget.GetClusterManifestUpdate), call.GetMethodInfo().Name); + } + + [Fact] + public void ManifestRpcTargetsHonorPreCanceledTokens() + { + var target = (ClusterManifestSystemTarget)RuntimeHelpers.GetUninitializedObject(typeof(ClusterManifestSystemTarget)); + var cancellationToken = new CancellationToken(canceled: true); + + Assert.Throws(() => { _ = target.GetClusterManifestHashSummary(cancellationToken); }); + Assert.Throws(() => { _ = target.GetSiloManifestHash(cancellationToken); }); + Assert.Throws(() => { _ = target.GetSiloManifestByHash(default, cancellationToken); }); + } + + [Fact] + public async Task DefaultOptions_ReuseContentHashesAndCaptureStartupSetting() + { + var cancellationToken = TestContext.Current.CancellationToken; + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo])); + var grainFactory = Substitute.For(); + var options = new ClusterManifestOptions(); + Assert.True(options.EnableContentAddressedRetrieval); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, options); + var hash = ManifestHashCalculator.ComputeHash(provider.LocalGrainManifest); + var remote = new CanonicalManifestCacheTarget(hash, provider.LocalGrainManifest); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(remote); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(remote); + Assert.Same(provider.LocalGrainManifest, Assert.Single(GetCachedManifests(provider)).Value); + + options.EnableContentAddressedRetrieval = false; + await InitializeProviderAsync(provider, cancellationToken); + Assert.True(await UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken)); + + Assert.Same(provider.LocalGrainManifest, provider.Current.Silos[remoteSilo]); + Assert.Equal(1, remote.HashRequests); + Assert.Equal(0, remote.ManifestByHashRequests); + Assert.Equal(0, remote.LegacyManifestRequests); + } + + [Fact] + public async Task DisabledOptions_RetrieveDirectlyWithoutHashesOrPeerRepair() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + var snapshot = CreateActiveMembershipSnapshot(1, localSilo, peers); + using var membership = new TestClusterMembershipService(snapshot); + var grainFactory = Substitute.For(); + var remoteManifest = CreateGrainManifest(); + var legacy = Substitute.For(); + legacy.GetSiloManifest(Arg.Any()).Returns(new ValueTask(remoteManifest)); + foreach (var peer in peers) + { + grainFactory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(legacy); + } + + var options = new ClusterManifestOptions { EnableContentAddressedRetrieval = false }; + Assert.False(options.EnableContentAddressedRetrieval); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, options); + var memoizedHashes = GetMemoizedManifestHashes(); + + Assert.False(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out _)); + Assert.Null(GetCachedManifests(provider)); + + // Runtime settings are captured at construction, even if the options object is later changed. + options.EnableContentAddressedRetrieval = true; + await InitializeProviderAsync(provider, TestContext.Current.CancellationToken); + Assert.True(await UpdateManifestAsync(provider, snapshot, TestContext.Current.CancellationToken)); + + Assert.Equal(new MajorMinorVersion(1, 1), provider.Current.Version); + Assert.All(peers, peer => Assert.Same(remoteManifest, provider.Current.Silos[peer])); + Assert.Equal(2, legacy.ReceivedCalls().Count()); + Assert.All(legacy.ReceivedCalls(), call => Assert.Equal(TestContext.Current.CancellationToken, call.GetArguments()[0])); + Assert.DoesNotContain(grainFactory.ReceivedCalls(), call => + call.GetMethodInfo().GetGenericArguments().Contains(typeof(IClusterManifestSystemTarget))); + Assert.Null(GetCachedManifests(provider)); + Assert.False(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out _)); + Assert.False(memoizedHashes.TryGetValue(remoteManifest, out _)); + Assert.Equal(0, typeof(ClusterManifestProvider).GetField("_peerProbeRound", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(provider)); + + membership.Update(CreateMembershipSnapshot(2, (localSilo, SiloStatus.Active))); + Assert.Equal(new MajorMinorVersion(2, 0), provider.Current.Version); + Assert.Equal(localSilo, Assert.Single(provider.Current.Silos).Key); + Assert.Null(GetCachedManifests(provider)); + Assert.False(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out _)); + } + + [Fact] + public async Task DisabledOptions_CallerCancellationStopsDirectFetch() + { + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + var snapshot = CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo]); + using var membership = new TestClusterMembershipService(snapshot); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var pending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var remote = Substitute.For(); + remote.GetSiloManifest(Arg.Any()).Returns(call => + { + entered.SetResult(call.Arg()); + return new ValueTask(pending.Task); + }); + var grainFactory = Substitute.For(); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(remote); + await using var provider = CreateClusterManifestProvider( + localSilo, membership, grainFactory, new ClusterManifestOptions { EnableContentAddressedRetrieval = false }); + await InitializeProviderAsync(provider, cancellation.Token); + var update = UpdateManifestAsync(provider, snapshot, cancellation.Token); + try + { + Assert.Equal(cancellation.Token, await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => update.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.DoesNotContain(remoteSilo, provider.Current.Silos.Keys); + Assert.Null(GetCachedManifests(provider)); + } + finally + { + pending.TrySetResult(CreateGrainManifest()); + } + } + + [Fact] + public async Task DisabledServer_ComputesHashesOnDemandAndCachesSummaryByVersion() + { + var localSilo = CreateSiloAddress(11111, 1); + using var membership = new TestClusterMembershipService(CreateMembershipSnapshot(1, (localSilo, SiloStatus.Active))); + var grainFactory = Substitute.For(); + await using var provider = CreateClusterManifestProvider( + localSilo, membership, grainFactory, new ClusterManifestOptions { EnableContentAddressedRetrieval = false }); + using var services = new ServiceCollection() + .AddMetrics() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .BuildServiceProvider(); + var localSiloDetails = Substitute.For(); + localSiloDetails.SiloAddress.Returns(localSilo); + var shared = new SystemTargetShared( + runtimeClient: null!, + localSiloDetails, + NullLoggerFactory.Instance, + Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + timerRegistry: null!, + new ActivationDirectory(services.GetRequiredService()), + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService()); + using var target = new ClusterManifestSystemTarget(membership, provider, shared); + var memoizedHashes = GetMemoizedManifestHashes(); + + Assert.False(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out _)); + Assert.Same(provider.LocalGrainManifest, await target.GetSiloManifest(TestContext.Current.CancellationToken)); + Assert.Same(provider.Current, await target.GetClusterManifest(TestContext.Current.CancellationToken)); + Assert.False(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out _)); + + var hash = await target.GetSiloManifestHash(TestContext.Current.CancellationToken); + Assert.True(memoizedHashes.TryGetValue(provider.LocalGrainManifest, out var memoized)); + Assert.Equal(memoized.Value, hash); + Assert.Same(provider.LocalGrainManifest, await target.GetSiloManifestByHash(hash, TestContext.Current.CancellationToken)); + Assert.Null(await target.GetSiloManifestByHash(new ManifestHash("mismatch"), TestContext.Current.CancellationToken)); + var summary = await target.GetClusterManifestHashSummary(TestContext.Current.CancellationToken); + Assert.Equal(new MajorMinorVersion(1, 0), summary.Version); + Assert.Equal(hash, Assert.Single(summary.SiloManifestHashes).Value); + Assert.Same(summary, await target.GetClusterManifestHashSummary(TestContext.Current.CancellationToken)); + + membership.Update(CreateMembershipSnapshot(2, (localSilo, SiloStatus.ShuttingDown))); + var updated = await target.GetClusterManifestHashSummary(TestContext.Current.CancellationToken); + Assert.NotSame(summary, updated); + Assert.Equal(new MajorMinorVersion(2, 0), updated.Version); + Assert.Empty(updated.SiloManifestHashes); + Assert.Null(GetCachedManifests(provider)); + } + + [Theory] + [InlineData("hash")] + [InlineData("body")] + [InlineData("legacy")] + public async Task DirectFetch_CallerCancellation_ReachesRpcAndStopsWaiting(string phase) + { + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + var snapshot = CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo]); + using var membership = new TestClusterMembershipService(snapshot); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingHash = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingLegacy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var remoteManifest = CreateGrainManifest(); + var hash = ManifestHashCalculator.ComputeHash(remoteManifest); + var remote = Substitute.For(); + remote.GetSiloManifestHash(Arg.Any()).Returns(call => + { + if (phase == "hash") + { + entered.TrySetResult(call.Arg()); + return new ValueTask(pendingHash.Task); + } + + return phase == "legacy" ? ValueTask.FromException(new NotSupportedException()) : new ValueTask(hash); + }); + remote.GetSiloManifestByHash(hash, Arg.Any()).Returns(call => + { + entered.TrySetResult(call.Arg()); + return new ValueTask(pendingBody.Task); + }); + var legacy = Substitute.For(); + legacy.GetSiloManifest(Arg.Any()).Returns(call => + { + entered.TrySetResult(call.Arg()); + return new ValueTask(pendingLegacy.Task); + }); + var grainFactory = Substitute.For(); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(remote); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(legacy); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); + await InitializeProviderAsync(provider, cancellation.Token); + var update = UpdateManifestAsync(provider, snapshot, cancellation.Token); + try + { + var rpcToken = await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(cancellation.Token, rpcToken); + cancellation.Cancel(); + + Assert.True(rpcToken.IsCancellationRequested); + await Assert.ThrowsAnyAsync(() => update.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.DoesNotContain(remoteSilo, provider.Current.Silos.Keys); + if (phase != "legacy") + { + Assert.Empty(legacy.ReceivedCalls()); + } + } + finally + { + cancellation.Cancel(); + pendingHash.TrySetResult(hash); + pendingBody.TrySetResult(remoteManifest); + pendingLegacy.TrySetResult(remoteManifest); + } + } + + [Fact] + public async Task UpdateManifest_ReusesVerifiedManifestBeforePublication() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1), CreateSiloAddress(11114, 1) }; + var snapshot = CreateActiveMembershipSnapshot(1, localSilo, peers); + using var membership = new TestClusterMembershipService(snapshot); + var grainFactory = Substitute.For(); + var remoteManifest = CreateGrainManifest(); + var remoteHash = ManifestHashCalculator.ComputeHash(remoteManifest); + var firstTarget = new CanonicalManifestCacheTarget(remoteHash, remoteManifest); + var secondTarget = new CanonicalManifestCacheTarget(remoteHash, remoteManifest); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, peers[0]) + .Returns(firstTarget); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, peers[1]) + .Returns(secondTarget); + var pendingHash = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowTarget = Substitute.For(); + slowTarget.GetClusterManifestHashSummary(Arg.Any()).Returns(ValueTask.FromException(new NotSupportedException())); + slowTarget.GetSiloManifestHash(Arg.Any()).Returns(new ValueTask(pendingHash.Task)); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, peers[2]) + .Returns(slowTarget); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); + var localHash = ManifestHashCalculator.ComputeHash(provider.LocalGrainManifest); + Assert.Same(provider.LocalGrainManifest, Assert.Single(GetCachedManifests(provider)).Value); + Assert.NotEqual(localHash, remoteHash); + var initialManifest = provider.Current; + var cache = GetCachedManifests(provider); + await InitializeProviderAsync(provider, TestContext.Current.CancellationToken); + var update = UpdateManifestAsync(provider, snapshot, TestContext.Current.CancellationToken); + + Assert.False(update.IsCompleted); + Assert.Same(initialManifest, provider.Current); + Assert.Same(cache, GetCachedManifests(provider)); + Assert.Same(remoteManifest, cache[remoteHash]); + Assert.Equal(1, firstTarget.HashRequests); + Assert.Equal(1, secondTarget.HashRequests); + Assert.Equal(1, firstTarget.ManifestByHashRequests + secondTarget.ManifestByHashRequests); + Assert.Equal(0, firstTarget.LegacyManifestRequests + secondTarget.LegacyManifestRequests); + + pendingHash.SetResult(localHash); + Assert.True(await update.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.NotSame(cache, GetCachedManifests(provider)); + Assert.Equal(new MajorMinorVersion(1, 1), provider.Current.Version); + Assert.Equal(remoteManifest, provider.Current.Silos[peers[0]]); + Assert.Equal(remoteManifest, provider.Current.Silos[peers[1]]); + } + + [Fact] + public async Task UpdateManifest_StaleFetchCannotPopulateCurrentCache() + { + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + var snapshot = CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo]); + using var membership = new TestClusterMembershipService(snapshot); + var grainFactory = Substitute.For(); + var remoteManifest = CreateGrainManifest(); + var remoteTarget = Substitute.For(); + var pendingManifest = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fetchStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hash = ManifestHashCalculator.ComputeHash(remoteManifest); + remoteTarget.GetSiloManifestHash(Arg.Any()).Returns(new ValueTask(hash)); + remoteTarget.GetSiloManifestByHash(hash, Arg.Any()).Returns(_ => + { + fetchStarted.TrySetResult(); + return new ValueTask(pendingManifest.Task); + }); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo) + .Returns(remoteTarget); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); + Assert.Equal(new MajorMinorVersion(1, 0), provider.Current.Version); + await InitializeProviderAsync(provider, TestContext.Current.CancellationToken); + var update = UpdateManifestAsync(provider, snapshot, TestContext.Current.CancellationToken); + await fetchStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var fetchCache = GetCachedManifests(provider); + + membership.Update(CreateMembershipSnapshot(2, (localSilo, SiloStatus.Active))); + Assert.Equal(new MajorMinorVersion(2, 0), provider.Current.Version); + pendingManifest.SetResult(remoteManifest); + + Assert.False(await update.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.DoesNotContain(remoteSilo, provider.Current.Silos.Keys); + Assert.Same(remoteManifest, fetchCache[hash]); + Assert.NotSame(fetchCache, GetCachedManifests(provider)); + Assert.Same(provider.LocalGrainManifest, Assert.Single(GetCachedManifests(provider)).Value); + } + + [Fact] + public async Task DirectFetch_CanceledHashRequest_UsesLegacyManifest() + { + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + var remoteManifest = CreateGrainManifest(); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo])); + var grainFactory = CreateGrainFactory(remoteSilo, remoteManifest); + var remoteTarget = Substitute.For(); + remoteTarget.GetSiloManifestHash(Arg.Any()).Returns(new ValueTask( + Task.FromCanceled(new CancellationToken(canceled: true)))); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo) + .Returns(remoteTarget); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); + var observed = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + try + { + var current = await observed.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(remoteManifest, current.Silos[remoteSilo]); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + } + + [Fact] + [TestCategory("BVT")] + public async Task ClusterManifestProviderReusesCacheForCanonicalManifestHash() + { + var localSilo = CreateSiloAddress(11201, 1); + var remoteSilo = CreateSiloAddress(11202, 1); + using var membership = new TestClusterMembershipService(CreateMembershipSnapshot( + 1, + (localSilo, SiloStatus.Active))); + var grainFactory = Substitute.For(); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); + var canonicalHash = ManifestHashCalculator.ComputeHash(provider.LocalGrainManifest); + var remoteTarget = new CanonicalManifestCacheTarget(canonicalHash, provider.LocalGrainManifest); + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, remoteSilo) + .Returns(remoteTarget); + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, remoteSilo) + .Returns(remoteTarget); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + var observed = ObserveManifestAsync(provider, new MajorMinorVersion(2, 1), TestContext.Current.CancellationToken); + membership.Update(CreateMembershipSnapshot( + 2, + (localSilo, SiloStatus.Active), + (remoteSilo, SiloStatus.Active))); + + var current = await observed.WaitAsync( + TimeSpan.FromSeconds(10), + TestContext.Current.CancellationToken); + + Assert.Same(provider.LocalGrainManifest, current.Silos[remoteSilo]); + Assert.Equal(canonicalHash, ManifestHashCalculator.ComputeHash(current.Silos[remoteSilo])); + Assert.Equal(1, remoteTarget.HashRequests); + Assert.Equal(0, remoteTarget.ManifestByHashRequests); + Assert.Equal(0, remoteTarget.LegacyManifestRequests); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + } + } + + private static ConditionalWeakTable> GetMemoizedManifestHashes() => + (ConditionalWeakTable>)MemoizedHashesField.GetValue(null)!; + + private sealed class CanonicalManifestCacheTarget(ManifestHash hash, GrainManifest fallbackManifest) + : IClusterManifestSystemTarget, ISiloManifestSystemTarget + { + private int _hashRequests; + private int _manifestByHashRequests; + private int _legacyManifestRequests; + + public int HashRequests => Volatile.Read(ref _hashRequests); + + public int ManifestByHashRequests => Volatile.Read(ref _manifestByHashRequests); + + public int LegacyManifestRequests => Volatile.Read(ref _legacyManifestRequests); + + public ValueTask GetClusterManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromException(new NotSupportedException()); + } + + public ValueTask GetClusterManifestUpdate( + MajorMinorVersion previousVersion, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromException(new NotSupportedException()); + } + + public ValueTask GetClusterManifestHashSummary(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromException(new NotSupportedException()); + } + + public ValueTask GetSiloManifestHash(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _hashRequests); + return new(hash); + } + + public ValueTask GetSiloManifestByHash(ManifestHash requestedHash, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _manifestByHashRequests); + return new(fallbackManifest); + } + + public ValueTask GetSiloManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _legacyManifestRequests); + return new(fallbackManifest); + } + } +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Metrics.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Metrics.cs new file mode 100644 index 00000000000..cb161acd370 --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Metrics.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + [Theory] + [InlineData("error")] + [InlineData("mismatch")] + public async Task FallbackMetrics_DistinguishRpcErrorsAndMismatchedContent(string reason) + { + var cancellationToken = TestContext.Current.CancellationToken; + using var metrics = new ManifestMetrics(); + var local = CreateSiloAddress(11111, 1); + var peer = CreateSiloAddress(11112, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, [peer])); + var remoteManifest = CreateGrainManifest(); + var advertised = new ManifestHash("mismatched-content"); + var remote = Substitute.For(); + remote.GetSiloManifestHash(Arg.Any()).Returns(reason == "error" + ? ValueTask.FromException(new NotSupportedException()) + : new ValueTask(advertised)); + remote.GetSiloManifestByHash(advertised, Arg.Any()).Returns(new ValueTask(remoteManifest)); + var factory = CreateGrainFactory(peer, remoteManifest); + factory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(remote); + await using var provider = CreateClusterManifestProvider( + local, membership, factory, new FakeTimeProvider(), NullLogger.Instance, metrics.Instruments); + await InitializeProviderAsync(provider, cancellationToken); + + Assert.True(await UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken)); + Assert.Equal(remoteManifest, provider.Current.Silos[peer]); + var fallback = Assert.Single(metrics.Find(InstrumentNames.MANIFEST_FALLBACKS)); + Assert.Equal(1, fallback.Value); + Assert.Equal(reason, Assert.Single(fallback.Tags).Value); + Assert.Equal("success", Assert.Single(metrics.Find(InstrumentNames.MANIFEST_RETRIEVAL_DURATION)).Tags["status"]); + Assert.False(GetCachedManifests(provider).ContainsKey(advertised)); + } + + [Fact] + public async Task RetrievalMetrics_RecordCacheHitsMissesFallbackAndDuration() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var metrics = new ManifestMetrics(); + var time = new FakeTimeProvider(); + var local = CreateSiloAddress(11111, 1); + var first = CreateSiloAddress(11112, 1); + var second = CreateSiloAddress(11113, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, [first])); + var remoteManifest = CreateGrainManifest(); + var hash = ManifestHashCalculator.ComputeHash(remoteManifest); + var hashTarget = Substitute.For(); + hashTarget.GetSiloManifestHash(Arg.Any()).Returns(new ValueTask(hash)); + hashTarget.GetSiloManifestByHash(hash, Arg.Any()).Returns(new ValueTask((GrainManifest?)null)); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var directTarget = Substitute.For(); + directTarget.GetSiloManifest(Arg.Any()).Returns(_ => + { + entered.TrySetResult(); + return new ValueTask(release.Task); + }); + var factory = Substitute.For(); + foreach (var peer in new[] { first, second }) + { + factory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(hashTarget); + factory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(directTarget); + } + + await using var provider = CreateClusterManifestProvider(local, membership, factory, time, NullLogger.Instance, metrics.Instruments); + await InitializeProviderAsync(provider, cancellationToken); + try + { + var update = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + time.Advance(TimeSpan.FromMilliseconds(25)); + release.SetResult(remoteManifest); + Assert.True(await update.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + + membership.Update(CreateActiveMembershipSnapshot(2, local, [first, second])); + Assert.True(await UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken)); + Assert.Single(directTarget.ReceivedCalls()); + Assert.Equal(1, metrics.Sum(InstrumentNames.MANIFEST_CACHE_LOOKUPS, ("result", "miss"), ("source", "silo"))); + Assert.Equal(1, metrics.Sum(InstrumentNames.MANIFEST_CACHE_LOOKUPS, ("result", "hit"), ("source", "silo"))); + Assert.Equal(1, metrics.Sum(InstrumentNames.MANIFEST_FALLBACKS, ("reason", "missing"))); + var durations = metrics.Find(InstrumentNames.MANIFEST_RETRIEVAL_DURATION); + Assert.Equal(new[] { 25d, 0d }, durations.Select(measurement => measurement.Value)); + Assert.All(durations, measurement => + { + Assert.Equal("ms", measurement.Unit); + Assert.Equal("hash", measurement.Tags["mode"]); + Assert.Equal("success", measurement.Tags["status"]); + Assert.Equal(2, measurement.Tags.Count); + }); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_PEER_PROBES)); + } + finally + { + release.TrySetResult(remoteManifest); + } + } + + [Theory] + [InlineData("success")] + [InlineData("error")] + [InlineData("canceled")] + public async Task RetrievalMetrics_RecordDirectOutcomeWithoutHashActivity(string outcome) + { + var cancellationToken = TestContext.Current.CancellationToken; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var metrics = new ManifestMetrics(); + var time = new FakeTimeProvider(); + var local = CreateSiloAddress(11111, 1); + var peer = CreateSiloAddress(11112, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, [peer])); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = Substitute.For(); + target.GetSiloManifest(Arg.Any()).Returns(_ => + { + entered.TrySetResult(); + return new ValueTask(release.Task); + }); + var factory = Substitute.For(); + factory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(target); + await using var provider = CreateClusterManifestProvider( + local, membership, factory, time, NullLogger.Instance, + metrics.Instruments, new ClusterManifestOptions { EnableContentAddressedRetrieval = false }); + await InitializeProviderAsync(provider, cancellationToken); + var update = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellation.Token); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + time.Advance(TimeSpan.FromMilliseconds(12)); + switch (outcome) + { + case "success": + release.SetResult(CreateGrainManifest()); + Assert.True(await update); + break; + case "error": + release.SetException(new InvalidOperationException("Direct retrieval failed.")); + Assert.False(await update); + break; + case "canceled": + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => update.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + break; + } + + var duration = Assert.Single(metrics.Find(InstrumentNames.MANIFEST_RETRIEVAL_DURATION)); + Assert.Equal(12, duration.Value); + Assert.Equal("ms", duration.Unit); + Assert.Equal("direct", duration.Tags["mode"]); + Assert.Equal(outcome, duration.Tags["status"]); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_CACHE_LOOKUPS)); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_FALLBACKS)); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_PEER_PROBES)); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_PEER_REPAIRS)); + } + finally + { + release.TrySetResult(CreateGrainManifest()); + } + } + + [Theory] + [InlineData("timeout")] + [InlineData("canceled")] + [InlineData("error")] + [InlineData("success")] + public async Task PeerMetrics_RecordLocalAttemptOutcomesAndAdmission(string outcome) + { + var cancellationToken = TestContext.Current.CancellationToken; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var metrics = new ManifestMetrics(); + var time = new FakeTimeProvider(); + var local = CreateSiloAddress(11111, 1); + var peer = CreateSiloAddress(11112, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, [peer])); + var pending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = Substitute.For(); + target.GetClusterManifestHashSummary(Arg.Any()).Returns(new ValueTask(pending.Task)); + var factory = Substitute.For(); + factory.GetSystemTarget(Constants.ManifestProviderType, peer).Returns(target); + await using var provider = CreateClusterManifestProvider(local, membership, factory, time, NullLogger.Instance, metrics.Instruments); + await InitializeProviderAsync(provider, cancellationToken); + var cache = new ConcurrentDictionary(); + var hash = ManifestHashCalculator.ComputeHash(provider.LocalGrainManifest); + cache[hash] = provider.LocalGrainManifest; + var probes = Enumerable.Range(0, 3).Select(_ => ProbePeerAsync(provider, peer, [peer], cache, cancellation.Token)).ToArray(); + try + { + await ProbePeerAsync(provider, peer, [peer], cache, cancellationToken); + Assert.Equal(1, metrics.Sum(InstrumentNames.MANIFEST_PEER_PROBES, ("status", "skipped"))); + switch (outcome) + { + case "timeout": + time.Advance(TimeSpan.FromSeconds(1)); + break; + case "canceled": + cancellation.Cancel(); + break; + case "error": + pending.SetException(new NotSupportedException()); + break; + case "success": + pending.SetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), new() { [peer] = hash })); + break; + } + + if (outcome == "canceled") + { + await Assert.ThrowsAnyAsync(() => Task.WhenAll(probes)); + } + else + { + await Task.WhenAll(probes); + } + + Assert.Equal(3, metrics.Sum(InstrumentNames.MANIFEST_PEER_PROBES, ("status", outcome))); + Assert.All(metrics.Find(InstrumentNames.MANIFEST_PEER_PROBES), measurement => Assert.Single(measurement.Tags)); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_PEER_REPAIRS)); + } + finally + { + pending.TrySetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), [])); + } + } + + private sealed class ManifestMetrics : IDisposable + { + private readonly ServiceProvider _services; + private readonly MeterListener _listener = new(); + private readonly ConcurrentQueue _measurements = new(); + + public ManifestMetrics() + { + _services = new ServiceCollection().AddMetrics().AddSingleton().BuildServiceProvider(); + var orleans = _services.GetRequiredService(); + Instruments = new ClusterManifestInstruments(orleans); + _listener.InstrumentPublished = (instrument, listener) => + { + if (ReferenceEquals(instrument.Meter, orleans.Meter) && instrument.Name.StartsWith("orleans-manifest-", StringComparison.Ordinal)) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _listener.Start(); + } + + public ClusterManifestInstruments Instruments { get; } + + public ManifestMeasurement[] Find(string name) => _measurements.Where(measurement => measurement.Name == name).ToArray(); + + public double Sum(string name, params (string Name, string Value)[] tags) => + Find(name).Where(measurement => tags.All(tag => Equals(measurement.Tags[tag.Name], tag.Value))).Sum(measurement => measurement.Value); + + private void Record(Instrument instrument, double value, ReadOnlySpan> tags) => + _measurements.Enqueue(new(instrument.Name, instrument.Unit, value, tags.ToArray().ToDictionary(tag => tag.Key, tag => tag.Value))); + + public void Dispose() + { + _listener.Dispose(); + _services.Dispose(); + } + } + + private sealed record ManifestMeasurement(string Name, string? Unit, double Value, Dictionary Tags); +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.Cancellation.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.Cancellation.cs new file mode 100644 index 00000000000..fdbcd277d74 --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.Cancellation.cs @@ -0,0 +1,355 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task PeerRepair_LocalCompletionReleasesSlotsBeforeLateResponses(bool waitForUpdate, bool cancelCaller, bool lateFailure) + { + var cancellationToken = TestContext.Current.CancellationToken; + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 4).Select(port => CreateSiloAddress(port, 1)).ToArray(); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var timeProvider = new FakeTimeProvider(); + var logger = new PeerProbeLogger(expectedTimeoutCount: 3); + var requests = new ManifestRequestLog(expectedProbeCount: 6, expectedLegacyFetchCount: 8); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingUpdate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allUpdatesEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lateManifest = CreateGrainManifest(); + var lateHash = ManifestHashCalculator.ComputeHash(lateManifest); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(1, 0), + peers.ToDictionary(peer => peer, _ => lateHash)); + var lateUpdate = new ClusterManifestUpdate( + new MajorMinorVersion(1, 0), + peers.ToImmutableDictionary(peer => peer, _ => lateManifest), + includesAllActiveServers: true); + var updateRequests = 0; + var recovered = false; + GrainManifest recoveredManifest = null!; + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requests.RecordProbe(peer); + return waitForUpdate || recovered ? Task.FromResult(summary) : pendingSummary.Task; + }, + getUpdate: (_, _) => + { + if (Interlocked.Increment(ref updateRequests) == 3) + { + allUpdatesEntered.TrySetResult(); + } + + return recovered ? Task.FromResult(null) : pendingUpdate.Task; + }, + getLegacyManifest: _ => + { + requests.RecordLegacyFetch(peer); + return recovered ? Task.FromResult(recoveredManifest) + : Task.FromException(new InvalidOperationException("Direct fetch temporarily unavailable.")); + })); + await using var provider = CreateClusterManifestProvider(localSilo, membership, CreateGrainFactory(targets), timeProvider, logger); + recoveredManifest = provider.LocalGrainManifest; + Assert.NotEqual(lateHash, ManifestHashCalculator.ComputeHash(recoveredManifest)); + var initial = provider.Current; + var originalCache = GetCachedManifests(provider); + await InitializeProviderAsync(provider, cancellationToken); + try + { + var first = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellation.Token); + await requests.WaitForProbeCountAsync(3, cancellationToken); + if (waitForUpdate) + { + await allUpdatesEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + var atCapacity = ProbePeerAsync(provider, peers[0], peers, originalCache, cancellationToken); + await atCapacity.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(3, requests.ProbeAddresses.Count); + Assert.False(first.IsCompleted); + + if (cancelCaller) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => first.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + } + else + { + timeProvider.Advance(TimeSpan.FromSeconds(1)); + Assert.False(await first.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + } + + Assert.Same(initial, provider.Current); + recovered = true; + membership.Update(CreateActiveMembershipSnapshot(2, localSilo, peers)); + var retry = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken); + Assert.True(await retry.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Equal(6, requests.ProbeAddresses.Count); + Assert.Equal(8, requests.LegacyFetchAddresses.Count); + Assert.Equal(5, provider.Current.Silos.Count); + Assert.All(peers, peer => Assert.Same(recoveredManifest, provider.Current.Silos[peer])); + Assert.False(waitForUpdate ? pendingUpdate.Task.IsCompleted : pendingSummary.Task.IsCompleted); + var current = provider.Current; + var currentCache = GetCachedManifests(provider); + + if (lateFailure) + { + if (waitForUpdate) + { + pendingUpdate.SetException(new InvalidOperationException("Late update failure.")); + } + else + { + pendingSummary.SetException(new InvalidOperationException("Late summary failure.")); + } + + await logger.WaitForLateFailureCountAsync(3, cancellationToken); + } + else + { + pendingSummary.TrySetResult(summary); + pendingUpdate.TrySetResult(lateUpdate); + await Task.WhenAll(pendingSummary.Task, pendingUpdate.Task); + } + + Assert.Same(current, provider.Current); + Assert.Same(currentCache, GetCachedManifests(provider)); + Assert.False(originalCache.ContainsKey(lateHash)); + Assert.False(currentCache.ContainsKey(lateHash)); + Assert.Equal(6, requests.ProbeAddresses.Count); + } + finally + { + pendingSummary.TrySetResult(summary); + pendingUpdate.TrySetResult(null); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task PeerRepair_Cancellation_ReachesSummaryAndUpdateRpc(bool waitForUpdate, bool cancelCaller) + { + var localSilo = CreateSiloAddress(11111, 1); + var remoteSilo = CreateSiloAddress(11112, 1); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, [remoteSilo])); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var timeProvider = new FakeTimeProvider(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingUpdate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var summary = new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), new Dictionary()); + var summaryToken = default(CancellationToken); + var remote = Substitute.For(); + remote.GetClusterManifestHashSummary(Arg.Any()).Returns(call => + { + summaryToken = call.Arg(); + if (waitForUpdate) + { + return new ValueTask(summary); + } + + entered.TrySetResult(summaryToken); + return new ValueTask(pendingSummary.Task); + }); + remote.GetClusterManifestUpdate(MajorMinorVersion.MinValue, Arg.Any()).Returns(call => + { + entered.TrySetResult(call.Arg()); + return new ValueTask(pendingUpdate.Task); + }); + var grainFactory = Substitute.For(); + grainFactory.GetSystemTarget(Constants.ManifestProviderType, remoteSilo).Returns(remote); + await using var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, timeProvider, NullLogger.Instance); + await InitializeProviderAsync(provider, cancellation.Token); + var probe = ProbePeerAsync(provider, remoteSilo, new[] { remoteSilo }, GetCachedManifests(provider), cancellation.Token); + try + { + var rpcToken = await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(rpcToken.CanBeCanceled); + Assert.Equal(summaryToken, rpcToken); + Assert.False(rpcToken.IsCancellationRequested); + if (cancelCaller) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => probe.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + else + { + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await probe.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(cancellation.IsCancellationRequested); + } + + Assert.True(rpcToken.IsCancellationRequested); + } + finally + { + cancellation.Cancel(); + pendingSummary.TrySetResult(summary); + pendingUpdate.TrySetResult(null); + } + } + + [Fact] + public async Task PeerRepair_CanceledPeerProbe_UsesLegacyManifest() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + var remoteManifest = CreateGrainManifest(); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => Task.FromCanceled(new CancellationToken(canceled: true)), + getUpdate: (_, _) => Task.FromResult(null), + getLegacyManifest: _ => Task.FromResult(remoteManifest))); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + await using var provider = CreateClusterManifestProvider(localSilo, membership, CreateGrainFactory(targets)); + var observed = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + try + { + var current = await observed.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.All(peers, peer => Assert.Equal(remoteManifest, current.Silos[peer])); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PeerRepair_StopCancellation_CompletesHungProbeProcessing() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 4).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 3, expectedLegacyFetchCount: peers.Length); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingLegacyFetch = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return pendingSummary.Task; + }, + getUpdate: (_, _) => Task.FromResult(null), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return pendingLegacyFetch.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + var stopped = false; + + try + { + await Task.WhenAll(requestLog.WaitForProbeCountAsync(3, TestContext.Current.CancellationToken), requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken)); + + await lifecycle.OnStop(TestContext.Current.CancellationToken); + stopped = true; + + Assert.Equal(GetExpectedProbePeers(localSilo, peers, round: 1), requestLog.ProbeAddresses); + Assert.Equal(3, requestLog.ProbeAddresses.Count); + Assert.DoesNotContain(peers[0], provider.Current.Silos.Keys); + } + finally + { + if (!stopped) + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task ClusterManifestProviderPreservesCallerCancellationDuringPeerFill() + { + var localSilo = CreateSiloAddress(11801, 1); + var peers = Enumerable.Range(11802, 2).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var requestLog = new ManifestRequestLog(expectedProbeCount: peers.Length, expectedLegacyFetchCount: peers.Length); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingLegacyManifest = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return pendingSummary.Task; + }, + getUpdate: (_, _) => Task.FromResult(null), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return pendingLegacyManifest.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + var stopped = false; + + try + { + await Task.WhenAll( + requestLog.WaitForProbeCountAsync(peers.Length, TestContext.Current.CancellationToken), + requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken)); + + var legacyFetchCountBeforeStop = requestLog.LegacyFetchAddresses.Count; + + await lifecycle.OnStop(TestContext.Current.CancellationToken); + stopped = true; + + Assert.Equal(peers.Length, requestLog.ProbeAddresses.Count); + Assert.Equal(legacyFetchCountBeforeStop, requestLog.LegacyFetchAddresses.Count); + Assert.DoesNotContain(peers[0], provider.Current.Silos.Keys); + Assert.DoesNotContain(peers[1], provider.Current.Silos.Keys); + } + finally + { + if (!stopped) + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + + pendingLegacyManifest.TrySetResult(CreateGrainManifest()); + pendingSummary.TrySetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), [])); + provider.Dispose(); + membership.Dispose(); + } + } +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.cs new file mode 100644 index 00000000000..fc6f5843cae --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.PeerRepair.cs @@ -0,0 +1,592 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + [Fact] + public async Task PeerRepair_PartialResult_CompletesRemainingSilosWithoutWaitingForRepairedSiloFetch() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + var remoteManifest = CreateGrainManifest(); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(1, 1), + new Dictionary { [peers[0]] = ManifestHashCalculator.ComputeHash(remoteManifest) }); + var peerUpdate = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), + ImmutableDictionary.Empty.Add(peers[0], remoteManifest), + includesAllActiveServers: false); + var pendingDirectFetch = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => Task.FromResult(summary), + getUpdate: (_, _) => Task.FromResult(peerUpdate), + getLegacyManifest: _ => peer == peers[0] ? pendingDirectFetch.Task : Task.FromResult(remoteManifest))); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + await using var provider = CreateClusterManifestProvider(localSilo, membership, CreateGrainFactory(targets)); + var observed = ObserveManifestAsync(provider, new MajorMinorVersion(1, 2), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + try + { + var current = await observed.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.All(peers, peer => Assert.Equal(remoteManifest, current.Silos[peer])); + Assert.False(pendingDirectFetch.Task.IsCompleted); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PeerRepair_HungPeersAndHealthyLaterPeer_UsesAtMostThreeConcurrentProbesAndFakeOneSecondTimeout() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 4).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var timeProvider = new FakeTimeProvider(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 3, expectedLegacyFetchCount: peers.Length); + var logger = new PeerProbeLogger(expectedTimeoutCount: 2); + var directFetchRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hungProbeCompletions = peers.ToDictionary( + static peer => peer, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + var selectedPeers = GetExpectedProbePeers(localSilo, peers, round: 1); + var healthyPeer = selectedPeers[2]; + var remoteManifest = CreateGrainManifest(); + var remoteHashes = peers.ToDictionary(static peer => peer, _ => ManifestHashCalculator.ComputeHash(remoteManifest)); + var summary = new ClusterManifestHashSummary(new MajorMinorVersion(1, 1), remoteHashes); + var update = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), + peers.ToImmutableDictionary(static peer => peer, _ => remoteManifest), + includesAllActiveServers: true); + var healthyUpdateRequested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var activeProbeCount = 0; + var maximumActiveProbeCount = 0; + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + var active = Interlocked.Increment(ref activeProbeCount); + UpdateMaximum(ref maximumActiveProbeCount, active); + if (peer == healthyPeer) + { + Interlocked.Decrement(ref activeProbeCount); + return Task.FromResult(summary); + } + + // This peer ignores RPC cancellation; its late completion is bounded by the test lifetime. + return AwaitProbeAsync(hungProbeCompletions[peer].Task, () => Interlocked.Decrement(ref activeProbeCount), TestContext.Current.CancellationToken); + }, + getUpdate: (version, _) => + { + if (peer == healthyPeer) + { + healthyUpdateRequested.TrySetResult(version); + } + + return Task.FromResult(update); + }, + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return directFetchRelease.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, timeProvider, logger); + var repairedManifest = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(requestLog.WaitForProbeCountAsync(3, TestContext.Current.CancellationToken), requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken), healthyUpdateRequested.Task); + var requestedVersion = await healthyUpdateRequested.Task; + + Assert.Equal(selectedPeers, requestLog.ProbeAddresses); + Assert.Equal(3, maximumActiveProbeCount); + Assert.Equal(3, requestLog.ProbeAddresses.Count); + Assert.Equal(MajorMinorVersion.MinValue, requestedVersion); + + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await logger.WaitForTimeoutCountAsync(2, TestContext.Current.CancellationToken); + Assert.Equal(2, logger.TimeoutCount); + Assert.Equal(3, requestLog.ProbeAddresses.Count); + + foreach (var hungPeer in selectedPeers.Take(2)) + { + hungProbeCompletions[hungPeer].TrySetException( + new InvalidOperationException($"Late peer probe failure from {hungPeer}.")); + } + + await logger.WaitForLateFailureCountAsync(2, TestContext.Current.CancellationToken); + Assert.Equal(2, logger.LateFailureCount); + + var repaired = await repairedManifest; + + Assert.Equal(new MajorMinorVersion(1, 1), repaired.Version); + Assert.All(peers, peer => Assert.Equal(remoteManifest, repaired.Silos[peer])); + Assert.False(directFetchRelease.Task.IsCompleted); + + directFetchRelease.TrySetException(new InvalidOperationException("Peer repair already supplied the manifests.")); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task PeerRepair_PartialResult_PublishesBeforeHungDirectFetchesComplete() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 3).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var repairedPeer = peers[0]; + var remoteManifest = CreateGrainManifest(); + var remoteHash = ManifestHashCalculator.ComputeHash(remoteManifest); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(1, 1), + new Dictionary { [repairedPeer] = remoteHash }); + var update = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), + ImmutableDictionary.Empty.Add(repairedPeer, remoteManifest), + includesAllActiveServers: false); + var pendingDirectFetch = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var requestLog = new ManifestRequestLog(expectedProbeCount: peers.Length, expectedLegacyFetchCount: peers.Length); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return Task.FromResult(summary); + }, + getUpdate: (_, _) => Task.FromResult(update), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return pendingDirectFetch.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider( + localSilo, + membership, + grainFactory, + new FakeTimeProvider(), + NullLogger.Instance); + var repairedManifest = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll( + requestLog.WaitForProbeCountAsync(peers.Length, TestContext.Current.CancellationToken), + requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken)); + + var repaired = await repairedManifest.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.Equal(remoteManifest, repaired.Silos[repairedPeer]); + Assert.Contains(localSilo, repaired.Silos.Keys); + Assert.DoesNotContain(peers.Skip(1), repaired.Silos.Keys.Contains); + Assert.False(pendingDirectFetch.Task.IsCompleted); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task UpdateManifest_StartsLegacyFetchBeforeHungPeerProbesComplete() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 4).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 3, expectedLegacyFetchCount: peers.Length); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var legacyFetchRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return pendingSummary.Task; + }, + getUpdate: (_, _) => Task.FromResult(null), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return legacyFetchRelease.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(requestLog.WaitForProbeCountAsync(3, TestContext.Current.CancellationToken), requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken)); + + Assert.Equal(3, requestLog.ProbeAddresses.Count); + Assert.Equal(peers.Length, requestLog.LegacyFetchAddresses.Count); + Assert.False(pendingSummary.Task.IsCompleted); + + legacyFetchRelease.TrySetResult(CreateGrainManifest()); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task PeerRepair_RequestsUpdateFromMajorMinorVersionMinValue() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 3).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 3, expectedLegacyFetchCount: peers.Length); + var directFetchRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var remoteManifest = CreateGrainManifest(); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(1, 1), + peers.ToDictionary(static peer => peer, _ => ManifestHashCalculator.ComputeHash(remoteManifest))); + var update = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), + peers.ToImmutableDictionary(static peer => peer, _ => remoteManifest), + includesAllActiveServers: true); + var requestedVersions = new List(); + var updateRequestsStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return Task.FromResult(summary); + }, + getUpdate: (version, _) => + { + lock (requestedVersions) + { + requestedVersions.Add(version); + if (requestedVersions.Count == 3) + { + updateRequestsStarted.TrySetResult(); + } + } + + return Task.FromResult(update); + }, + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return directFetchRelease.Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var repairedManifest = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken), updateRequestsStarted.Task); + + lock (requestedVersions) + { + Assert.Equal(3, requestedVersions.Count); + Assert.All(requestedVersions, version => Assert.Equal(MajorMinorVersion.MinValue, version)); + } + + directFetchRelease.TrySetException(new InvalidOperationException("Peer repair must supply the manifests after direct fetches fail.")); + var repaired = await repairedManifest; + + Assert.All(peers, peer => Assert.Equal(remoteManifest, repaired.Silos[peer])); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task PeerRepair_RepeatedMembershipUpdates_RotateObservedPeerSelections() + { + var localSilo = CreateSiloAddress(11111, 1); + var peers = Enumerable.Range(11112, 4).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 6, expectedLegacyFetchCount: 0); + var emptySummary = new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), new Dictionary()); + var emptyUpdate = new ClusterManifestUpdate( + new MajorMinorVersion(1, 0), + ImmutableDictionary.Empty, + includesAllActiveServers: false); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return Task.FromResult(emptySummary); + }, + getUpdate: (_, _) => Task.FromResult(emptyUpdate), + getLegacyManifest: _ => Task.FromException(new InvalidOperationException("Direct fetch intentionally unavailable.")))); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await requestLog.WaitForProbeCountAsync(3, TestContext.Current.CancellationToken); + var secondAttemptStarted = requestLog.WaitForProbeCountAsync(6, TestContext.Current.CancellationToken); + membership.Update(CreateActiveMembershipSnapshot(2, localSilo, peers)); + await secondAttemptStarted; + + var firstSelection = requestLog.ProbeAddresses.Take(3).ToArray(); + var secondSelection = requestLog.ProbeAddresses.Skip(3).Take(3).ToArray(); + + Assert.Equal(3, firstSelection.Length); + Assert.Equal(3, secondSelection.Length); + Assert.NotEqual(firstSelection[0], secondSelection[0]); + AssertContiguousCyclicSegment(peers, firstSelection); + AssertContiguousCyclicSegment(peers, secondSelection); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task ClusterManifestProviderRejectsMismatchedPeerHashAndFallsBackToLegacyFetch() + { + var localSilo = CreateSiloAddress(11601, 1); + var peers = Enumerable.Range(11602, 2).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var peerA = peers[0]; + var peerB = peers[1]; + var remoteManifestA = CreateGrainManifest(); + var remoteManifestB = new GrainManifest( + ImmutableDictionary.Empty, + ImmutableDictionary.Empty); + var legacyManifestB = new GrainManifest( + ImmutableDictionary.Empty.Add( + TestGrainType, + new GrainProperties(CreatePropertyDictionary( + new KeyValuePair(WellKnownGrainTypeProperties.TypeName, "LegacyFallbackTest")))), + ImmutableDictionary.Empty); + // Use a fabricated hash that no other silo's manifest will ever validate against, so the + // hash-cache reuse optimization cannot coincidentally "validate" peerB's mismatched claim. + var mismatchedHashForB = new ManifestHash("intentionally-invalid-hash-for-peerB"); + Assert.NotEqual(mismatchedHashForB, ManifestHashCalculator.ComputeHash(remoteManifestA)); + Assert.NotEqual(mismatchedHashForB, ManifestHashCalculator.ComputeHash(remoteManifestB)); + + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(1, 1), + new Dictionary + { + [peerA] = ManifestHashCalculator.ComputeHash(remoteManifestA), + [peerB] = mismatchedHashForB, + }); + var update = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), + ImmutableDictionary.Empty.Add(peerA, remoteManifestA).Add(peerB, remoteManifestB), + includesAllActiveServers: true); + + var requestLog = new ManifestRequestLog(expectedProbeCount: peers.Length, expectedLegacyFetchCount: peers.Length); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return Task.FromResult(summary); + }, + getUpdate: (_, _) => Task.FromResult(update), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return Task.FromResult(peer.Equals(peerB) ? legacyManifestB : remoteManifestA); + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, new FakeTimeProvider(), NullLogger.Instance); + var repairedManifest = ObserveManifestAsync(provider, new MajorMinorVersion(1, 2), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + var repaired = await repairedManifest.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // The mismatched hash advertised for peerB must be rejected: its peer-supplied manifest never + // enters the published result, and the legacy direct fetch is what ultimately resolves it. + Assert.Same(legacyManifestB, repaired.Silos[peerB]); + Assert.NotEqual(ManifestHashCalculator.ComputeHash(remoteManifestB), ManifestHashCalculator.ComputeHash(repaired.Silos[peerB])); + Assert.Equal(ManifestHashCalculator.ComputeHash(remoteManifestA), ManifestHashCalculator.ComputeHash(repaired.Silos[peerA])); + Assert.Contains(peerB, requestLog.LegacyFetchAddresses); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + provider.Dispose(); + membership.Dispose(); + } + } + + [Fact] + public async Task ClusterManifestProviderRotatesBoundedPeerProbesBeforeLegacyFallback() + { + var localSilo = CreateSiloAddress(11701, 1); + var peers = Enumerable.Range(11702, 4).Select(port => CreateSiloAddress(port, 1)).OrderBy(static address => address).ToArray(); + var timeProvider = new FakeTimeProvider(); + var requestLog = new ManifestRequestLog(expectedProbeCount: 3, expectedLegacyFetchCount: peers.Length); + var logger = new PeerProbeLogger(expectedTimeoutCount: 3); + var expectedProbedPeers = GetExpectedProbePeers(localSilo, peers, round: 1); + var unprobedPeer = peers.Except(expectedProbedPeers).Single(); + var hungProbeCompletions = peers.ToDictionary( + static peer => peer, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + var directCompletions = peers.ToDictionary( + static peer => peer, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + // Each peer's fallback manifest must be structurally distinct: ClusterManifest canonicalizes + // (deduplicates) structurally-equal GrainManifest instances within a single published manifest, + // which would otherwise make different peers' entries reference-equal regardless of provenance. + var legacyManifests = peers.ToDictionary( + peer => peer, + peer => new GrainManifest( + ImmutableDictionary.Empty.Add( + TestGrainType, + new GrainProperties(CreatePropertyDictionary( + new KeyValuePair(WellKnownGrainTypeProperties.TypeName, $"LegacyFallbackTest-{peer}")))), + ImmutableDictionary.Empty)); + var targets = peers.ToDictionary( + peer => peer, + peer => new TestClusterManifestSystemTarget( + getHashSummary: _ => + { + requestLog.RecordProbe(peer); + return hungProbeCompletions[peer].Task; + }, + getUpdate: (_, _) => Task.FromResult(null), + getLegacyManifest: _ => + { + requestLog.RecordLegacyFetch(peer); + return directCompletions[peer].Task; + })); + var grainFactory = CreateGrainFactory(targets); + var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, localSilo, peers)); + var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory, timeProvider, logger); + var repairedManifest = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(requestLog.WaitForProbeCountAsync(3, TestContext.Current.CancellationToken), requestLog.WaitForLegacyFetchCountAsync(peers.Length, TestContext.Current.CancellationToken)); + + // Peer probing is bounded to at most three concurrent probes and selects the exact rotating, + // contiguous cyclic segment of candidates; the fourth peer is never probed. + Assert.Equal(expectedProbedPeers, requestLog.ProbeAddresses); + Assert.Equal(3, requestLog.ProbeAddresses.Count); + Assert.DoesNotContain(unprobedPeer, requestLog.ProbeAddresses); + AssertContiguousCyclicSegment(peers, requestLog.ProbeAddresses); + + timeProvider.Advance(TimeSpan.FromSeconds(1)); + await logger.WaitForTimeoutCountAsync(3, TestContext.Current.CancellationToken); + Assert.Equal(3, logger.TimeoutCount); + + foreach (var peer in peers) + { + directCompletions[peer].SetResult(legacyManifests[peer]); + } + + var repaired = await repairedManifest.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // Since every bounded probe timed out, the fallback direct fetch is what supplies every silo, + // including the peer that the bounded rotation never selected for probing. + Assert.All(peers, peer => Assert.Same(legacyManifests[peer], repaired.Silos[peer])); + } + finally + { + await lifecycle.OnStop(TestContext.Current.CancellationToken); + foreach (var peer in peers) + { + directCompletions[peer].TrySetResult(legacyManifests[peer]); + hungProbeCompletions[peer].TrySetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), [])); + } + + provider.Dispose(); + membership.Dispose(); + } + } + + private static SiloAddress[] GetExpectedProbePeers(SiloAddress localSilo, SiloAddress[] peers, int round) + { + var start = (int)((uint)(localSilo.GetConsistentHashCode() + round) % (uint)peers.Length); + return Enumerable.Range(0, Math.Min(3, peers.Length)) + .Select(index => peers[(start + index) % peers.Length]) + .ToArray(); + } + + private static async Task AwaitProbeAsync( + Task task, + Action onCompleted, + CancellationToken cancellationToken) + { + try + { + return await task.WaitAsync(cancellationToken); + } + finally + { + onCompleted(); + } + } + + private static void UpdateMaximum(ref int maximum, int value) + { + while (true) + { + var current = Volatile.Read(ref maximum); + if (current >= value || Interlocked.CompareExchange(ref maximum, value, current) == current) + { + return; + } + } + } + + private static void AssertContiguousCyclicSegment(IReadOnlyList candidates, IReadOnlyList selection) + { + var start = Array.IndexOf(candidates.ToArray(), selection[0]); + Assert.NotEqual(-1, start); + Assert.Equal( + selection, + Enumerable.Range(0, selection.Count).Select(index => candidates[(start + index) % candidates.Count])); + } +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.TestHelpers.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.TestHelpers.cs new file mode 100644 index 00000000000..ee1598c1ed3 --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.TestHelpers.cs @@ -0,0 +1,531 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Orleans.Runtime.Utilities; +using Orleans.Serialization; +using Orleans.Serialization.Configuration; +using Orleans.Serialization.TypeSystem; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + private static readonly GrainType TestGrainType = GrainType.Create("test"); + + private static readonly GrainInterfaceType TestInterfaceType = GrainInterfaceType.Create("test.interface"); + + private static readonly MethodInfo InitializeMethod = typeof(ClusterManifestProvider) + .GetMethod("Initialize", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly MethodInfo UpdateManifestMethod = typeof(ClusterManifestProvider) + .GetMethod("UpdateManifest", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly MethodInfo ProbePeerForManifestsMethod = typeof(ClusterManifestProvider) + .GetMethod("ProbePeerForManifests", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly FieldInfo ManifestCacheField = typeof(ClusterManifestProvider) + .GetField("_manifestCache", BindingFlags.Instance | BindingFlags.NonPublic)!; + + private static ClusterManifestProvider CreateClusterManifestProvider( + SiloAddress localSilo, + TestClusterMembershipService membership, + IInternalGrainFactory grainFactory, + ClusterManifestOptions? options = null, + ClusterManifestInstruments? instruments = null) => + CreateClusterManifestProvider( + localSilo, + membership, + grainFactory, + TimeProvider.System, + NullLogger.Instance, + instruments, + options); + + private static IInternalGrainFactory CreateGrainFactory(SiloAddress remoteSilo, GrainManifest remoteManifest) + { + var grainFactory = Substitute.For(); + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, remoteSilo) + .Returns(new TestSiloManifestSystemTarget(remoteManifest)); + return grainFactory; + } + + private static GrainManifest CreateGrainManifest() + { + var grains = ImmutableDictionary.CreateRange( + [ + new KeyValuePair( + TestGrainType, + new GrainProperties(CreatePropertyDictionary( + [ + new KeyValuePair(WellKnownGrainTypeProperties.TypeName, "Test"), + new KeyValuePair(WellKnownGrainTypeProperties.FullTypeName, "UnitTests.Grains.Test"), + new KeyValuePair($"{WellKnownGrainTypeProperties.ImplementedInterfacePrefix}0", TestInterfaceType.ToString()) + ]))) + ]); + var interfaces = ImmutableDictionary.CreateRange( + [ + new KeyValuePair( + TestInterfaceType, + new GrainInterfaceProperties(CreatePropertyDictionary( + [ + new KeyValuePair(WellKnownGrainInterfaceProperties.TypeName, "ITest"), + new KeyValuePair(WellKnownGrainInterfaceProperties.Version, "1") + ]))) + ]); + + return new GrainManifest(grains, interfaces); + } + + private static ImmutableDictionary CreatePropertyDictionary(params KeyValuePair[] properties) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal, StringComparer.Ordinal); + foreach (var property in properties) + { + builder.Add(property.Key, property.Value); + } + + return builder.ToImmutable(); + } + + private static SiloManifestProvider CreateSiloManifestProvider() + { + var typeConverter = CreateTypeConverter(); + var interfaceTypeResolver = new GrainInterfaceTypeResolver([new TestGrainInterfaceTypeProvider()], typeConverter); + var typeNameProvider = new TypeNameGrainPropertiesProvider(); + var options = new GrainTypeOptions(); + options.Classes.Add(typeof(TestManifestGrain)); + options.Interfaces.Add(typeof(ITestManifestGrain)); + + return new SiloManifestProvider( + [typeNameProvider, new ImplementedInterfaceProvider(interfaceTypeResolver)], + [typeNameProvider, new TestGrainInterfacePropertiesProvider()], + Options.Create(options), + new GrainTypeResolver([new TestGrainTypeProvider()], typeConverter), + interfaceTypeResolver, + typeConverter); + } + + internal interface ITestManifestGrain : IGrainWithStringKey; + + internal sealed class TestManifestGrain : ITestManifestGrain; + + private sealed class TestGrainTypeProvider : IGrainTypeProvider + { + public bool TryGetGrainType(Type type, out GrainType grainType) + { + if (type == typeof(TestManifestGrain)) + { + grainType = TestGrainType; + return true; + } + + grainType = default; + return false; + } + } + + private sealed class TestGrainInterfaceTypeProvider : IGrainInterfaceTypeProvider + { + public bool TryGetGrainInterfaceType(Type type, out GrainInterfaceType grainInterfaceType) + { + if (type == typeof(ITestManifestGrain)) + { + grainInterfaceType = TestInterfaceType; + return true; + } + + grainInterfaceType = default; + return false; + } + } + + private sealed class TestGrainInterfacePropertiesProvider : IGrainInterfacePropertiesProvider + { + public void Populate(Type interfaceType, GrainInterfaceType grainInterfaceType, Dictionary properties) + { + properties[WellKnownGrainInterfaceProperties.Version] = "1"; + } + } + + private static Orleans.Serialization.TypeSystem.TypeConverter CreateTypeConverter() + { + return new Orleans.Serialization.TypeSystem.TypeConverter( + Array.Empty(), + Array.Empty(), + Array.Empty(), + Options.Create(new TypeManifestOptions { AllowAllTypes = true }), + new CachedTypeResolver()); + } + + private static ClusterMembershipSnapshot CreateMembershipSnapshot( + long version, + params (SiloAddress SiloAddress, SiloStatus Status)[] members) + { + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var (siloAddress, status) in members) + { + builder[siloAddress] = new ClusterMember(siloAddress, status, siloAddress.ToString()); + } + + return new ClusterMembershipSnapshot(builder.ToImmutable(), new MembershipVersion(version)); + } + + private static SiloAddress CreateSiloAddress(int port, int generation) + { + return SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), generation); + } + + private static async Task StartAsync(ClusterManifestProvider provider, CancellationToken cancellationToken) + { + var lifecycle = new SiloLifecycleSubject(NullLoggerFactory.Instance.CreateLogger()); + ((ILifecycleParticipant)provider).Participate(lifecycle); + await lifecycle.OnStart(cancellationToken); + return lifecycle; + } + + private sealed class TestClusterMembershipService : IClusterMembershipService, IDisposable + { + private readonly AsyncEnumerable _updates; + private ClusterMembershipSnapshot _currentSnapshot = ClusterMembershipSnapshot.Default; + + public TestClusterMembershipService(ClusterMembershipSnapshot initialSnapshot) + { + _updates = new AsyncEnumerable( + initialValue: initialSnapshot, + updateValidator: (previous, proposed) => proposed.Version > previous.Version, + onPublished: update => Volatile.Write(ref _currentSnapshot, update)); + } + + public ClusterMembershipSnapshot CurrentSnapshot + { + get => Volatile.Read(ref _currentSnapshot); + } + + public IAsyncEnumerable MembershipUpdates => _updates; + + public void Update(ClusterMembershipSnapshot snapshot) => _updates.Publish(snapshot); + + public ValueTask Refresh(MembershipVersion minimumVersion = default, CancellationToken cancellationToken = default) => default; + + public Task TryKill(SiloAddress siloAddress) => Task.FromResult(false); + + public void Dispose() => _updates.Dispose(); + } + + private sealed class TestSiloManifestSystemTarget(GrainManifest manifest) : ISiloManifestSystemTarget + { + public ValueTask GetSiloManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(manifest); + } + } + + private static Task InitializeProviderAsync(ClusterManifestProvider provider, CancellationToken cancellationToken) => + (Task)InitializeMethod.Invoke(provider, [cancellationToken])!; + + private static Task UpdateManifestAsync( + ClusterManifestProvider provider, + ClusterMembershipSnapshot snapshot, + CancellationToken cancellationToken) => + (Task)UpdateManifestMethod.Invoke(provider, [snapshot, cancellationToken])!; + + private static Task ProbePeerAsync( + ClusterManifestProvider provider, + SiloAddress peer, + IReadOnlyCollection missingSilos, + ConcurrentDictionary cache, + CancellationToken cancellationToken) => + (Task)ProbePeerForManifestsMethod.Invoke(provider, [peer, missingSilos, cache, cancellationToken])!; + + private static ConcurrentDictionary GetCachedManifests(ClusterManifestProvider provider) => + (ConcurrentDictionary)ManifestCacheField.GetValue(provider)!; + + private static ClusterManifestProvider CreateClusterManifestProvider( + SiloAddress localSilo, + TestClusterMembershipService membership, + IInternalGrainFactory grainFactory, + TimeProvider timeProvider, + ILogger logger, + ClusterManifestInstruments? instruments = null, + ClusterManifestOptions? options = null) + { + var siloManifestProvider = CreateSiloManifestProvider(); + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, localSilo) + .Returns(new TestSiloManifestSystemTarget(siloManifestProvider.SiloManifest)); + + var services = new ServiceCollection() + .AddSingleton(grainFactory) + .AddMetrics() + .AddSingleton() + .AddSingleton() + .BuildServiceProvider(); + + var localSiloDetails = Substitute.For(); + localSiloDetails.SiloAddress.Returns(localSilo); + + return new ClusterManifestProvider( + localSiloDetails, + siloManifestProvider, + membership, + Substitute.For(), + logger, + services, + timeProvider, + Options.Create(options ?? new ClusterManifestOptions()), + instruments ?? services.GetRequiredService()); + } + + private static ClusterMembershipSnapshot CreateActiveMembershipSnapshot( + long version, + SiloAddress localSilo, + SiloAddress[] peers) + { + var members = new (SiloAddress SiloAddress, SiloStatus Status)[peers.Length + 1]; + members[0] = (localSilo, SiloStatus.Active); + for (var index = 0; index < peers.Length; index++) + { + members[index + 1] = (peers[index], SiloStatus.Active); + } + + return CreateMembershipSnapshot(version, members); + } + + private static IInternalGrainFactory CreateGrainFactory( + IReadOnlyDictionary targets) + { + var grainFactory = Substitute.For(); + foreach (var (siloAddress, target) in targets) + { + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, siloAddress) + .Returns(target); + grainFactory + .GetSystemTarget(Constants.ManifestProviderType, siloAddress) + .Returns(target); + } + + return grainFactory; + } + + private static async Task ObserveManifestAsync( + ClusterManifestProvider provider, + MajorMinorVersion expectedVersion, + CancellationToken cancellationToken) + { + await using var updates = provider.Updates.GetAsyncEnumerator(cancellationToken); + while (await updates.MoveNextAsync()) + { + if (updates.Current.Version >= expectedVersion) + { + return updates.Current; + } + } + + throw new InvalidOperationException($"The manifest update stream ended before version {expectedVersion} was published."); + } + + private sealed class TestClusterManifestSystemTarget( + Func> getHashSummary, + Func> getUpdate, + Func> getLegacyManifest) : IClusterManifestSystemTarget, ISiloManifestSystemTarget + { + public ValueTask GetClusterManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromException( + new NotSupportedException("This test target only supports peer repair requests.")); + } + + public ValueTask GetClusterManifestUpdate( + MajorMinorVersion previousVersion, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(getUpdate(previousVersion, cancellationToken)); + } + + public ValueTask GetClusterManifestHashSummary(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(getHashSummary(cancellationToken)); + } + + public ValueTask GetSiloManifestHash(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return ValueTask.FromException(new InvalidOperationException("Use the legacy manifest fetch path.")); + } + + public ValueTask GetSiloManifestByHash(ManifestHash hash, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return new((GrainManifest?)null); + } + + public ValueTask GetSiloManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return new(getLegacyManifest(cancellationToken)); + } + } + + private sealed class ManifestRequestLog(int expectedProbeCount, int expectedLegacyFetchCount) + { + private readonly object _lock = new(); + private readonly List _probeAddresses = []; + private readonly List _legacyFetchAddresses = []; + private readonly Dictionary _probeWaiters = []; + private readonly Dictionary _legacyFetchWaiters = []; + + public IReadOnlyList ProbeAddresses + { + get + { + lock (_lock) + { + return _probeAddresses.ToArray(); + } + } + } + + public IReadOnlyList LegacyFetchAddresses + { + get + { + lock (_lock) + { + return _legacyFetchAddresses.ToArray(); + } + } + } + + public void RecordProbe(SiloAddress address) + { + lock (_lock) + { + _probeAddresses.Add(address); + CompleteWaiters(_probeWaiters, _probeAddresses.Count); + } + } + + public void RecordLegacyFetch(SiloAddress address) + { + lock (_lock) + { + _legacyFetchAddresses.Add(address); + CompleteWaiters(_legacyFetchWaiters, _legacyFetchAddresses.Count); + } + } + + public Task WaitForProbeCountAsync(int count, CancellationToken cancellationToken) => + WaitForCountAsync(_probeWaiters, _probeAddresses, count, expectedProbeCount, cancellationToken); + + public Task WaitForLegacyFetchCountAsync(int count, CancellationToken cancellationToken) => + WaitForCountAsync(_legacyFetchWaiters, _legacyFetchAddresses, count, expectedLegacyFetchCount, cancellationToken); + + private Task WaitForCountAsync( + Dictionary waiters, + List addresses, + int count, + int expectedCount, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + Assert.True(count <= expectedCount || expectedCount == 0, $"Expected no more than {expectedCount} requests, but waited for {count}."); + if (addresses.Count >= count) + { + return Task.CompletedTask; + } + + if (!waiters.TryGetValue(count, out var completion)) + { + completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + waiters.Add(count, completion); + } + + return completion.Task.WaitAsync(cancellationToken); + } + } + + private static void CompleteWaiters(Dictionary waiters, int count) + { + foreach (var (expectedCount, completion) in waiters) + { + if (count >= expectedCount) + { + completion.TrySetResult(); + } + } + } + } + + private sealed class PeerProbeLogger(int expectedTimeoutCount) : ILogger + { + private readonly TaskCompletionSource _timeoutsObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _lateFailuresObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _timeoutCount; + private int _lateFailureCount; + + public int TimeoutCount => Volatile.Read(ref _timeoutCount); + + public int LateFailureCount => Volatile.Read(ref _lateFailureCount); + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var message = formatter(state, exception); + if (message.StartsWith("Cluster manifest peer probe to ", StringComparison.Ordinal)) + { + if (Interlocked.Increment(ref _timeoutCount) == expectedTimeoutCount) + { + _timeoutsObserved.TrySetResult(); + } + } + else if (message.StartsWith("Cluster manifest peer probe task for ", StringComparison.Ordinal)) + { + if (Interlocked.Increment(ref _lateFailureCount) == expectedTimeoutCount) + { + _lateFailuresObserved.TrySetResult(); + } + } + } + + public Task WaitForTimeoutCountAsync(int count, CancellationToken cancellationToken) + { + Assert.Equal(expectedTimeoutCount, count); + return _timeoutsObserved.Task.WaitAsync(cancellationToken); + } + + public Task WaitForLateFailureCountAsync(int count, CancellationToken cancellationToken) + { + Assert.Equal(expectedTimeoutCount, count); + return _lateFailuresObserved.Task.WaitAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Timing.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Timing.cs new file mode 100644 index 00000000000..216054834e8 --- /dev/null +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.Timing.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Xunit; + +namespace UnitTests.Manifest; + +public partial class ClusterManifestProviderTests +{ + [Fact] + public async Task ConcurrentColdFetches_PublishCanonicalManifestAfterBodiesComplete() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, peers)); + var manifests = peers.Select(_ => CreateGrainManifest()).ToArray(); + var hash = ManifestHashCalculator.ComputeHash(manifests[0]); + var bodies = peers.Select(_ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var requests = 0; + var factory = Substitute.For(); + for (var index = 0; index < peers.Length; index++) + { + var body = bodies[index]; + var target = Substitute.For(); + target.GetClusterManifestHashSummary(Arg.Any()).Returns(ValueTask.FromException(new NotSupportedException())); + target.GetSiloManifestHash(Arg.Any()).Returns(new ValueTask(hash)); + target.GetSiloManifestByHash(hash, Arg.Any()).Returns(_ => + { + requests++; + return new ValueTask(body.Task); + }); + factory.GetSystemTarget(Constants.ManifestProviderType, peers[index]).Returns(target); + } + + await using var provider = CreateClusterManifestProvider(local, membership, factory); + await InitializeProviderAsync(provider, cancellationToken); + var update = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken); + try + { + Assert.Equal(2, requests); + Assert.False(update.IsCompleted); + bodies[0].SetResult(manifests[0]); + Assert.DoesNotContain(peers[1], provider.Current.Silos.Keys); + bodies[1].SetResult(manifests[1]); + Assert.True(await update.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Equal(manifests[0], provider.Current.Silos[peers[0]]); + Assert.Same(provider.Current.Silos[peers[0]], provider.Current.Silos[peers[1]]); + Assert.Equal(2, GetCachedManifests(provider).Count); + } + finally + { + bodies[0].TrySetResult(manifests[0]); + bodies[1].TrySetResult(manifests[1]); + } + } + + [Fact] + public async Task CompletedDirectRetrieval_PublishesBeforeOptionalPeerDeadline() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + var time = new FakeTimeProvider(); + using var metrics = new ManifestMetrics(); + var requests = new ManifestRequestLog(expectedProbeCount: 2, expectedLegacyFetchCount: 2); + var pendingSummary = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingDirect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tokens = new List(); + var targets = peers.ToDictionary(peer => peer, peer => new TestClusterManifestSystemTarget( + getHashSummary: token => + { + tokens.Add(token); + requests.RecordProbe(peer); + return pendingSummary.Task; + }, + getUpdate: (_, _) => throw new InvalidOperationException("The pending summary has not completed."), + getLegacyManifest: _ => + { + requests.RecordLegacyFetch(peer); + return pendingDirect.Task; + })); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, peers)); + await using var provider = CreateClusterManifestProvider( + local, membership, CreateGrainFactory(targets), time, NullLogger.Instance, metrics.Instruments); + await InitializeProviderAsync(provider, cancellationToken); + var before = time.GetTimestamp(); + var update = UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken); + try + { + await requests.WaitForProbeCountAsync(2, cancellationToken); + await requests.WaitForLegacyFetchCountAsync(2, cancellationToken); + pendingDirect.SetResult(CreateGrainManifest()); + + Assert.True(await update.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Equal(before, time.GetTimestamp()); + Assert.False(pendingSummary.Task.IsCompleted); + Assert.Equal(3, provider.Current.Silos.Count); + Assert.All(tokens, token => Assert.True(token.IsCancellationRequested)); + Assert.Equal(2, metrics.Sum(InstrumentNames.MANIFEST_PEER_PROBES, ("status", "canceled"))); + Assert.Empty(metrics.Find(InstrumentNames.MANIFEST_PEER_REPAIRS)); + var current = provider.Current; + pendingSummary.SetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 1), [])); + Assert.Same(current, provider.Current); + } + finally + { + pendingSummary.TrySetResult(new ClusterManifestHashSummary(new MajorMinorVersion(1, 0), [])); + pendingDirect.TrySetResult(CreateGrainManifest()); + } + } + + [Fact] + public async Task PeerRepairMetrics_CountOnlyPublishedMissingEntries() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSiloAddress(11111, 1); + var peers = new[] { CreateSiloAddress(11112, 1), CreateSiloAddress(11113, 1) }; + using var metrics = new ManifestMetrics(); + var remoteManifest = CreateGrainManifest(); + var hash = ManifestHashCalculator.ComputeHash(remoteManifest); + var summary = new ClusterManifestHashSummary(new MajorMinorVersion(1, 1), peers.ToDictionary(peer => peer, _ => hash)); + var remoteUpdate = new ClusterManifestUpdate( + new MajorMinorVersion(1, 1), peers.ToImmutableDictionary(peer => peer, _ => remoteManifest), true); + var targets = peers.ToDictionary(peer => peer, _ => new TestClusterManifestSystemTarget( + getHashSummary: _ => Task.FromResult(summary), + getUpdate: (_, _) => Task.FromResult(remoteUpdate), + getLegacyManifest: _ => Task.FromException(new InvalidOperationException("Use peer repair.")))); + using var membership = new TestClusterMembershipService(CreateActiveMembershipSnapshot(1, local, peers)); + await using var provider = CreateClusterManifestProvider( + local, membership, CreateGrainFactory(targets), new FakeTimeProvider(), NullLogger.Instance, metrics.Instruments); + await InitializeProviderAsync(provider, cancellationToken); + + Assert.True(await UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken)); + Assert.Equal(2, metrics.Sum(InstrumentNames.MANIFEST_PEER_REPAIRS)); + Assert.Equal(2, metrics.Sum(InstrumentNames.MANIFEST_PEER_PROBES, ("status", "success"))); + Assert.Equal(2, metrics.Sum(InstrumentNames.MANIFEST_CACHE_LOOKUPS, ("result", "miss"), ("source", "peer"))); + Assert.All(peers, peer => Assert.Equal(remoteManifest, provider.Current.Silos[peer])); + + Assert.True(await UpdateManifestAsync(provider, membership.CurrentSnapshot, cancellationToken)); + Assert.Equal(2, metrics.Sum(InstrumentNames.MANIFEST_PEER_REPAIRS)); + Assert.All(metrics.Find(InstrumentNames.MANIFEST_PEER_REPAIRS), measurement => Assert.Empty(measurement.Tags)); + } +} diff --git a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.cs b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.cs index c2d950fb025..c0ee813b6af 100644 --- a/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.cs +++ b/test/Orleans.Core.Tests/Manifest/ClusterManifestProviderTests.cs @@ -3,26 +3,17 @@ using System.Collections.Immutable; using System.Linq; using System.Net; -using System.Reflection; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using NSubstitute; using Orleans.Configuration; using Orleans.Metadata; using Orleans.Runtime; -using Orleans.Runtime.Metadata; -using Orleans.Runtime.Utilities; using Orleans.Runtime.Versions; using Orleans.Runtime.Versions.Compatibility; using Orleans.Runtime.Versions.Selector; -using Orleans.Serialization; -using Orleans.Serialization.Configuration; -using Orleans.Serialization.TypeSystem; using Orleans.Versions.Compatibility; using Orleans.Versions.Selector; using TestExtensions; @@ -33,11 +24,8 @@ namespace UnitTests.Manifest; [TestSuite("BVT")] [TestProvider("None")] [TestCategory("BVT"), TestCategory("Manifest")] -public class ClusterManifestProviderTests +public partial class ClusterManifestProviderTests { - private static readonly GrainType TestGrainType = GrainType.Create("test"); - private static readonly GrainInterfaceType TestInterfaceType = GrainInterfaceType.Create("test.interface"); - [Fact] public void Current_WhenLocalSiloIsNotActive_ResolvesTypeFromLocalManifest() { @@ -92,12 +80,15 @@ public async Task Current_WhenMembershipVersionAdvances_PrunesNonActiveSilosAtFi (remoteSilo, SiloStatus.Active))); var grainFactory = CreateGrainFactory(remoteSilo, remoteManifest); var provider = CreateClusterManifestProvider(localSilo, membership, grainFactory); - var lifecycle = await StartAsync(provider); + var observed = ObserveManifestAsync(provider, new MajorMinorVersion(1, 1), TestContext.Current.CancellationToken); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); try { - await Until(() => provider.Current.Version == new MajorMinorVersion(1, 1) - && provider.Current.Silos.ContainsKey(remoteSilo)); + var initial = await observed.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(new MajorMinorVersion(1, 1), initial.Version); + Assert.Contains(remoteSilo, initial.Silos.Keys); + Assert.Equal(2, GetCachedManifests(provider).Count); membership.Update(CreateMembershipSnapshot( 2, @@ -109,6 +100,7 @@ public async Task Current_WhenMembershipVersionAdvances_PrunesNonActiveSilosAtFi Assert.Equal(new MajorMinorVersion(2, 0), current.Version); Assert.Contains(localSilo, current.Silos.Keys); Assert.DoesNotContain(remoteSilo, current.Silos.Keys); + Assert.Same(provider.LocalGrainManifest, Assert.Single(GetCachedManifests(provider)).Value); } finally { @@ -146,11 +138,11 @@ public async Task Current_WhenRemoteSiloBecomesActive_IncludesLocalManifestBefor Assert.Contains(localSilo, pruned.Silos.Keys); Assert.DoesNotContain(remoteSilo, pruned.Silos.Keys); - var lifecycle = await StartAsync(provider); + var lifecycle = await StartAsync(provider, TestContext.Current.CancellationToken); try { await Until(() => provider.Current.Version == new MajorMinorVersion(2, 1) - && provider.Current.Silos.ContainsKey(remoteSilo)); + && provider.Current.Silos.ContainsKey(remoteSilo), TestContext.Current.CancellationToken); } finally { @@ -159,31 +151,6 @@ public async Task Current_WhenRemoteSiloBecomesActive_IncludesLocalManifestBefor } } - [Fact] - public async Task ClientProvider_UpdateCancellation_DoesNotFetchLegacyManifest() - { - var provider = (ClientClusterManifestProvider)RuntimeHelpers.GetUninitializedObject( - typeof(ClientClusterManifestProvider)); - var remoteProvider = Substitute.For(); - using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); - cancellation.Cancel(); - remoteProvider - .GetClusterManifestUpdate(default, cancellation.Token) - .Returns(_ => new ValueTask( - Task.FromCanceled(cancellation.Token))); - var method = typeof(ClientClusterManifestProvider).GetMethod( - "GetClusterManifestUpdate", - BindingFlags.Instance | BindingFlags.NonPublic)!; - var task = (Task)method.Invoke( - provider, - [remoteProvider, default(MajorMinorVersion), cancellation.Token])!; - - await Assert.ThrowsAnyAsync(() => task); - - var call = Assert.Single(remoteProvider.ReceivedCalls()); - Assert.Equal(nameof(IClusterManifestSystemTarget.GetClusterManifestUpdate), call.GetMethodInfo().Name); - } - [Fact] public void GrainVersionManifest_UpdatesSupportedSilosWhenClusterManifestVersionChanges() { @@ -342,41 +309,6 @@ public async Task CachedVersionSelectorManager_SerializesRefreshesForTheSameKey( Assert.Equal(2, selector.CallCount); } - private static ClusterManifestProvider CreateClusterManifestProvider( - SiloAddress localSilo, - TestClusterMembershipService membership, - IInternalGrainFactory grainFactory) - { - var siloManifestProvider = CreateSiloManifestProvider(); - grainFactory - .GetSystemTarget(Constants.ManifestProviderType, localSilo) - .Returns(new TestSiloManifestSystemTarget(siloManifestProvider.SiloManifest)); - - var services = new ServiceCollection() - .AddSingleton(grainFactory) - .BuildServiceProvider(); - - var localSiloDetails = Substitute.For(); - localSiloDetails.SiloAddress.Returns(localSilo); - - return new ClusterManifestProvider( - localSiloDetails, - siloManifestProvider, - membership, - Substitute.For(), - NullLogger.Instance, - services); - } - - private static IInternalGrainFactory CreateGrainFactory(SiloAddress remoteSilo, GrainManifest remoteManifest) - { - var grainFactory = Substitute.For(); - grainFactory - .GetSystemTarget(Constants.ManifestProviderType, remoteSilo) - .Returns(new TestSiloManifestSystemTarget(remoteManifest)); - return grainFactory; - } - private static CachedVersionSelectorManager CreateCachedVersionSelectorManager(GrainVersionManifest manifest) { var services = new ServiceCollection(); @@ -402,189 +334,17 @@ private static ClusterManifest CreateClusterManifest(long major, long minor, par silos.ToImmutableDictionary(silo => silo, _ => manifest)); } - private static GrainManifest CreateGrainManifest() - { - var grains = ImmutableDictionary.CreateRange( - [ - new KeyValuePair( - TestGrainType, - new GrainProperties(CreatePropertyDictionary( - [ - new KeyValuePair(WellKnownGrainTypeProperties.TypeName, "Test"), - new KeyValuePair(WellKnownGrainTypeProperties.FullTypeName, "UnitTests.Grains.Test"), - new KeyValuePair($"{WellKnownGrainTypeProperties.ImplementedInterfacePrefix}0", TestInterfaceType.ToString()) - ]))) - ]); - var interfaces = ImmutableDictionary.CreateRange( - [ - new KeyValuePair( - TestInterfaceType, - new GrainInterfaceProperties(CreatePropertyDictionary( - [ - new KeyValuePair(WellKnownGrainInterfaceProperties.TypeName, "ITest"), - new KeyValuePair(WellKnownGrainInterfaceProperties.Version, "1") - ]))) - ]); - - return new GrainManifest(grains, interfaces); - } - - private static ImmutableDictionary CreatePropertyDictionary(params KeyValuePair[] properties) - { - var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal, StringComparer.Ordinal); - foreach (var property in properties) - { - builder.Add(property.Key, property.Value); - } - - return builder.ToImmutable(); - } - - private static SiloManifestProvider CreateSiloManifestProvider() - { - var typeConverter = CreateTypeConverter(); - var interfaceTypeResolver = new GrainInterfaceTypeResolver([new TestGrainInterfaceTypeProvider()], typeConverter); - var typeNameProvider = new TypeNameGrainPropertiesProvider(); - var options = new GrainTypeOptions(); - options.Classes.Add(typeof(TestManifestGrain)); - options.Interfaces.Add(typeof(ITestManifestGrain)); - - return new SiloManifestProvider( - [typeNameProvider, new ImplementedInterfaceProvider(interfaceTypeResolver)], - [typeNameProvider, new TestGrainInterfacePropertiesProvider()], - Options.Create(options), - new GrainTypeResolver([new TestGrainTypeProvider()], typeConverter), - interfaceTypeResolver, - typeConverter); - } - - internal interface ITestManifestGrain : IGrainWithStringKey; - - internal sealed class TestManifestGrain : ITestManifestGrain; - - private sealed class TestGrainTypeProvider : IGrainTypeProvider - { - public bool TryGetGrainType(Type type, out GrainType grainType) - { - if (type == typeof(TestManifestGrain)) - { - grainType = TestGrainType; - return true; - } - - grainType = default; - return false; - } - } - - private sealed class TestGrainInterfaceTypeProvider : IGrainInterfaceTypeProvider - { - public bool TryGetGrainInterfaceType(Type type, out GrainInterfaceType grainInterfaceType) - { - if (type == typeof(ITestManifestGrain)) - { - grainInterfaceType = TestInterfaceType; - return true; - } - - grainInterfaceType = default; - return false; - } - } - - private sealed class TestGrainInterfacePropertiesProvider : IGrainInterfacePropertiesProvider - { - public void Populate(Type interfaceType, GrainInterfaceType grainInterfaceType, Dictionary properties) - { - properties[WellKnownGrainInterfaceProperties.Version] = "1"; - } - } - - private static Orleans.Serialization.TypeSystem.TypeConverter CreateTypeConverter() - { - return new Orleans.Serialization.TypeSystem.TypeConverter( - Array.Empty(), - Array.Empty(), - Array.Empty(), - Options.Create(new TypeManifestOptions { AllowAllTypes = true }), - new CachedTypeResolver()); - } - - private static ClusterMembershipSnapshot CreateMembershipSnapshot( - long version, - params (SiloAddress SiloAddress, SiloStatus Status)[] members) - { - var builder = ImmutableDictionary.CreateBuilder(); - foreach (var (siloAddress, status) in members) - { - builder[siloAddress] = new ClusterMember(siloAddress, status, siloAddress.ToString()); - } - - return new ClusterMembershipSnapshot(builder.ToImmutable(), new MembershipVersion(version)); - } - - private static SiloAddress CreateSiloAddress(int port, int generation) - { - return SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), generation); - } - - private static async Task StartAsync(ClusterManifestProvider provider) - { - var lifecycle = new SiloLifecycleSubject(NullLoggerFactory.Instance.CreateLogger()); - ((ILifecycleParticipant)provider).Participate(lifecycle); - await lifecycle.OnStart(); - return lifecycle; - } - - private static async Task Until(Func condition) + private static async Task Until(Func condition, CancellationToken cancellationToken) { var timeout = 10_000; while (!condition() && (timeout -= 10) > 0) { - await Task.Delay(10); + await Task.Delay(10, cancellationToken); } Assert.True(timeout > 0); } - private sealed class TestClusterMembershipService : IClusterMembershipService, IDisposable - { - private readonly AsyncEnumerable _updates; - private ClusterMembershipSnapshot _currentSnapshot = ClusterMembershipSnapshot.Default; - - public TestClusterMembershipService(ClusterMembershipSnapshot initialSnapshot) - { - _updates = new AsyncEnumerable( - initialValue: initialSnapshot, - updateValidator: (previous, proposed) => proposed.Version > previous.Version, - onPublished: update => Volatile.Write(ref _currentSnapshot, update)); - } - - public ClusterMembershipSnapshot CurrentSnapshot - { - get => Volatile.Read(ref _currentSnapshot); - } - - public IAsyncEnumerable MembershipUpdates => _updates; - - public void Update(ClusterMembershipSnapshot snapshot) => _updates.Publish(snapshot); - - public ValueTask Refresh(MembershipVersion minimumVersion = default, CancellationToken cancellationToken = default) => default; - - public Task TryKill(SiloAddress siloAddress) => Task.FromResult(false); - - public void Dispose() => _updates.Dispose(); - } - - private sealed class TestSiloManifestSystemTarget(GrainManifest manifest) : ISiloManifestSystemTarget - { - public ValueTask GetSiloManifest(CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - return new(manifest); - } - } - private sealed class BlockingVersionSelector : IVersionSelector { private readonly TaskCompletionSource _entered = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -616,12 +376,13 @@ private sealed class TestClusterManifestProvider(ClusterManifest initialManifest { public ClusterManifest Current { get; set; } = initialManifest; - public IAsyncEnumerable Updates => GetUpdates(); + public IAsyncEnumerable Updates => GetUpdates(TestContext.Current.CancellationToken); public GrainManifest LocalGrainManifest { get; } = CreateGrainManifest(); - private async IAsyncEnumerable GetUpdates() + private async IAsyncEnumerable GetUpdates([EnumeratorCancellation] CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); yield return Current; await Task.CompletedTask; } diff --git a/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestHashSummarySerializationTests.cs b/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestHashSummarySerializationTests.cs new file mode 100644 index 00000000000..b6c94bad2b2 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestHashSummarySerializationTests.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Threading; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Manifest; + +[TestSuite("BVT"), TestProvider("None")] +[TestCategory("BVT"), TestCategory("Serialization")] +public sealed class ClusterManifestHashSummarySerializationTests +{ + [Fact] + public void ManifestRpcContractsExposeCancellationTokens() + { + foreach (var contract in new[] { typeof(IClusterManifestSystemTarget), typeof(ISiloManifestSystemTarget) }) + { + foreach (var method in contract.GetMethods()) + { + var parameters = method.GetParameters(); + var cancellation = Assert.Single(parameters, parameter => parameter.ParameterType == typeof(CancellationToken)); + Assert.Equal("cancellationToken", cancellation.Name); + Assert.Equal(parameters[^1], cancellation); + if (method.Name is nameof(IClusterManifestSystemTarget.GetClusterManifestHashSummary) + or nameof(IClusterManifestSystemTarget.GetSiloManifestHash) + or nameof(IClusterManifestSystemTarget.GetSiloManifestByHash)) + { + Assert.False(cancellation.HasDefaultValue); + } + } + } + } + + [Fact] + public void ManifestHashCalculatorReusesHashForImmutableManifest() + { + var manifest = new GrainManifest( + ImmutableDictionary.Empty, + ImmutableDictionary.Empty); + + var first = ManifestHashCalculator.ComputeHash(manifest); + var second = ManifestHashCalculator.ComputeHash(manifest); + + Assert.Same(first.Value, second.Value); + Assert.Equal("163E7F66D58036CCB1D0B0058D8F46E7CD639816F570E5EB32853EA73634E4CD", first.Value); + } + + [Fact] + public void ClusterManifestHashSummaryRoundTripsThroughSerializer() + { + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + + var siloA = SiloAddress.New(IPAddress.Loopback, 11111, 1); + var siloB = SiloAddress.New(IPAddress.Loopback, 11112, 2); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(3, 7), + new Dictionary + { + [siloA] = new ManifestHash("hash-a"), + [siloB] = new ManifestHash("hash-b"), + }); + + var roundTripped = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(summary))); + + Assert.Equal(summary.Version, roundTripped.Version); + Assert.Equal(summary.SiloManifestHashes.Count, roundTripped.SiloManifestHashes.Count); + Assert.Equal(new ManifestHash("hash-a"), roundTripped.SiloManifestHashes[siloA]); + Assert.Equal(new ManifestHash("hash-b"), roundTripped.SiloManifestHashes[siloB]); + } + + [Fact] + public void ClusterManifestHashSummaryRoundTripsHashAndSiloAddress() + { + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var silo = SiloAddress.New(IPAddress.Parse("127.0.0.42"), 23456, 7); + var expectedHash = new ManifestHash("sha256:0123456789abcdef"); + var summary = new ClusterManifestHashSummary( + new MajorMinorVersion(11, 13), + new Dictionary { [silo] = expectedHash }); + + var result = serializer.Deserialize( + serializer.SerializeToArray(summary)); + + Assert.Equal(new MajorMinorVersion(11, 13), result!.Version); + var entry = Assert.Single(result.SiloManifestHashes); + Assert.Equal(silo, entry.Key); + Assert.Equal(expectedHash, entry.Value); + Assert.Equal("sha256:0123456789abcdef", entry.Value.Value); + } + + [Fact] + public void ClusterManifestHashFetchContractPreservesManifestHashIdentity() + { + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var expected = new ManifestHash("sha256:fedcba9876543210"); + + var result = serializer.Deserialize(serializer.SerializeToArray(expected)); + var method = typeof(IClusterManifestSystemTarget).GetMethod( + nameof(IClusterManifestSystemTarget.GetSiloManifestByHash)); + + Assert.Equal(expected, result); + Assert.Equal("sha256:fedcba9876543210", result.Value); + Assert.NotNull(method); + Assert.Equal(typeof(ValueTask), method!.ReturnType); + Assert.Collection( + method.GetParameters(), + parameter => + { + Assert.Equal("hash", parameter.Name); + Assert.Equal(typeof(ManifestHash), parameter.ParameterType); + }, + parameter => + { + Assert.Equal("cancellationToken", parameter.Name); + Assert.Equal(typeof(CancellationToken), parameter.ParameterType); + }); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestLifecycleTests.cs b/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestLifecycleTests.cs new file mode 100644 index 00000000000..be59f73d70b --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Manifest/ClusterManifestLifecycleTests.cs @@ -0,0 +1,96 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Metadata; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.Manifest; + +[TestSuite("BVT"), TestProvider("None")] +[TestCategory("BVT"), TestCategory("Manifest")] +public sealed class ClusterManifestLifecycleTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DefaultProviders_ConvergeAcrossJoinsDeparturesAndRollingModeChanges(bool mixedModes) + { + var cancellationToken = TestContext.Current.CancellationToken; + var overrides = new ConcurrentDictionary(); + var builder = new InProcessTestClusterBuilder(2); + builder.ConfigureHost(host => TestDefaultConfiguration.ConfigureHostConfiguration(host.Configuration)); + builder.ConfigureSilo((specific, silo) => + { + if (overrides.TryGetValue(specific.SiloName, out var configured)) + { + silo.Configure(options => options.EnableContentAddressedRetrieval = configured); + } + else if (mixedModes && specific.SiloName != "Silo_0") + { + silo.Configure(options => options.EnableContentAddressedRetrieval = false); + } + }); + await using var cluster = builder.Build(); + await cluster.DeployAsync(cancellationToken); + await AssertConvergedAsync(cluster, "initial deployment", cancellationToken); + Assert.True(cluster.Silos[0].ServiceProvider.GetRequiredService>().Value.EnableContentAddressedRetrieval); + Assert.Equal(!mixedModes, cluster.Silos[1].ServiceProvider.GetRequiredService>().Value.EnableContentAddressedRetrieval); + + var joined = await cluster.StartAdditionalSiloAsync().WaitAsync(cancellationToken); + await AssertConvergedAsync(cluster, "additional silo join", cancellationToken); + await cluster.StopSiloAsync(joined, cancellationToken); + await AssertConvergedAsync(cluster, "additional silo departure", cancellationToken); + + var rolling = cluster.Silos[1]; + var previousAddress = rolling.SiloAddress; + overrides[rolling.Name] = mixedModes; + rolling = Assert.IsType(await cluster.RestartSiloAsync(rolling).WaitAsync(cancellationToken)); + Assert.NotEqual(previousAddress, rolling.SiloAddress); + Assert.Equal(mixedModes, rolling.ServiceProvider.GetRequiredService>().Value.EnableContentAddressedRetrieval); + await AssertConvergedAsync(cluster, "rolling mode change", cancellationToken); + + overrides[rolling.Name] = !mixedModes; + rolling = Assert.IsType(await cluster.RestartSiloAsync(rolling).WaitAsync(cancellationToken)); + Assert.Equal(!mixedModes, rolling.ServiceProvider.GetRequiredService>().Value.EnableContentAddressedRetrieval); + await AssertConvergedAsync(cluster, "rolling mode rollback", cancellationToken); + } + + private static async Task AssertConvergedAsync(InProcessTestCluster cluster, string phase, CancellationToken cancellationToken) + { + var silos = cluster.GetActiveSilos().ToArray(); + var expected = silos.ToDictionary(silo => silo.SiloAddress, silo => silo.ServiceProvider.GetRequiredService().LocalGrainManifest); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(30)); + var observations = silos.Select(async silo => + { + var provider = silo.ServiceProvider.GetRequiredService(); + await using var updates = provider.Updates.GetAsyncEnumerator(deadline.Token); + while (await updates.MoveNextAsync()) + { + var current = updates.Current; + if (current.Silos.Count == expected.Count && expected.All(entry => + current.Silos.TryGetValue(entry.Key, out var manifest) && manifest.Equals(entry.Value))) + { + Assert.Equal(provider.LocalGrainManifest, current.Silos[silo.SiloAddress]); + return; + } + } + + Assert.Fail($"Manifest update stream ended during {phase} on {silo.SiloAddress}."); + }).ToArray(); + + try + { + await Task.WhenAll(observations); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new TimeoutException($"Manifest convergence failed during {phase}. Expected: {string.Join(", ", expected.Keys)}. " + + string.Join("; ", silos.Select(silo => + $"{silo.SiloAddress}: {string.Join(", ", silo.ServiceProvider.GetRequiredService().Current.Silos.Keys)}"))); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestHashCalculatorTests.cs b/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestHashCalculatorTests.cs new file mode 100644 index 00000000000..42016cd1d42 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestHashCalculatorTests.cs @@ -0,0 +1,379 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using Orleans.Metadata; +using Orleans.Runtime; +using Orleans.Runtime.Metadata; +using Xunit; + +namespace UnitTests.Manifest; + +[TestSuite("BVT"), TestProvider("None")] +[TestCategory("BVT"), TestCategory("Manifest")] +public sealed class ManifestHashCalculatorTests +{ + [Fact] + public void ManifestHashIsIndependentOfDictionaryOrdering() + { + var manifest1 = CreateManifest( + ("grain-b", "placement", "random"), + ("grain-a", "placement", "local")); + var manifest2 = CreateManifest( + ("grain-a", "placement", "local"), + ("grain-b", "placement", "random")); + + Assert.Equal(ManifestHashCalculator.ComputeHash(manifest1), ManifestHashCalculator.ComputeHash(manifest2)); + } + + [Fact] + public void ManifestHashIncludesEntryBoundaries() + { + var manifest1 = CreateManifestWithProperties( + ("a", [("b", "c")]), + ("d", [("e", "f"), ("g", "h")])); + var manifest2 = CreateManifestWithProperties( + ("a", [("b", "c"), ("d", "e")]), + ("f", [("g", "h")])); + + Assert.NotEqual(ManifestHashCalculator.ComputeHash(manifest1), ManifestHashCalculator.ComputeHash(manifest2)); + } + + [Fact] + public void ManifestHashUsesRawTypeIdentifierBytes() + { + var properties = new GrainProperties( + ImmutableDictionary.Empty + .WithComparers(StringComparer.Ordinal)); + var first = new GrainManifest( + ImmutableDictionary.Empty + .Add(new GrainType([0x80]), properties), + ImmutableDictionary.Empty); + var second = new GrainManifest( + ImmutableDictionary.Empty + .Add(new GrainType([0x81]), properties), + ImmutableDictionary.Empty); + + Assert.NotEqual(ManifestHashCalculator.ComputeHash(first), ManifestHashCalculator.ComputeHash(second)); + } + + [Fact] + public void ManifestHashDistinguishesNullAndEmptyPropertyValues() + { + var nullProperties = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + nullProperties["value"] = null!; + var emptyProperties = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + emptyProperties["value"] = string.Empty; + var nullManifest = new GrainManifest( + ImmutableDictionary.Empty + .Add(GrainType.Create("grain"), new GrainProperties(nullProperties.ToImmutable())), + ImmutableDictionary.Empty); + var emptyManifest = new GrainManifest( + ImmutableDictionary.Empty + .Add(GrainType.Create("grain"), new GrainProperties(emptyProperties.ToImmutable())), + ImmutableDictionary.Empty); + + Assert.NotEqual( + ManifestHashCalculator.ComputeHash(nullManifest), + ManifestHashCalculator.ComputeHash(emptyManifest)); + } + + [Fact] + public void ManifestHashIncludesCanonicalEncodingVersion() + { + var manifest = CreatePhaseOneGrainManifest( + new GrainType([0x80, 0x00]), + ("k", "\uD800")); + + var actual = ManifestHashCalculator.ComputeHash(manifest); + + Assert.Equal("DD34377AD1C69045F2B55D6DCAB099A7A36C0A6B6909DC2C54D5B6AC1FA9EBF9", actual.Value); + + // The same canonical input with only its encoding version changed from 2 to 3. + byte[] versionThreeInput = + [ + 0, 0, 0, 3, + 0, 0, 0, 1, + 0, 0, 0, 2, 0x80, 0x00, + 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0x6B, + 0, 0, 0, 1, 0xD8, 0, + 0, 0, 0, 0, + ]; + var versionThreeHash = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(versionThreeInput)); + + Assert.Equal("C2CF5517ABADEF216C52FEDB4D9C4F36681DA429457CA8848EF921083C532A83", versionThreeHash); + Assert.NotEqual(actual.Value, versionThreeHash); + } + + [Theory] + [InlineData("a", "bc", "ab", "c")] + [InlineData("", "abc", "a", "bc")] + [InlineData("abc", "", "ab", "c")] + public void ManifestHashLengthPrefixesSeparatePropertyKeysAndValues( + string firstKey, string firstValue, string secondKey, string secondValue) + { + Assert.Equal(firstKey + firstValue, secondKey + secondValue); + var first = CreatePhaseOneGrainManifest(GrainType.Create("grain"), (firstKey, firstValue)); + var second = CreatePhaseOneGrainManifest(GrainType.Create("grain"), (secondKey, secondValue)); + + Assert.NotEqual(ManifestHashCalculator.ComputeHash(first), ManifestHashCalculator.ComputeHash(second)); + } + + [Fact] + public void ManifestHashDistinguishesNestedStructureBoundaries() + { + var first = CreatePhaseOneGrainManifest( + (new GrainType([0x61]), [("b", "c")]), + (new GrainType([0x64]), [("e", "f"), ("g", "h")])); + var second = CreatePhaseOneGrainManifest( + (new GrainType([0x61]), [("b", "c"), ("d", "e")]), + (new GrainType([0x66]), [("g", "h")])); + + var firstHash = ManifestHashCalculator.ComputeHash(first); + var secondHash = ManifestHashCalculator.ComputeHash(second); + + Assert.NotEqual(firstHash, secondHash); + Assert.Equal(firstHash, ManifestHashCalculator.ComputeHash(first)); + Assert.Equal(secondHash, ManifestHashCalculator.ComputeHash(second)); + } + + [Fact] + public void ManifestHashDistinguishesGrainAndInterfaceSections() + { + byte[] identifier = [0x78, 0x00, 0x80]; + var grainManifest = CreatePhaseOneGrainManifest( + new GrainType(identifier), + ("field", "value")); + var interfaceManifest = CreatePhaseOneInterfaceManifest( + new GrainInterfaceType(new IdSpan(identifier)), + ("field", "value")); + + var grainHash = ManifestHashCalculator.ComputeHash(grainManifest); + var interfaceHash = ManifestHashCalculator.ComputeHash(interfaceManifest); + + Assert.NotEqual(grainHash, interfaceHash); + Assert.Equal(grainHash, ManifestHashCalculator.ComputeHash(grainManifest)); + Assert.Equal(interfaceHash, ManifestHashCalculator.ComputeHash(interfaceManifest)); + } + + [Fact] + public void ManifestHashPreservesInvalidUtf16CodeUnits() + { + var first = CreatePhaseOneGrainManifest( + GrainType.Create("invalid-utf16"), + ("value", "\uD800x\uDC00")); + var second = CreatePhaseOneGrainManifest( + GrainType.Create("invalid-utf16"), + ("value", "\uD801x\uDC01")); + + var firstHash = ManifestHashCalculator.ComputeHash(first); + var secondHash = ManifestHashCalculator.ComputeHash(second); + + Assert.NotEqual(firstHash, secondHash); + Assert.Equal(firstHash, ManifestHashCalculator.ComputeHash(first)); + Assert.Equal(secondHash, ManifestHashCalculator.ComputeHash(second)); + } + + [Fact] + public void ManifestHashUsesArbitraryRawGrainTypeIdentifierBytes() + { + var first = CreatePhaseOneGrainManifest( + new GrainType([0x80, 0x00, 0xFE]), + ("key", "value")); + var second = CreatePhaseOneGrainManifest( + new GrainType([0x80, 0x00, 0xFF]), + ("key", "value")); + + var firstHash = ManifestHashCalculator.ComputeHash(first); + var secondHash = ManifestHashCalculator.ComputeHash(second); + + Assert.NotEqual(firstHash, secondHash); + Assert.Equal(firstHash, ManifestHashCalculator.ComputeHash(first)); + } + + [Fact] + public void ManifestHashUsesArbitraryRawInterfaceTypeIdentifierBytes() + { + var first = CreatePhaseOneInterfaceManifest( + new GrainInterfaceType(new IdSpan([0x80, 0x00, 0xFE])), + ("key", "value")); + var second = CreatePhaseOneInterfaceManifest( + new GrainInterfaceType(new IdSpan([0x80, 0x00, 0xFF])), + ("key", "value")); + + var firstHash = ManifestHashCalculator.ComputeHash(first); + var secondHash = ManifestHashCalculator.ComputeHash(second); + + Assert.NotEqual(firstHash, secondHash); + Assert.Equal(secondHash, ManifestHashCalculator.ComputeHash(second)); + } + + [Fact] + public void ManifestHashDistinguishesNullAndEmptyStrings() + { + var nullManifest = CreatePhaseOneGrainManifest( + GrainType.Create("null-string"), + ("value", null)); + var emptyManifest = CreatePhaseOneGrainManifest( + GrainType.Create("null-string"), + ("value", string.Empty)); + + var nullHash = ManifestHashCalculator.ComputeHash(nullManifest); + var emptyHash = ManifestHashCalculator.ComputeHash(emptyManifest); + + Assert.NotEqual(nullHash, emptyHash); + Assert.Equal(nullHash, ManifestHashCalculator.ComputeHash(nullManifest)); + Assert.Equal(emptyHash, ManifestHashCalculator.ComputeHash(emptyManifest)); + } + + [Fact] + public void ManifestHashDistinguishesDefaultAndEmptyGrainTypeIdentifiers() + { + var defaultManifest = CreatePhaseOneGrainManifest(default, ("key", "value")); + var emptyManifest = CreatePhaseOneGrainManifest(new GrainType([]), ("key", "value")); + + var defaultHash = ManifestHashCalculator.ComputeHash(defaultManifest); + var emptyHash = ManifestHashCalculator.ComputeHash(emptyManifest); + + Assert.NotEqual(defaultHash, emptyHash); + Assert.Equal(defaultHash, ManifestHashCalculator.ComputeHash(defaultManifest)); + Assert.Equal(emptyHash, ManifestHashCalculator.ComputeHash(emptyManifest)); + } + + [Fact] + public void ManifestHashDistinguishesDefaultAndEmptyInterfaceTypeIdentifiers() + { + var defaultManifest = CreatePhaseOneInterfaceManifest(default, ("key", "value")); + var emptyManifest = CreatePhaseOneInterfaceManifest( + new GrainInterfaceType(new IdSpan([])), + ("key", "value")); + + var defaultHash = ManifestHashCalculator.ComputeHash(defaultManifest); + var emptyHash = ManifestHashCalculator.ComputeHash(emptyManifest); + + Assert.NotEqual(defaultHash, emptyHash); + Assert.Equal(defaultHash, ManifestHashCalculator.ComputeHash(defaultManifest)); + Assert.Equal(emptyHash, ManifestHashCalculator.ComputeHash(emptyManifest)); + } + + [Fact] + public void ManifestHashIsStableAcrossEquivalentInsertionOrders() + { + var first = CreatePhaseOneOrderedManifest(reverse: false); + var second = CreatePhaseOneOrderedManifest(reverse: true); + + var firstHash = ManifestHashCalculator.ComputeHash(first); + var secondHash = ManifestHashCalculator.ComputeHash(second); + + Assert.Equal(firstHash, secondHash); + Assert.Equal(firstHash, ManifestHashCalculator.ComputeHash(first)); + Assert.Equal(secondHash, ManifestHashCalculator.ComputeHash(second)); + } + + private static GrainManifest CreateManifest(params (string Grain, string Key, string Value)[] grains) + { + var grainBuilder = ImmutableDictionary.CreateBuilder(); + foreach (var grain in grains) + { + var properties = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + properties[grain.Key] = grain.Value; + grainBuilder[GrainType.Create(grain.Grain)] = new GrainProperties( + properties.ToImmutable()); + } + + return new GrainManifest( + grainBuilder.ToImmutable(), + ImmutableDictionary.Empty); + } + + private static GrainManifest CreateManifestWithProperties(params (string Grain, (string Key, string Value)[] Properties)[] grains) + { + var grainBuilder = ImmutableDictionary.CreateBuilder(); + foreach (var grain in grains) + { + var properties = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var property in grain.Properties) + { + properties[property.Key] = property.Value; + } + + grainBuilder[GrainType.Create(grain.Grain)] = new GrainProperties(properties.ToImmutable()); + } + + return new GrainManifest( + grainBuilder.ToImmutable(), + ImmutableDictionary.Empty); + } + + private static GrainManifest CreatePhaseOneGrainManifest( + GrainType grainType, + params (string Key, string? Value)[] properties) => + CreatePhaseOneGrainManifest((grainType, properties)); + + private static GrainManifest CreatePhaseOneGrainManifest( + params (GrainType GrainType, (string Key, string? Value)[] Properties)[] grains) + { + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var grain in grains) + { + builder.Add(grain.GrainType, new GrainProperties(CreatePhaseOneProperties(grain.Properties))); + } + + return new GrainManifest( + builder.ToImmutable(), + ImmutableDictionary.Empty); + } + + private static GrainManifest CreatePhaseOneInterfaceManifest( + GrainInterfaceType interfaceType, + params (string Key, string? Value)[] properties) + { + var interfaces = ImmutableDictionary.Empty + .Add(interfaceType, new GrainInterfaceProperties(CreatePhaseOneProperties(properties))); + return new GrainManifest( + ImmutableDictionary.Empty, + interfaces); + } + + private static ImmutableDictionary CreatePhaseOneProperties( + params (string Key, string? Value)[] properties) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var property in properties) + { + builder.Add(property.Key, property.Value!); + } + + return builder.ToImmutable(); + } + + private static GrainManifest CreatePhaseOneOrderedManifest(bool reverse) + { + var grainEntries = new[] + { + (GrainType.Create("grain-a"), CreatePhaseOneProperties(("z", "last"), ("a", "first"))), + (GrainType.Create("grain-b"), CreatePhaseOneProperties(("version", "2"))), + }; + var interfaceEntries = new[] + { + (GrainInterfaceType.Create("interface-a"), CreatePhaseOneProperties(("z", "last"), ("a", "first"))), + (GrainInterfaceType.Create("interface-b"), CreatePhaseOneProperties(("version", "2"))), + }; + var grains = ImmutableDictionary.CreateBuilder(); + var interfaces = ImmutableDictionary.CreateBuilder(); + + foreach (var (grainType, properties) in reverse ? grainEntries.AsEnumerable().Reverse() : grainEntries) + { + var entries = reverse ? properties.Reverse() : properties; + grains.Add(grainType, new GrainProperties(entries.ToImmutableDictionary(StringComparer.Ordinal))); + } + + foreach (var (interfaceType, properties) in reverse ? interfaceEntries.AsEnumerable().Reverse() : interfaceEntries) + { + var entries = reverse ? properties.Reverse() : properties; + interfaces.Add(interfaceType, new GrainInterfaceProperties(entries.ToImmutableDictionary(StringComparer.Ordinal))); + } + + return new GrainManifest(grains.ToImmutable(), interfaces.ToImmutable()); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestPeerProbeTests.cs b/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestPeerProbeTests.cs new file mode 100644 index 00000000000..aae68d628d0 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Manifest/ManifestPeerProbeTests.cs @@ -0,0 +1,275 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Metadata; +using Orleans.Runtime.Metadata; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.Manifest; + +[TestSuite("BVT")] +[TestProvider("None")] +[TestCategory("BVT"), TestCategory("Manifest")] +public sealed class ManifestPeerProbeTests(ManifestPeerProbeTests.Fixture fixture) : IClassFixture +{ + private static readonly GrainType TargetType = SystemTargetGrainId.CreateGrainType("manifest-probe-test"); + private static readonly GrainType LegacyTargetType = SystemTargetGrainId.CreateGrainType("legacy-manifest-test"); + + public sealed class Fixture : BaseInProcessTestClusterFixture + { + protected override void ConfigureTestCluster(InProcessTestClusterBuilder builder) + { + builder.ConfigureSilo((_, silo) => + { + silo.Services.AddSingleton(); + silo.Services.AddSingleton>(services => services.GetRequiredService()); + silo.Services.AddSingleton(); + silo.Services.AddSingleton>(services => services.GetRequiredService()); + }); + } + } + + [Fact] + public async Task DefaultOptions_LegacyOnlyPeerFallsBackThroughRealGeneratedProxy() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = fixture.HostedCluster.Silos[0]; + var remote = fixture.HostedCluster.Silos[1]; + var actualFactory = local.ServiceProvider.GetRequiredService(); + var hashProxy = actualFactory.GetSystemTarget(LegacyTargetType, remote.SiloAddress); + var legacyProxy = actualFactory.GetSystemTarget(LegacyTargetType, remote.SiloAddress); + var factory = Substitute.For(); + factory.GetSystemTarget(Constants.ManifestProviderType, remote.SiloAddress).Returns(hashProxy); + factory.GetSystemTarget(Constants.ManifestProviderType, remote.SiloAddress).Returns(legacyProxy); + using var services = new ServiceCollection().AddSingleton(factory).BuildServiceProvider(); + var membership = local.ServiceProvider.GetRequiredService(); + await using var provider = new ClusterManifestProvider( + local.ServiceProvider.GetRequiredService(), + local.ServiceProvider.GetRequiredService(), + membership, + local.ServiceProvider.GetRequiredService(), + NullLogger.Instance, + services, + TimeProvider.System, + Options.Create(new ClusterManifestOptions()), + local.ServiceProvider.GetRequiredService()); + var target = remote.ServiceProvider.GetRequiredService(); + var previousRequests = target.Requests; + var initialize = typeof(ClusterManifestProvider).GetMethod("Initialize", BindingFlags.Instance | BindingFlags.NonPublic)!; + await (Task)initialize.Invoke(provider, [cancellationToken])!; + var update = (Task)typeof(ClusterManifestProvider).GetMethod("UpdateManifest", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(provider, [membership.CurrentSnapshot, cancellationToken])!; + + Assert.True(await update.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken)); + Assert.Equal(previousRequests + 1, target.Requests); + Assert.Equal(target.Manifest, provider.Current.Silos[remote.SiloAddress]); + Assert.Equal(2, provider.Current.Silos.Count); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task LocalCompletion_ReleasesSlotsAndSignalsPeerCancellation(bool waitForUpdate, bool cancelCaller) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = fixture.HostedCluster.Silos[0]; + var remote = fixture.HostedCluster.Silos[1]; + var localServices = local.ServiceProvider; + Assert.False(localServices.GetRequiredService>().Value.WaitForCancellationAcknowledgement); + var actualFactory = localServices.GetRequiredService(); + var proxy = actualFactory.GetSystemTarget(TargetType, remote.SiloAddress); + var factory = Substitute.For(); + factory.GetSystemTarget(Constants.ManifestProviderType, remote.SiloAddress).Returns(proxy); + using var services = new ServiceCollection().AddSingleton(factory).BuildServiceProvider(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var time = new FakeTimeProvider(); + await using var provider = new ClusterManifestProvider( + localServices.GetRequiredService(), + localServices.GetRequiredService(), + localServices.GetRequiredService(), + localServices.GetRequiredService(), + NullLogger.Instance, + services, + time, + Options.Create(new ClusterManifestOptions()), + localServices.GetRequiredService()); + var initialize = typeof(ClusterManifestProvider).GetMethod("Initialize", BindingFlags.Instance | BindingFlags.NonPublic)!; + await (Task)initialize.Invoke(provider, [cancellationToken])!; + var target = remote.ServiceProvider.GetRequiredService(); + var scenario = new ProbeScenario(waitForUpdate); + target.Scenario = scenario; + var initialSummaryRequests = target.SummaryRequests; + var probes = Enumerable.Range(0, 3).Select(_ => ProbeAsync(provider, remote.SiloAddress, cancellation.Token)).ToArray(); + try + { + await scenario.Entered.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + Assert.Equal(initialSummaryRequests + 3, target.SummaryRequests); + + await ProbeAsync(provider, remote.SiloAddress, cancellationToken).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + Assert.Equal(initialSummaryRequests + 3, target.SummaryRequests); + Assert.All(probes, probe => Assert.False(probe.IsCompleted)); + if (cancelCaller) + { + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => + Task.WhenAll(probes).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken)); + } + else + { + time.Advance(TimeSpan.FromSeconds(1)); + await Task.WhenAll(probes).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + } + + await scenario.CancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + Assert.False(scenario.Finished.Task.IsCompleted); + + target.Scenario = null; + await ProbeAsync(provider, remote.SiloAddress, cancellationToken).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + Assert.Equal(initialSummaryRequests + 4, target.SummaryRequests); + Assert.False(scenario.Finished.Task.IsCompleted); + + scenario.Release(); + await scenario.Finished.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + await ProbeAsync(provider, remote.SiloAddress, cancellationToken).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + Assert.Equal(initialSummaryRequests + 5, target.SummaryRequests); + } + finally + { + cancellation.Cancel(); + scenario.Release(); + try + { + await Task.WhenAll(probes).WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + } + + await scenario.Finished.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + target.Scenario = null; + } + } + + private static Task ProbeAsync(ClusterManifestProvider provider, SiloAddress peer, CancellationToken cancellationToken) => + (Task)typeof(ClusterManifestProvider).GetMethod("ProbePeerForManifests", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(provider, [peer, new[] { peer }, new ConcurrentDictionary(), cancellationToken])!; + + private sealed class ProbeScenario(bool waitForUpdate) + { + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _entered; + private int _canceled; + private int _finished; + + public bool WaitForUpdate => waitForUpdate; + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource CancellationObserved { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Finished { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public async Task WaitAsync(CancellationToken cancellationToken) + { + using var registration = cancellationToken.Register(() => + { + if (Interlocked.Increment(ref _canceled) == 3) + { + CancellationObserved.TrySetResult(); + } + }); + if (Interlocked.Increment(ref _entered) == 3) + { + Entered.TrySetResult(); + } + + // Observe cancellation while deliberately keeping the remote invocation alive. + await _release.Task; + if (Interlocked.Increment(ref _finished) == 3) + { + Finished.TrySetResult(); + } + } + + public void Release() => _release.TrySetResult(); + } + + private sealed class PeerTarget : SystemTarget, IClusterManifestSystemTarget, ILifecycleParticipant + { + private ProbeScenario? _scenario; + private int _summaryRequests; + + public PeerTarget(SystemTargetShared shared) : base(TargetType, shared) + { + shared.ActivationDirectory.RecordNewTarget(this); + } + + public ProbeScenario? Scenario + { + get => Volatile.Read(ref _scenario); + set => Volatile.Write(ref _scenario, value); + } + + public int SummaryRequests => Volatile.Read(ref _summaryRequests); + + public async ValueTask GetClusterManifestHashSummary(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _summaryRequests); + if (Scenario is { WaitForUpdate: false } scenario) + { + await scenario.WaitAsync(cancellationToken); + } + + return new ClusterManifestHashSummary(MajorMinorVersion.MinValue, []); + } + + public async ValueTask GetClusterManifestUpdate(MajorMinorVersion previousVersion, CancellationToken cancellationToken = default) + { + if (Scenario is { WaitForUpdate: true } scenario) + { + await scenario.WaitAsync(cancellationToken); + } + + return null; + } + + public ValueTask GetClusterManifest(CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public ValueTask GetSiloManifestHash(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask GetSiloManifestByHash(ManifestHash hash, CancellationToken cancellationToken) => throw new NotSupportedException(); + + public void Participate(ISiloLifecycle lifecycle) + { + } + } + + private sealed class LegacyManifestTarget : SystemTarget, ISiloManifestSystemTarget, ILifecycleParticipant + { + private int _requests; + + public LegacyManifestTarget(SystemTargetShared shared, SiloManifestProvider provider) : base(LegacyTargetType, shared) + { + Manifest = provider.SiloManifest; + shared.ActivationDirectory.RecordNewTarget(this); + } + + public GrainManifest Manifest { get; } + public int Requests => Volatile.Read(ref _requests); + + public ValueTask GetSiloManifest(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + Interlocked.Increment(ref _requests); + return new(Manifest); + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + } +}