diff --git a/.github/scripts/dissemination-compatibility.ps1 b/.github/scripts/dissemination-compatibility.ps1 new file mode 100644 index 00000000000..b7121b98ced --- /dev/null +++ b/.github/scripts/dissemination-compatibility.ps1 @@ -0,0 +1,143 @@ +[CmdletBinding()] +param( + [string] $BaselineDirectory = '.dissemination-baseline', + [string] $ArtifactsDirectory = 'Artifacts/DisseminationCompatibility', + [switch] $SkipBuild +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$baselineSha = '6739589254b746a8790cf53524e6abe372bb53d4' +$root = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +$artifacts = [System.IO.Path]::GetFullPath((Join-Path $root $ArtifactsDirectory)) +$binaries = Join-Path $artifacts 'bin' +$results = Join-Path $artifacts 'results' +New-Item -ItemType Directory -Force $binaries, $results | Out-Null + +function Invoke-DotNet([string[]] $Arguments) { + & dotnet @Arguments + if ($LASTEXITCODE -ne 0) { + throw "dotnet failed ($LASTEXITCODE): $($Arguments -join ' ')" + } +} + +function Get-Revision([string] $Directory) { + $revision = & git -C $Directory rev-parse HEAD + if ($LASTEXITCODE -ne 0) { + throw "Not a source checkout: $Directory" + } + return $revision.Trim() +} + +function Publish-Silo([string] $Checkout, [string] $Runtime, [string] $Revision) { + $project = Join-Path $Checkout 'test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj' + $output = Join-Path $binaries $Runtime + Invoke-DotNet @( + 'publish', $project, '--configuration', 'Release', '--framework', 'net10.0', + '--output', $output, '--verbosity', 'minimal', + "-p:HarnessRuntime=$Runtime", "-p:SourceRevisionId=$Revision", + '-p:OfficialBuild=true', "-p:BUILD_SOURCEVERSION=$Revision", + "-bl:$(Join-Path $artifacts "build-$Runtime.binlog")" + ) + $assemblyPath = Join-Path $output 'Orleans.Runtime.dll' + $identity = [System.Reflection.AssemblyName]::GetAssemblyName($assemblyPath) + $assemblies = @{} + foreach ($file in Get-ChildItem -LiteralPath $output -Filter 'Orleans*.dll' -File) { + $name = [System.Reflection.AssemblyName]::GetAssemblyName($file.FullName) + $assemblies[$name.Name] = @{ + AssemblyVersion = $name.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash + } + } + @{ + Runtime = $Runtime + SourceRevision = $Revision + AssemblyName = $identity.Name + AssemblyVersion = $identity.Version.ToString() + Sha256 = (Get-FileHash -LiteralPath $assemblyPath -Algorithm SHA256).Hash + Assemblies = $assemblies + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $output 'manifest.json') +} + +if (!$SkipBuild) { + if ($env:GITHUB_ACTIONS -ne 'true') { + throw 'Building the pinned baseline is CI-only. Download the workflow binary artifact and use -SkipBuild for explicit local reproduction.' + } + + $baseline = [System.IO.Path]::GetFullPath((Join-Path $root $BaselineDirectory)) + if ($baseline -eq $root) { + throw 'The baseline must not be the feature checkout.' + } + $baselineTopLevel = (& git -C $baseline rev-parse --show-toplevel) + if ($LASTEXITCODE -ne 0 -or [System.IO.Path]::GetFullPath($baselineTopLevel.Trim()) -ne $baseline) { + throw 'The baseline must have its own isolated git checkout, not merely a directory in the feature checkout.' + } + if ((Get-Revision $baseline) -ne $baselineSha) { + throw "Baseline must be pinned to $baselineSha." + } + $sourceChanges = & git -C $baseline status --porcelain --untracked-files=all -- src + if ($LASTEXITCODE -ne 0 -or $sourceChanges) { + throw 'Pinned baseline runtime sources have local modifications.' + } + if (Test-Path -LiteralPath (Join-Path $baseline 'src/Orleans.Runtime/Dissemination')) { + throw 'The pinned baseline unexpectedly contains the new dissemination implementation.' + } + + # Only the test executable is copied. Every Orleans source project resolves inside its own checkout. + $harnessSource = Join-Path $root 'test/Dissemination.IntegrationHarness' + $harnessDestination = Join-Path $baseline 'test/Dissemination.IntegrationHarness' + if (Test-Path -LiteralPath $harnessDestination) { + throw "Refusing to overwrite an existing baseline harness: $harnessDestination" + } + New-Item -ItemType Directory -Force $harnessDestination | Out-Null + foreach ($part in @('Silo', 'Shared')) { + New-Item -ItemType Directory -Force (Join-Path $harnessDestination $part) | Out-Null + Get-ChildItem -LiteralPath (Join-Path $harnessSource $part) -File | + Where-Object { $_.Extension -in '.cs', '.csproj' } | + Copy-Item -Destination (Join-Path $harnessDestination $part) + } + + Publish-Silo $baseline 'Old' $baselineSha + Publish-Silo $root 'New' (Get-Revision $root) + Invoke-DotNet @( + 'publish', (Join-Path $root 'test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj'), + '--configuration', 'Release', '--framework', 'net10.0', + '--output', (Join-Path $binaries 'Runner'), '--verbosity', 'minimal', + "-bl:$(Join-Path $artifacts 'build-runner.binlog')" + ) +} + +$env:ORLEANS_DISSEMINATION_OLD = Join-Path $binaries 'Old' +$env:ORLEANS_DISSEMINATION_NEW = Join-Path $binaries 'New' +$env:ORLEANS_DISSEMINATION_RESULTS = $results +$runner = Join-Path $binaries 'Runner/Dissemination.IntegrationHarness.Tests.dll' +if (!(Test-Path -LiteralPath $runner)) { + throw "Published runner not found: $runner" +} +$testClass = 'Orleans.Dissemination.IntegrationHarness.VersionSkewTests' +$minimum = '10' + +@{ + PinnedBaseline = $baselineSha + CurrentCommit = Get-Revision $root + StartUtc = [DateTimeOffset]::UtcNow + Command = "dotnet $runner --filter-class $testClass --minimum-expected-tests $minimum --report-trx" + Environment = 'Separate loopback OS processes with a controlled test membership provider.' +} | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $results 'invocation.json') + +Push-Location $root +try { + Invoke-DotNet @( + $runner, '--filter-class', 'Orleans.Dissemination.IntegrationHarness.CompatibilityHarnessTests', + '--minimum-expected-tests', '8', '--report-trx', + '--results-directory', (Join-Path $results 'harness-checks') + ) + + Invoke-DotNet @( + $runner, '--filter-class', $testClass, '--minimum-expected-tests', $minimum, + '--report-trx', '--results-directory', $results + ) +} +finally { + Pop-Location +} diff --git a/.github/workflows/dissemination-compatibility.yml b/.github/workflows/dissemination-compatibility.yml new file mode 100644 index 00000000000..bfa2fc77038 --- /dev/null +++ b/.github/workflows/dissemination-compatibility.yml @@ -0,0 +1,78 @@ +name: Dissemination compatibility + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/dissemination-compatibility.yml' + - '.github/scripts/dissemination-compatibility.ps1' + - 'test/Dissemination.IntegrationHarness/**' + - 'src/Orleans.Core.Abstractions/IDs/SiloAddress.cs' + - 'src/Orleans.Runtime/Dissemination/**' + - 'src/Orleans.Runtime/MembershipService/**' + - 'src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs' + - 'src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs' + - 'src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs' + - 'src/Orleans.Runtime/Hosting/DefaultSiloServices.cs' + - 'src/Orleans.Runtime/OrleansContracts.txt' + - 'src/Orleans.Core/OrleansContracts.txt' + - 'src/Orleans.Core/Runtime/Constants.cs' + - 'src/Orleans.Core/Runtime/MembershipTableSnapshot.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IDeploymentLoadPublisher.cs' + - 'src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs' + - 'src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs' + - 'src/Orleans.Core/Configuration/Options/DisseminationOptions.cs' + merge_group: + types: [checks_requested] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dissemination-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_NOLOGO: true + +jobs: + version-skew: + name: Pinned-binary upgrade, rollback and recovery + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - name: Isolated pre-dissemination checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + repository: dotnet/orleans + ref: 6739589254b746a8790cf53524e6abe372bb53d4 + path: .dissemination-baseline + persist-credentials: false + - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5 + with: + global-json-file: global.json + - name: Build isolated binaries and run bounded merge-gate checks + shell: pwsh + run: ./.github/scripts/dissemination-compatibility.ps1 + - name: Preserve test results and process diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-compatibility-results + retention-days: 14 + path: | + Artifacts/DisseminationCompatibility/results + Artifacts/DisseminationCompatibility/*.binlog + - name: Preserve reproducible binary pair + if: success() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: dissemination-compatibility-binaries + retention-days: 7 + path: Artifacts/DisseminationCompatibility/bin diff --git a/docs/site/src/content/docs/host/monitoring/index.md b/docs/site/src/content/docs/host/monitoring/index.md index e60e9c0e51b..268616b89aa 100644 --- a/docs/site/src/content/docs/host/monitoring/index.md +++ b/docs/site/src/content/docs/host/monitoring/index.md @@ -1,7 +1,7 @@ --- title: Orleans observability description: Configure OpenTelemetry logging, metrics, and tracing for Orleans. -ms.date: 08/02/2026 +ms.date: 09/04/2026 ms.topic: overview --- @@ -79,6 +79,7 @@ The example's 10% head-sampling ratio is a starting point, not a universal produ ## Next steps - [Monitor Orleans metrics](metrics.md) +- [Monitor runtime dissemination](runtime-dissemination.md) - [Interpret Orleans signals](signals.md) - [Troubleshoot Orleans incidents](troubleshooting.md) - [Secure and operate the Orleans Dashboard](../../dashboard/index.md) diff --git a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md index 6fc6dc0abd5..f745ae32608 100644 --- a/docs/site/src/content/docs/host/monitoring/metrics-catalog.md +++ b/docs/site/src/content/docs/host/monitoring/metrics-catalog.md @@ -1,7 +1,7 @@ --- title: Orleans metrics catalog description: Complete reference for metrics emitted by the Microsoft.Orleans meter. -ms.date: 08/18/2026 +ms.date: 09/04/2026 ms.topic: reference --- @@ -65,6 +65,29 @@ Request latency covers the interval until the caller's callback completes, inclu `Destination` and `silo` contain silo addresses and incarnations. They are useful during incident diagnosis and can create new time series during restarts. +## Runtime dissemination + +| Instrument | Type | Unit | Attributes | Description | +|---|---|---|---|---| +| `orleans-dissemination-anti-entropy-digests` | C | `digests` | `direction` | Digest entries sent in successful outbound exchanges or received in inbound exchanges. | +| `orleans-dissemination-anti-entropy-exchanges` | C | `operations` | `direction`, `truncated` | Successful anti-entropy requests and responses. `truncated=true` indicates that response budgets left candidates for a later exchange. | +| `orleans-dissemination-anti-entropy-failures` | C | `operations` | `reason` | Outbound anti-entropy peer operations which ended in `timeout` or `error`. | +| `orleans-dissemination-anti-entropy-values` | C | `values` | `direction` | Repair values returned by successful anti-entropy exchanges. | +| `orleans-dissemination-broadcast-received` | C | `messages` | `namespace`, `kind` | Broadcast batches received, counted once for each namespace represented in the batch. | +| `orleans-dissemination-broadcast-scheduled` | C | `schedules` | `reason` | Per-peer pump schedules split into `immediate` and `retry`. | +| `orleans-dissemination-broadcast-send-failures` | C | `attempts` | `reason` | Broadcast send attempts which ended in `timeout` or `error`. | +| `orleans-dissemination-broadcast-sent` | C | `messages` | `namespace`, `kind` | Successfully completed broadcast sends, counted once for each namespace represented in the batch. | +| `orleans-dissemination-bytes-sent` | C | `bytes` | `namespace`, `kind` | Serialized dissemination payload bytes in successful broadcasts. | +| `orleans-dissemination-payload-dropped` | C | `values` | `namespace`, `reason` | Values rejected by a payload guard. | +| `orleans-dissemination-pump-failures` | C | `failures` | `status` | Unexpected peer-pump failures split into `recovered` and `permanent`. | +| `orleans-dissemination-publications` | C | `operations` | `namespace`, `result`, `reason` | Publication attempts classified as `accepted` or `rejected`. Rejection reasons identify disabled namespaces, unavailable membership or values, invalid versions or repairs, and over-budget repairs. | +| `orleans-dissemination-queue-admission-rejected` | C | `keys` | `namespace`, `reason` | New distinct keys rejected after a namespace reached its per-peer pending-key limit. | +| `orleans-dissemination-values-applied` | C | `values` | `namespace`, `result` | Received values classified by the namespace apply result. | +| `orleans-dissemination-values-received` | C | `values` | `namespace`, `kind` | Values included in received broadcast batches before individual application. | +| `orleans-dissemination-values-sent` | C | `values` | `namespace`, `kind` | Values included in successfully completed broadcast sends. | + +All dissemination attributes have bounded runtime-defined values. Keys and peer addresses are available through opt-in diagnostic events instead of metric attributes. See [Monitor runtime dissemination](runtime-dissemination.md). + ## Scheduling, activations, and grains | Instrument | Type | Unit | Attributes | Description | diff --git a/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md new file mode 100644 index 00000000000..2afae48fb15 --- /dev/null +++ b/docs/site/src/content/docs/host/monitoring/runtime-dissemination.md @@ -0,0 +1,96 @@ +--- +title: Monitor runtime dissemination +description: Monitor convergence, retry pressure, repair, backpressure, and fallback behavior in Orleans runtime dissemination. +ms.date: 09/10/2026 +ms.topic: concept-article +--- + +# Monitor runtime dissemination + +Runtime dissemination accelerates convergence of membership and deployment-load state. Monitor its progress and failure signals alongside the authoritative membership, placement, messaging, and networking signals. + +The instruments use the standard `Microsoft.Orleans` meter: + +```dotnetcli +dotnet-counters monitor -n --counters Microsoft.Orleans +``` + +See [Runtime state dissemination](../../implementation/runtime-dissemination.md) for protocol invariants and failure semantics and the [metrics catalog](metrics-catalog.md#runtime-dissemination) for the complete instrument reference. + +## Build a convergence dashboard + +Use rates over the same interval and split by `namespace` where available: + +| Signal | Interpretation | +|---|---| +| Broadcast sent and received | Tree traffic reaches peers. Compare cluster-wide rates during active publication. | +| Values sent, received, and applied | Receipt measures delivered work. Break apply `result` into applied, duplicate, obsolete, and rejected values. | +| Broadcast scheduled with `reason=retry` | A peer pump retained work after incomplete delivery or repair. | +| Broadcast send failures | `timeout` indicates the hop lifetime expired; `error` indicates an RPC or transport failure. | +| Anti-entropy exchanges and values | Periodic repair remains active and returns bounded work. | +| Anti-entropy failures | Selected repair peers timed out or failed. Rotating peer selection uses other peers when independent repair capacity is available. | +| Publications | `accepted` measures work admitted to peer pumps. `rejected` identifies disabled, unavailable, invalid, over-budget, or queue-rejected publications which retain the component's direct path. | +| Queue admission rejected | One namespace reached its per-peer distinct-key bound. | +| Pump failures | `recovered` records an isolated iteration failure; `permanent` records a pump whose waiters were failed explicitly. | + +The `truncated=true` anti-entropy series means a response exhausted an item or byte budget while more candidates remained. Occasional truncation is bounded continuation. Sustained truncation with low applied-value throughput indicates that budgets, payload sizes, or publication churn are limiting convergence. + +## Alert on sustained loss of progress + +Page on user-visible impact first. Dissemination signals identify a contributing runtime path: + +- Alert on permanent pump failures because one destination has stopped accepting broadcast work until its peer state is recreated. +- Alert on sustained queue admission rejection because updates for new keys are using fallback paths or waiting for capacity. +- Correlate broadcast and anti-entropy failure rates with `orleans-messaging-sent-failed`, socket churn, membership changes, and deployment events. +- Compare rejected publications with the runtime component's update rate. A rollout can temporarily increase direct-path use while peer capability evidence is established. +- Investigate sustained retry scheduling when successful broadcasts and applied values remain flat. + +Timeout and retry spikes during a rolling restart or network partition can be expected. Healthy recovery produces successful exchanges, declining retry and failure rates, and continued applied or duplicate outcomes after connectivity returns. + +## Diagnose by symptom + +### One silo has persistent retries + +Compare that instance's send failures with messaging and socket metrics. Inspect `Orleans.Runtime.Dissemination.DisseminationBroadcastQueue` logs for the peer and retry delay. A permanent pump failure is logged at Error; a recovered iteration failure is logged at Warning. + +One destination occupies at most one active local broadcast slot. Its hop deadline or pump cancellation releases that slot and signals the RPC's cancellation token. A queued retry retains identities until admission. Other ready destinations use released slots in FIFO order, and backlogged peers rejoin admission per batch. + +### Traffic stops despite bounded retries + +Check local attempt duration and timeout rates against the namespace hop lifetime. Local TTL expiry ends a pump's wait and releases its slot. Repair has a separate local attempt limit from , so it can continue during broadcast saturation. + +Correlate sustained lack of progress with retry backoff, scheduler starvation, timer processing, and transport failures. Inspect pending messaging RPCs separately: normally supplies their terminal response bound even when cancellation acknowledgment is delayed. Tune concurrency together with hop lifetimes and retry cadence to control new attempt rates during transport failures. + +Retry scheduling metrics describe timer decisions. The send-gate diagnostic identifies requests which entered the FIFO; cancelling a flush observer leaves that shared request queued. Bound explicit flush and drain waits using caller cancellation tokens. Repair selection skips locally busy destinations until a later round. Successful responses completed within a repair round still apply when another selected peer misses the deadline. + +### Queue admission is rejected + +Identify the `namespace` and compare update cardinality with . Increase the limit only after confirming the process has memory headroom and the destination can drain work. Broadcast item and byte limits still bound each transmission. + +Capacity planning must include all retained peer/namespace pairs: pending-key storage scales with their summed per-peer limits, and acknowledgment ledgers scale with known keys. Active local attempts materialize at most one batch per broadcast slot, bounded by . Cached namespace payloads, metadata, and messaging buffers retained by pending RPCs remain additional costs. + +### Anti-entropy remains truncated + +Compare returned values with , , and the namespace payload limit. Persistent cursors continue from the bounded response across later exchanges, so verify that exchanges and applied values continue rather than alerting on truncation alone. + +### Rejected publications increase + +Break down by `namespace` and `reason`. Validate peer version compatibility, payload limits, and namespace enablement. `stopping` identifies closed protocol admission; `queue-rejected` identifies a full or stopping broadcast queue. Correlate these reasons with queue admission rejection and silo lifecycle activity. Deployment-load and membership integrations retain direct paths, so also inspect their component-specific logs and metrics for the authoritative outcome. + +## DiagnosticListener events + +The `Microsoft.Orleans.Dissemination` diagnostic listener exposes detailed, opt-in events: + +| Event | Use | +|---|---| +| `Dissemination.ValueApply` | Inspect one value's namespace, key, version transition, peer, result, and payload size. | +| `Dissemination.PayloadDrop` | Inspect a value rejected by a payload guard. | +| `Dissemination.BroadcastScheduled` | Observe immediate and retry scheduling with due time and attempt. | +| `Dissemination.SendGate` | Observe broadcast FIFO entry (`Kind=broadcast`, `Stage=queued`) or a local repair attempt releasing its admission (`Kind=repair`, `Stage=released`), including timeout and cancellation. | +| `Dissemination.QueueAdmissionRejected` | Observe the peer, namespace, configured limit, and bounded rejection reason. | + +The send-gate event is emitted outside scheduler locks after queue insertion or accounting release. Admission or cancellation can race with event delivery, so it records a transition rather than a current queue-length measurement. + +Its typed payload includes `LocalSilo`, `Peer`, `Timestamp`, and bounded `Kind` and `Stage` strings. The repair-release transition can also report cancellation or failure before an admitted exchange sends its request. A later exchange may already have reused released capacity when an observer processes the event. + +Value events include keys and peer addresses; admission events include peer addresses. Enable them for a time-limited investigation and apply the deployment's telemetry data and retention policy. Metrics intentionally omit keys and peer addresses to keep long-term cardinality bounded. diff --git a/docs/site/src/content/docs/implementation/index.md b/docs/site/src/content/docs/implementation/index.md index 723ed8606e4..b5f5bc5e377 100644 --- a/docs/site/src/content/docs/implementation/index.md +++ b/docs/site/src/content/docs/implementation/index.md @@ -1,7 +1,7 @@ --- title: Orleans runtime architecture description: An advanced guide to the protocols, invariants, and extension points inside the Orleans runtime. -ms.date: 08/02/2026 +ms.date: 09/04/2026 ms.topic: overview --- @@ -38,6 +38,7 @@ Use this runtime map and the following topic list to choose the required depth. - [Runtime architecture](runtime-architecture.md) follows a call through client, messaging, placement, directory, activation, and scheduling components. - [Activation lifecycle and migration](activation-lifecycle.md) explains creation, activation, collection, deactivation, and state transfer. - [Cluster membership](cluster-management.md) describes the failure detector, membership table, ordered views, and death-vote protocol. +- [Runtime state dissemination](runtime-dissemination.md) explains deterministic broadcast, acknowledged peer state, bounded repair, mixed-version behavior, and integration boundaries. - [Grain directory](grain-directory.md) distinguishes the default `LocalGrainDirectory` DHT from the experimental distributed directory. - [Scheduling and turn execution](scheduler.md) explains `WorkItemGroup`, continuations, interleaving, and single-threaded execution. - [Messaging and delivery semantics](messaging-delivery-guarantees.md) traces requests and explains why a timeout has an unknown outcome. diff --git a/docs/site/src/content/docs/implementation/runtime-dissemination.md b/docs/site/src/content/docs/implementation/runtime-dissemination.md new file mode 100644 index 00000000000..f9df8b461dc --- /dev/null +++ b/docs/site/src/content/docs/implementation/runtime-dissemination.md @@ -0,0 +1,203 @@ +--- +title: Runtime state dissemination +description: Architecture, invariants, repair, failure semantics, and integration boundaries for Orleans runtime dissemination. +ms.date: 09/16/2026 +ms.topic: concept-article +--- + +# Runtime state dissemination + +Runtime dissemination is an internal silo-to-silo protocol which accelerates convergence of frequently changing runtime state. A deterministic tree carries new versions quickly, acknowledgments record peer knowledge, and bounded anti-entropy repairs loss, reordering, partitions, and membership skew. Each participating runtime component remains authoritative for its own state. + +The dissemination protocol currently carries: + +- deployment-load statistics among Active silos; +- membership snapshots among Joining, Active, ShuttingDown, and Stopping silos. + +The deployment-load publisher and membership manager retain their direct delivery paths and validation rules. Tree broadcast and anti-entropy form one convergence feature: broadcast accelerates new updates, and repair discovers and delivers state missed by the tree. + +## Components and responsibilities + +```mermaid +flowchart LR + Producer[Runtime state producer] --> Namespace[IDisseminationNamespace] + Namespace --> Protocol[DisseminationProtocol] + Protocol --> Queue[Per-peer broadcast pumps] + Protocol --> Cohort[Load cohorts and publication receipts] + Cohort --> Queue + Cohort -.-> Producer + Queue --> Target[Remote dissemination system target] + Target --> Apply[Namespace apply] + Apply --> Forward[Deterministic forwarding tree] + Protocol <--> Repair[Bounded anti-entropy] + Namespace --> Authority[Authoritative runtime component] + Repair --> Authority +``` + +`DisseminationProtocol` coordinates routing, peer capability evidence, anti-entropy, and application isolation. `DisseminationRootBatcher` owns the aggregation root's bounded latest-key cohorts, producer contributions, and seal receipts. `DisseminationBroadcastQueue` owns per-peer scheduling, retry, drain, and acknowledged-version ledgers. `DisseminationMembership` projects one membership snapshot into topology-specific member sets. Each `IDisseminationNamespace` owns serialization, current versions, payload limits, repair construction, and application semantics. + +Queue entries contain `(namespace, key)` identities. A pump acquires both destination ownership and a global broadcast slot before materializing an update. Membership peer ledgers also retain one immutable comparison snapshot from the last accepted, exactly acknowledged broadcast. Coalesced notifications compare that snapshot with current owner state after admission, preserving cumulative changes and reversions while waiting peers retain shared snapshot references rather than serialized messages. + +Single-key publication and received-batch forwarding share one synchronous queue-admission path. A borrowed notification span is consumed under the pump lock before scheduling once, and diagnostic callbacks run after releasing the lock. Peer capability records store confirmed namespace identities for the lifetime of that silo generation; explicit rejection or membership removal retires the confirmation. + +Applied state wakes each forwarding child, including same-version membership liveness advances. Duplicate deliveries wake children whose versions are still missing and whose queues contain no equivalent work. This lets a first tree delivery forward state already learned through a direct path while suppressing repeated forwarding cycles between silos with different topology views. + +## Version and application invariants + +A `DisseminationValue` carries one key's update and version: + +- its zero `FromVersion` identifies a full value which can establish state at any receiver baseline; +- a positive `FromVersion` identifies a membership broadcast delta from that canonical view to `ToVersion`; +- acknowledged peer versions advance monotonically from explicit receiver evidence; +- duplicate and obsolete values leave authoritative state unchanged; and +- one rejected value does not prevent later values in the batch from being considered. + +Namespaces construct broadcasts separately from full anti-entropy repairs. Each result contains one value; batch assembly reserves an item slot before construction and retains unsent keys for subsequent bounded batches. Deployment-load broadcasts remain full latest samples. + +Broadcast senders advertise support for compact acknowledgments. The compact flag requires complete receive admission, accepted or duplicate application results, and exact agreement with each key's transmitted version. Namespace entries retain capability evidence with empty per-key lists. An exact compact acknowledgment also installs the membership comparison snapshot captured when constructing that send. Explicit higher versions and passive peer knowledge advance version evidence; a comparison snapshot is usable only while its version matches that evidence. + +Membership broadcasts contain changed entries and removed silo identities, together with base and target canonical view versions. A receiver at the base applies the transition. A receiver already at the target preserves canonical fields and every non-Dead row, merges maximum heartbeat timestamps, and prunes explicitly removed Dead rows. A coalesced delta across versions can span an intervening Dead transition and cleanup. Other receiver versions require full repair. A rejected delta makes the key eligible for the next anti-entropy round, even if its local version advanced recently. Relays construct their own sparse update against each child's comparison snapshot, so an ahead relay can distribute its newer accepted view rather than regress to an older incoming update. + +A peer without a usable comparison snapshot receives an empty delta at the sender's current version. A peer already at that version acknowledges the probe; a lagging peer obtains a full snapshot through repair or ordinary membership bootstrap. The retained snapshot establishes a canonical comparison baseline, while initial heartbeat and retained-Dead differences remain repair work. + +Membership anti-entropy uses the view version and a liveness/inventory fingerprint. Responses always contain full current snapshots, whose serialized payloads are cached until owner state changes. Same-version repair preserves canonical fields and non-Dead entries, merges maximum heartbeats, and reconciles independently pruned Dead rows without resurrecting them. Application checks the requested effects against resulting owner state before acknowledging acceptance, including when an authority refresh completes concurrently. + +Comparison snapshots are shared immutable objects, with one retained reference per existing outbound membership peer/key and constant-size namespace caches. They are pruned with peer and key knowledge. Healthy peers share the same snapshot; lagging peers can retain different snapshots, giving a worst-case memory cost proportional to outbound peers times membership size. Sparse payload caching uses snapshot identity/content, including same-version changes. + +Canonical membership versions are monotonic within a cluster lifetime; hosts can skip observed versions. Same-version snapshots can differ in maximum heartbeat progress and retention of Dead rows. Removing any other status requires a canonical view advance. Provider versioned compaction is a stronger storage contract and remains independently applicable. Development clustering's primary table defines that lifetime. Loss of that authority invalidates the continuing cluster, and an observed table rollback requests fatal silo termination. Full repair follows the membership owner's lifetime and validation rules. Deployment-load samples retain their timestamp ordering. + +Deployment-load versions are sample timestamps. Each repair is a full latest value. Application runs on the deployment-load publisher's scheduler and accepts samples only for Active silo generations in the membership oracle. Terminal membership transitions remove placement statistics, and the oracle's monotonic membership state rejects later samples for departed generations. + +After the owner accepts a load sample, its serialized payload is reused for forwarding and repair. Inventory maintenance enumerates namespace keys directly; membership exposes its single key, and deployment load uses the active-silo oracle. Each broadcast queue retains one reusable scratch inventory; individual pruning calls have exclusive ownership and clear all keys before returning it, including on failure. Version and fingerprint construction is reserved for operations which use that information. Receive batches which fit all item, byte, and namespace limits are processed directly from their existing collections; truncated batches retain bounded cursor selection. + +## Deterministic topology + +Every silo derives routing from the same ordered membership projection. Members are ordered by status and silo address. Silo addresses order by generation, port, and IP address, providing stable ordering across membership storage providers with different timestamp precision. Joining, Active, ShuttingDown, and Stopping entries participate. + +Membership uses the broadcast forest: for fanout `f` and zero-based member index `i`, a forwarding node selects children starting at `f * (i + 1)` and continuing for at most `f` members. An originator sends to the first `f` members, excluding itself, plus its normal forwarding children. + +Deployment load uses root aggregation followed by tree distribution. Node `i > 0` has parent `(i - 1) / f`; its children start at `f * i + 1`. Its fanout defaults to eight, independently of the membership forest. Each non-root producer sends its own fresh sample directly to the root through a dedicated aggregation RPC. The root's own sample enters the same cohort locally. A cohort captures its expected Active producer incarnations and retains the latest notification per key. It seals after every expected producer contributes, or after the configured publication period elapses from cohort opening. New arrivals preserve that deadline; duplicate messages and ordinary forwarding hints preserve distinct-producer counting. + +Ingress RPCs remain open asynchronously until the cohort seals and its distribution work is admitted. Their receipts carry the remaining delay to the cohort's next publication boundary: `max(0, cohort start + period - receipt time)`. The publisher applies that delay to its sampling timer. Complete, phase-aligned cohorts can seal quickly while preserving approximately one sample per silo per period. A cohort that reaches its deadline schedules its next sample promptly. Local monotonic timing handles these relative delays across machines. + +The ingress receipt path has independent local-attempt admission. Membership uses its existing immediate peer pumps while load receipts are held. Each admitted cohort enters child queues through bounded batch admission, and relays forward immediately. Explicit flush and shutdown seal partial cohorts within the caller's cancellation budget. A former root hands pending state to the current root, including when it has left Active membership. Shutdown seals cohorts before waiting for their held protocol admissions, then drains peer queues. + +Aggregation ledgers advance on broadcast acknowledgments, after the receiver processes the batch and attempts downstream queue admission. An ingress producer which is also a child receives its own update in the root's distribution batch and forwards it to its descendants. Passive evidence of a peer's local value serves repair; broadcast acknowledgments establish distribution progress. Anti-entropy repairs gaps left by bounded queue admission or changed topology. + +At one publication and one fitting cohort per second, root ingress contributes `N - 1` requests and tree distribution contributes another `N - 1`. The healthy steady-state model is `2 * (N - 1)`: 18 load requests per second at 10 silos, 198 at 100 silos, and 3,998 at 2,000 silos. An eight-child relay distributes approximately eight batches per second. Including the default periodic anti-entropy budget, the respective projections are 24, 258, and 5,198 requests per second, plus their replies. Root ingress remains concentrated at the root. These are algorithmic projections; production capacity and tail latency require measurement. Retries, topology transitions, forced drains, fallback, and wire-message splitting add work. Payload delivery includes each recipient's copy of the statistics. + +Cohort completeness depends on the slowest expected producer. Healthy aligned arrivals produce a short collection delay; an absent or paused producer makes the cohort use its full deadline. Repeatedly incomplete cohorts can leave samples approaching two publication periods old before replacement. Startup, membership changes, scheduling skew, and synchronized ingress/reply bursts are important latency and capacity cases. + +Active members are ordered by silo address. Ingress moves toward a smaller root; distribution moves toward larger children. A former root rejects new contribution requests, allowing the publisher's direct path to preserve delivery while subsequent publications resolve the new root. Already accepted cohort state and ordinary aggregation forwarding hints move toward the current root. Membership repair reconciles differing views. + +Namespaces select a membership scope before topology construction: + +| Namespace | Scope | Operational effect | +|---|---|---| +| Deployment load | Active members, root aggregation | Producers send to one root; already-aggregated batches traverse the distribution tree immediately. | +| Membership | All dissemination members | Joining and graceful-shutdown transitions can propagate. | + +Membership fanout is derived from the target hop count and configured bounds, or selected by the code-configured callback. Aggregation uses its independent fanout setting. At 2,000 silos, fanout eight gives at most four distribution hops. A membership change creates a new topology from the next snapshot; acknowledged ledgers and anti-entropy repair convergence across the transition. + +The projection cache tracks its authoritative source snapshot. On a cache miss it re-reads the owner under the cache lock, so a delayed reader follows the current view. Canonical view changes refresh routing and peer selection. Heartbeat-only updates reuse the existing topology, while rebuilt projections retain rotation progress. + +## Configuration and defaults + +Dissemination is opt-in. Enable and on each participating integration. The subsystem and namespace flags default to `false`; ordinary runtime delivery remains active. Explicitly enabled cluster and compatibility tests exercise broadcast and repair. Defaults bound concurrency, memory retention, payload size, and repair work: + +| Option | Default | Effect | +|---|---:|---| +| | 32 | Per-silo active local broadcast attempts. | +| | 8,192 | Values materialized in an outgoing batch or examined in an incoming batch or repair response. | +| | 1 MiB | Serialized payload bytes in an outgoing batch or admitted from an incoming batch or repair response. | +| | 2 | Target depth used to derive fanout. | +| / | 4 / 32 | Bounds for derived fanout. | +| | 8 | Maximum children per load-distribution relay. | +| | 1 second | Load sampling period, cohort deadline, and receipt-driven next-round target. | +| | 5 seconds | Repair-round cadence and retry-delay ceiling. | +| | 3 | Maximum peers selected in one repair round and independent per-silo active local repair attempt limit. | +| / | 8,192 / 1 MiB | Independent repair limits, capped by the global batch limits and negotiated with the peer. | +| | 1,024; load uses 8,192 | Distinct retained keys per namespace and peer, and in a root's pending set. | +| | 30 seconds | Independent local transport and application budgets for each hop. | +| | 10 seconds | Quiet period before a digest is offered for repair. | +| | 1 MiB | Maximum serialized value size for the namespace. | + +Each integration has its own . Operators can enable and tune membership and deployment-load dissemination independently while retaining the local concurrency and per-message bounds. + +Deployment load samples use the configured publication period, with receipt feedback aligning subsequent samples to cohort boundaries. The expected repair-update cadence remains five seconds. Producer ingress and relay forwarding send immediately; root receipts follow cohort sealing. The publication receipt budget is twice the publication period, capped by the platform timer limit: one period for collection and one for transport and scheduling. Ordinary RPC timeouts also apply. Caller cancellation stops the publication; an unavailable or rejecting root, receipt expiry, or a shorter RPC timeout selects the direct delivery path. Membership has fixed high priority. Configure the load pending-key bound to cover the intended active inventory and stalls; each newer notification replaces the retained value for that key rather than adding another sample. + +Inventory and peer-ledger maintenance runs on membership changes and bounded one-second maintenance opportunities instead of every received update. Failed passes remain eligible for retry. This keeps full inventory scans out of the normal per-message root path while promptly retiring peers when the routing view changes. + +The anti-entropy loop waits without periodic timer wakeups while the subsystem is disabled. An options-change notification wakes the loop when enablement changes; disabling it returns the loop to the dormant wait. Shutdown removes the options subscription and observes the loop's completion. + +Enablement requires representative measurements of convergence, RPC volume, serialized bytes, allocation, CPU, and tail latency across stable membership, churn, and partition recovery. Tree delivery can reduce RPC counts while increasing serialized bytes through forwarding and repair. Compare the original runtime, the updated explicitly disabled path, and the enabled path at the intended cluster sizes and publication rates, including skewed peer speeds and rolling upgrades. + +### Compatibility and recovery validation + +The `Dissemination compatibility` workflow builds the pre-feature runtime and current candidate in isolated checkouts, then starts separate silo processes. Assembly identities, hashes, and paths establish which binary each process actually loaded. Rolling-upgrade and rollback tests exercise old binaries alongside enabled and explicitly disabled current binaries. + +Four- and eight-silo partition tests cover all three runtime modes. Healing drains affected connections at both endpoints, including unfinished handshakes, then confirms acknowledged bidirectional control calls before offering one publication round. Convergence requires exact load inventory and values. Recovery diagnostics include connection draining and readiness in elapsed time. + +Additional tests isolate anti-entropy from tree delivery, direct gossip, and membership-table reads to establish snapshot and same-version heartbeat repair. Cancellation and partitioned shutdown have explicit completion bounds. Artifacts retain test results, binary manifests, process logs, and failure snapshots for diagnosis. Performance comparisons use separately maintained, explicitly invoked tooling. + +## Broadcast pumps and backpressure + +Each destination has one independent pump. New notifications schedule it immediately and replace pending notifications for the same namespace and key. Membership values are ordered first within a batch. Batches obey global item and byte bounds plus namespace payload bounds. An owner-wide managed admission gate combines the local broadcast limit with one active local broadcast attempt per destination. + +Admission uses an explicit FIFO linked list. Releasing a batch admits the oldest ready destinations, skipping peers with an active local attempt. Backlogged peers rejoin admission for every batch, behind other ready peers. New notifications schedule their destination immediately while retaining its FIFO position. Destination ownership uses the full `SiloAddress`, including generation, and ends with the local attempt. + +`MaxPendingItemCount` bounds distinct retained keys per namespace and peer. A notification for an admitted key refreshes its generation at the limit. A new key is rejected until acknowledgment-driven completion, an oversized repair, or membership pruning releases capacity. Oversized repairs retain acknowledged peer versions separately so a later publication can resume from the established baseline. The runtime emits a rejection metric and a diagnostic event without including the key. A rejected tree target makes publication return `false`, allowing the producer's direct path to deliver the update. + +The aggregate pending-key bound is topology-dependent: sum `MaxPendingItemCount` over the retained peer/namespace pairs, with identity-only snapshots for active flushes. Acknowledged-version ledgers additionally scale with known namespace keys and retained peers and are pruned against membership and the authoritative key inventory. + +With fixed configuration, active local broadcast attempts materialize at most `MaxConcurrentSends` batches, each bounded by `MaxBatchBytes`. Waiting peers retain identities until admission. Namespace-owned current values, identity and acknowledgment metadata, and messaging-layer buffers for pending RPCs are separate memory costs. Broadcast and aggregate repair concurrency limits are captured when the protocol is constructed; tune them before silo startup. + +A successful RPC is transport completion. The receiver response is the evidence which advances the peer ledger. Missing or insufficient acknowledgments retain work for repair. Transport timeout and failure requeue the generation with exponential backoff capped by the anti-entropy interval. A newer notification schedules an immediate attempt. + +The transport hop lifetime starts after admission. Local completion, timeout, or pump cancellation releases both the broadcast slot and destination ownership. Timeout signals the RPC's cancellation token and requeues the generation with backoff, allowing subsequent attempts and other ready destinations to proceed while the earlier RPC completes under the messaging runtime's lifetime rules. Pump cancellation removes pending admission requests; a concurrent grant is either used by the caller or released exactly once. + +Admission retains its FIFO position until capacity is available or the pump stops. An explicit flush caller supplies its own bounded cancellation token to limit its wait for capacity. Cancelling that observer throws cancellation while leaving the shared pump in its FIFO position. Shutdown uses its caller's cancellation budget to stop the pump and remove its pending admission. + +Unexpected iteration failures retain work and retry. A failure in the recovery path permanently fails the pump and explicitly completes flush and drain waiters with the failure. Shutdown can drain accepted work within the caller's cancellation budget or discard a removed peer's work during membership pruning. + +## Bounded anti-entropy + +An anti-entropy round rotates through at most `AntiEntropyPeerCount` eligible peers. Rotation progress persists across membership versions so frequent table updates still allow every eligible peer to participate. The broadest participating membership scope provides the global peer budget; each request includes only namespaces for which that peer is eligible. + +Repair has its own admission gate, independent of broadcast slots. Across overlapping rounds, at most `AntiEntropyPeerCount` local repair attempts are active, with at most one per destination. Busy slots and destinations are skipped until future rotating rounds. A round creates its shared digest snapshot and per-peer requests only after acquiring slots. Request digest storage scales with namespace key inventory. Each exchange bounds its local wait using the round's lifetime and cancellation token, and releases admission before the round completes. + +Digests identify namespace-owned keys, including missing state at version zero, so repair can discover work rejected during queue admission. Recently advanced streams suppress redundant probes until `ExpectedUpdateCadence` elapses. Responses obey item, batch-byte, payload-byte, and hop-lifetime bounds. Persistent per-requester cursors rotate truncated responses so hot early keys cannot starve later candidates. Cursor state for non-members is least-recently-used and bounded to 64 entries. + +Requests advertise the receiver's item and payload-byte limits. Responders use the smaller local and advertised limits, allowing their response cursor to provide fair service even when peers use different batch sizes. Older requests use the responder's own limits. Receivers independently cap incoming broadcasts and repair responses before applying values. Broadcast acknowledgments cover only the admitted keys at their actual local versions. Receive cursors advance through bounded ordered intervals for oversized deliveries. Cursor state is scoped to peer and direction, with at most 64 retained non-member cursors. Request digest inventory remains separate from these response admission budgets. + +Each selected full repair value is offered to its namespace. This preserves complementary same-version membership heartbeat advances from different peers; the owner merges them and classifies duplicates or obsolete state. Anti-entropy responses require a zero baseline and a positive destination version; membership deltas are admitted through broadcast instead. Responses which completed successfully within their local attempt are still applied when another peer exceeds the round deadline. Late RPC faults are observed, and subsequent rounds repair responses which missed the application window. During a partition, local attempts remain bounded and retries follow the configured cadence; after connectivity and membership views recover, rotating peer selection and cursors continue convergence. + +## Mixed-version behavior + +Peer namespace support is evidence-driven. Inbound traffic, broadcast acknowledgments, and authoritative anti-entropy responses confirm support for one silo generation. An explicit unsupported response revokes it and retires the publication generations covered by that flush. Publications admitted during the send remain queued with a fresh peer baseline. Membership pruning removes evidence for departed generations. + +Deployment-load publication uses the aggregation tree once all captured Active peers confirm namespace support. During bootstrap or mixed-version operation, direct fanout covers all Active peers so that an unsupported intermediate node cannot separate otherwise capable participants. Direct fanout also applies if dissemination is disabled, unavailable, rejected, or exceeds the publication receipt budget. Membership direct gossip begins before optional dissemination and keeps the caller's cancellation and shutdown deadline. + +## Concurrency and lifetime + +Protocol dictionaries use focused locks for membership projections, response cursors, value-update timestamps, peer capability evidence, and per-peer pump state. Network calls, namespace application, logging, diagnostic callbacks, and waiter continuations run outside those locks. Waiters use asynchronous continuations. + +Caller cancellation owns public operation lifetime and is checked before each received value is applied. Per-hop value TTL bounds the local broadcast RPC wait and anti-entropy exchange lifetime, independently of the runtime's cancellation-acknowledgment setting. Application has a separate local window capped by the receiving namespace's TTL. Broadcast values age from the start of receiver processing, including delays caused by earlier items; completed anti-entropy responses begin their application window after the exchanges finish. These windows use local elapsed time, preserving operation across clock skew. Namespace owners observe cancellation before committing queued or resumed work, including membership application following a pending refresh. A completed namespace result remains authoritative when local expiry races completion; caller cancellation still aborts remaining work. Shared admission waits are pump-cancellable and retain their FIFO position while capacity is unavailable. Explicit flush and drain observers use their caller's cancellation token, with no separate observer TTL; supply a bounded token when waiting indefinitely for admission is unacceptable. + +The shared `AdmissionGate` orders protocol operations against shutdown. Stopping closes admission to publications, incoming broadcasts, repair responses, and outgoing repair rounds. New publications return `false` with reason `stopping`, selecting the producer's direct path; new incoming RPCs fail explicitly. Shutdown cancels outgoing repair waits and drains admitted local operations before closing the broadcast queue, so accepted publishers and receivers finish enqueueing their work first. Accepted broadcast work then drains through acknowledgments, including retries with the usual backoff, within the shutdown caller's budget. Cancellation reports an incomplete drain and stops its queued admissions, local RPC waits, and pump timers. Owned loop cancellation completes normal cleanup; a canceled shutdown caller receives cancellation with its own token after cleanup. Each local attempt releases its managed admission lease during cleanup. Once pumps stop, the broadcast gate closes to further admissions. Repair cancellation is explicitly forwarded before disposing the round's linked sources, even when the local wait completes cancellation first. + +The protocol admission gate tracks local operation scopes. The separate send gates retain their concurrency limits, one-attempt-per-peer rule, and FIFO scheduling among ready destinations. Namespace application and remote execution retain their existing local budgets and cooperative cancellation contracts. + +A stalled destination occupies at most one active local broadcast slot and one independent local repair slot. Hop deadlines release those slots and signal cancellation, allowing queued broadcasts and future repair rounds to continue. Retry backoff, rotating peer selection, per-message budgets, and local admission together bound the pace and size of new attempts. + +The messaging runtime owns pending RPC completion. `InsideRuntimeClient` monitors callback expiry using the system-target response timeout, normally unless the method overrides it. Cancellation follows the configured acknowledgment policy, and `CallbackData.OnTimeout` completes and unregisters expired requests. Remote handlers can continue after a local dissemination attempt ends; their execution follows ordinary runtime cancellation and response-timeout behavior. + +Dissemination RPC contracts carry required cancellation tokens through their implementations, callers, and scheduler handoffs. Background loops receive their owned shutdown token explicitly. Deployment-load timer callbacks forward the timer's cancellation token through publication and direct statistics RPCs, and disposing the timer cancels an active publication. Fault observers follow late transport completion after a local broadcast or repair attempt ends. + +Deployment-load mutation and subscriber delivery run on the publisher's scheduler. Concurrent dictionary reads support namespace digests and repair construction. Direct publications retain their existing equal-timestamp notification behavior; dissemination classifies equal timestamps as duplicates so repair settles without repeated subscriber delivery. + +## Observability + +The standard `Microsoft.Orleans` meter exposes publication outcomes, broadcast volume, payload bytes, apply outcomes, retries, failures, anti-entropy work, drops, pump failures, and queue admission rejection. Dimensions are bounded to namespace, direction, kind, result, reason, truncation, and pump status. Diagnostic events provide payload-level apply/drop detail and deterministic pump scheduling detail for short-lived investigation. + +See [Monitor runtime dissemination](../host/monitoring/runtime-dissemination.md) for instrument semantics, dashboards, alerts, and failure diagnosis. diff --git a/docs/site/src/content/docs/toc.yml b/docs/site/src/content/docs/toc.yml index 02dd32bf151..1c1fa62c282 100644 --- a/docs/site/src/content/docs/toc.yml +++ b/docs/site/src/content/docs/toc.yml @@ -267,6 +267,8 @@ items: href: host/monitoring/signals.md - name: Metrics href: host/monitoring/metrics.md + - name: Runtime dissemination + href: host/monitoring/runtime-dissemination.md - name: Metrics catalog href: host/monitoring/metrics-catalog.md - name: Troubleshooting workflow @@ -347,6 +349,8 @@ items: href: implementation/cluster-management.md - name: Cluster manifest retrieval href: implementation/cluster-manifest-retrieval.md + - name: Runtime state dissemination + href: implementation/runtime-dissemination.md - name: Runtime lifecycle href: implementation/orleans-lifecycle.md - name: Scheduling and turn execution diff --git a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs index 246e7dbdbd6..bce7fab91f8 100644 --- a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs +++ b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs @@ -93,6 +93,16 @@ public TimeSpan MaxProbeTimeout /// Membership updates are disseminated using gossip by default. public bool UseLivenessGossip { get; set; } = true; + /// + /// Gets or sets dissemination options for membership updates. + /// + /// + /// Membership dissemination includes Joining, Active, ShuttingDown, and Stopping silos and is a + /// best-effort accelerator. Direct membership gossip delivers shutdown-critical updates and reaches peers + /// during mixed-version operation. + /// + public DisseminationNamespaceOptions Dissemination { get; set; } = new(); + /// /// Gets or sets the number of silos each silo probes for liveness. /// diff --git a/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs new file mode 100644 index 00000000000..13ba4aa991b --- /dev/null +++ b/src/Orleans.Core/Configuration/Options/DisseminationOptions.cs @@ -0,0 +1,199 @@ +using System; + +namespace Orleans.Configuration; + +/// +/// Options for configuring internal silo-to-silo dissemination. +/// +public sealed class DisseminationOptions +{ + /// + /// Gets or sets a value indicating whether the dissemination subsystem is enabled. + /// + /// . Enable dissemination explicitly after configuring the participating namespaces. + public bool Enabled { get; set; } + + /// + /// Gets or sets the maximum number of concurrent local dissemination broadcast attempts. + /// + /// + /// Each attempt releases capacity when its local wait completes, including timeout or cancellation. + /// + public int MaxConcurrentSends { get; set; } = 32; + + /// + /// Gets or sets the maximum total payload bytes in one dissemination batch. + /// + /// + /// Bounds outgoing batches and the values admitted from each incoming broadcast or repair response. + /// + public int MaxBatchBytes { get; set; } = 1024 * 1024; + + /// + /// Gets or sets the maximum number of items in one dissemination batch. + /// + /// + /// Bounds outgoing batches and the values examined in each incoming broadcast or repair response. + /// + public int MaxBatchItems { get; set; } = 8 * 1024; + + /// + /// Gets or sets overlay-specific dissemination options. + /// + public DisseminationOverlayOptions Overlay { get; set; } = new(); +} + +/// +/// Options for the dissemination overlay. +/// +public sealed class DisseminationOverlayOptions +{ + /// + /// Gets or sets the code-configured fanout selector for the membership broadcast forest. + /// + /// + /// The argument is the current member count for the membership broadcast forest. + /// The selector must return the same fanout on every silo for a given member count so that forwarding + /// routes agree across the cluster. + /// When this value is , , , + /// and are used to derive a fanout factor. + /// Aggregation trees use instead. + /// + public Func? FanOutFactor { get; set; } + + /// + /// Gets or sets the target number of tree hops used by the bindable fanout selector. + /// + public int TargetHopCount { get; set; } = 2; + + /// + /// Gets or sets the minimum fanout factor used by the bindable fanout selector. + /// + public int MinFanOutFactor { get; set; } = 4; + + /// + /// Gets or sets the maximum fanout factor used by the bindable fanout selector. + /// + public int MaxFanOutFactor { get; set; } = 32; + + /// + /// Gets or sets the maximum number of children per node in the aggregation distribution tree. + /// + /// + /// This value is independent of the membership broadcast forest's selector + /// and is clamped to the member count. Configure the same value on every silo so that distribution + /// routes agree across the cluster. Producers continue to send updates directly to the aggregation root. + /// + /// The default is 8 and the value must be greater than zero. + public int AggregationFanOutFactor { get; set; } = 8; + + /// + /// Gets or sets the interval between anti-entropy repair rounds. + /// + /// The interval is 5 seconds by default and must be between 1 millisecond and approximately 49.7 days. + public TimeSpan AntiEntropyInterval { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// Gets or sets the number of peers contacted during each anti-entropy repair round. + /// + /// + /// This also bounds concurrent local repair attempts across overlapping rounds. Each attempt releases + /// capacity when its local wait completes, including timeout or cancellation. + /// + public int AntiEntropyPeerCount { get; set; } = 3; + + /// + /// Gets or sets the maximum number of items in one anti-entropy repair batch. + /// + /// + /// The effective limit is the minimum of this value, , + /// and the peer's advertised receive budget. This allows repair batches to be tuned independently + /// of broadcast batches. + /// + /// The default is 8192 and the value must be greater than zero. + public int MaxAntiEntropyBatchItems { get; set; } = 8 * 1024; + + /// + /// Gets or sets the maximum total payload bytes in one anti-entropy repair batch. + /// + /// + /// The effective limit is the minimum of this value, , + /// and the peer's advertised receive budget. This allows repair batches to be tuned independently + /// of broadcast batches. + /// + /// The default is 1048576 and the value must be greater than zero. + public int MaxAntiEntropyBatchBytes { get; set; } = 1024 * 1024; + + internal int GetFanOutFactor(int memberCount) + { + var count = Math.Max(1, memberCount); + var selectedFanOut = FanOutFactor?.Invoke(count) ?? GetConfiguredFanOutFactor(count); + return Math.Clamp(selectedFanOut, 1, count); + } + + internal int GetConfiguredFanOutFactor(int memberCount) + { + var count = Math.Max(1, memberCount); + var targetHopCount = Math.Max(1, TargetHopCount); + var scaled = targetHopCount switch + { + 1 => count, + 2 => Math.Sqrt(count), + 3 => Math.Cbrt(count), + _ => Math.Pow(count, 1d / targetHopCount), + }; + var min = Math.Max(1, MinFanOutFactor); + var max = Math.Max(min, MaxFanOutFactor); + return (int)Math.Ceiling(Math.Max(min, Math.Min(scaled, max))); + } +} + +/// +/// Options for a dissemination namespace. +/// +public sealed class DisseminationNamespaceOptions +{ + /// + /// Gets or sets a value indicating whether this namespace is enabled. + /// + /// . Enable each participating namespace explicitly. + public bool Enabled { get; set; } + + /// + /// Gets or sets the hard maximum number of distinct pending keys retained for this namespace by each peer. + /// + /// + /// The bound applies independently to each namespace in each peer pump. A notification for a key which is + /// already retained updates that key at the limit, while a new distinct key is rejected with diagnostics until + /// acknowledged delivery or membership pruning releases capacity. Values are materialized from namespace state + /// only when they are sent, so this bounds retained identities rather than serialized batch bytes. + /// + public int MaxPendingItemCount { get; set; } = 1024; + + /// + /// Gets or sets the local transport and application budgets for a namespace value. + /// + /// + /// Each hop has independent local transport and application windows. Broadcast application ages from + /// the start of receiver processing, including earlier items in the batch. Anti-entropy application + /// starts a new window after exchanges finish. Received lifetimes are capped by the local namespace + /// setting, and owners observe cancellation before applying queued state. Forwarding re-materializes + /// the current value with a new hop lifetime. These local windows use each silo's own clock. + /// + /// The lifetime is 30 seconds by default and must be greater than zero. + public TimeSpan StaleItemTtl { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the expected cadence for updates in this namespace. + /// + /// + /// Anti-entropy requests omit keys whose version advanced within this interval. Duplicate values do not + /// postpone repair probes. + /// + public TimeSpan ExpectedUpdateCadence { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Gets or sets the maximum serialized payload size for this namespace. + /// + public int MaxPayloadBytes { get; set; } = 1024 * 1024; +} diff --git a/src/Orleans.Core/OrleansContracts.txt b/src/Orleans.Core/OrleansContracts.txt index 2536412af67..5a30a3f33cd 100644 --- a/src/Orleans.Core/OrleansContracts.txt +++ b/src/Orleans.Core/OrleansContracts.txt @@ -57,6 +57,11 @@ interface [GrainInterfaceType("Orleans.Runtime.IClusterManifestSystemTarget")] O interface [GrainInterfaceType("Orleans.Runtime.IDeploymentLoadPublisher")] Orleans.Runtime.IDeploymentLoadPublisher [Version(0)] C5255F0C: UpdateRuntimeStatistics(Orleans.Runtime.SiloAddress, Orleans.Runtime.SiloRuntimeStatistics, System.Threading.CancellationToken) -> Task +interface [GrainInterfaceType("Orleans.Runtime.IDisseminationSystemTarget")] Orleans.Runtime.IDisseminationSystemTarget [Version(0)] + EF58CB3D: ExchangeAntiEntropy(Orleans.Runtime.DisseminationAntiEntropyRequest, System.Threading.CancellationToken) -> Task + 8C810F59: PublishAggregated(Orleans.Runtime.DisseminationPublicationRequest, System.Threading.CancellationToken) -> Task + 017AA907: PushBroadcast(Orleans.Runtime.DisseminationBroadcastBatch, System.Threading.CancellationToken) -> Task + interface [GrainInterfaceType("Orleans.Runtime.IGrainCallCancellationExtension")] Orleans.Runtime.IGrainCallCancellationExtension [Version(0)] FA239824: CancelRequestAsync(Orleans.Runtime.GrainId, Orleans.Runtime.CorrelationId, System.Threading.CancellationToken) -> ValueTask diff --git a/src/Orleans.Core/Runtime/Constants.cs b/src/Orleans.Core/Runtime/Constants.cs index 8efb4156c60..d84129d9e8e 100644 --- a/src/Orleans.Core/Runtime/Constants.cs +++ b/src/Orleans.Core/Runtime/Constants.cs @@ -17,6 +17,7 @@ internal static class Constants public static readonly GrainType MembershipServiceType = SystemTargetGrainId.CreateGrainType("clustering"); public static readonly GrainType SystemMembershipTableType = SystemTargetGrainId.CreateGrainType("clustering.dev"); public static readonly GrainType DeploymentLoadPublisherSystemTargetType = SystemTargetGrainId.CreateGrainType("load-publisher"); + public static readonly GrainType DisseminationSystemTargetType = SystemTargetGrainId.CreateGrainType("dissemination"); public static readonly GrainType TestHooksSystemTargetType = SystemTargetGrainId.CreateGrainType("test.hooks"); public static readonly GrainType TransactionAgentSystemTargetType = SystemTargetGrainId.CreateGrainType("txn.agent"); public static readonly GrainType StreamProviderManagerAgentSystemTargetType = SystemTargetGrainId.CreateGrainType("stream.provider-manager"); @@ -46,6 +47,7 @@ internal static class Constants {CatalogType,"Catalog"}, {MembershipServiceType,"MembershipService"}, {DeploymentLoadPublisherSystemTargetType, "DeploymentLoadPublisherSystemTarget"}, + {DisseminationSystemTargetType, "DisseminationSystemTarget"}, {StreamProviderManagerAgentSystemTargetType,"StreamProviderManagerAgent"}, {TestHooksSystemTargetType,"TestHooksSystemTargetType"}, {TransactionAgentSystemTargetType,"TransactionAgentSystemTarget"}, diff --git a/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs new file mode 100644 index 00000000000..bae02f50690 --- /dev/null +++ b/src/Orleans.Core/SystemTargetInterfaces/IDisseminationSystemTarget.cs @@ -0,0 +1,322 @@ +namespace Orleans.Runtime; + +internal interface IDisseminationSystemTarget : ISystemTarget +{ + // The response carries receiver versions, which are the evidence used to sequence later repairs. + [Alias("017AA907")] + Task PushBroadcast(DisseminationBroadcastBatch batch, CancellationToken cancellationToken); + + [Alias("EF58CB3D")] + Task ExchangeAntiEntropy(DisseminationAntiEntropyRequest request, CancellationToken cancellationToken); + + [Alias("8C810F59")] + Task PublishAggregated(DisseminationPublicationRequest request, CancellationToken cancellationToken); +} + +[GenerateSerializer, Immutable] +internal sealed class DisseminationPublicationRequest +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public DisseminationNamespace Namespace { get; init; } + + [Id(2)] + public required DisseminationBroadcastValue Value { get; init; } +} + +// Admission covers the sealed cohort's distribution work. The delay targets its next sampling boundary. +[GenerateSerializer, Immutable] +internal readonly record struct DisseminationPublicationReceipt( + [property: Id(0)] bool Accepted, + [property: Id(1)] TimeSpan NextPublicationDelay); + +[GenerateSerializer, Immutable] +internal readonly struct DisseminationNamespace : IEquatable, IComparable, IComparable, ISpanFormattable +{ + [Id(0)] + private readonly string? _value; + + public DisseminationNamespace(string value) + { + ArgumentException.ThrowIfNullOrEmpty(value); + _value = value; + } + + public string Value => _value ?? throw new InvalidOperationException($"A default {nameof(DisseminationNamespace)} value is invalid."); + + public static implicit operator DisseminationNamespace(string value) => new(value); + + public static implicit operator string(DisseminationNamespace value) => value.Value; + + public static bool operator ==(DisseminationNamespace left, DisseminationNamespace right) => left.Equals(right); + + public static bool operator !=(DisseminationNamespace left, DisseminationNamespace right) => !left.Equals(right); + + public bool Equals(DisseminationNamespace other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is DisseminationNamespace other && Equals(other); + + public override int GetHashCode() => _value is null ? 0 : StringComparer.Ordinal.GetHashCode(_value); + + public int CompareTo(DisseminationNamespace other) => string.Compare(_value, other._value, StringComparison.Ordinal); + + public int CompareTo(object? obj) + { + if (obj is null) + { + return 1; + } + + return obj is DisseminationNamespace other + ? CompareTo(other) + : throw new ArgumentException($"Object must be of type {nameof(DisseminationNamespace)}.", nameof(obj)); + } + + public override string ToString() => _value ?? string.Empty; + + public string ToString(string? format, IFormatProvider? formatProvider) => _value ?? string.Empty; + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? formatProvider) + { + var value = _value.AsSpan(); + if (value.TryCopyTo(destination)) + { + charsWritten = value.Length; + return true; + } + + charsWritten = 0; + return false; + } +} + +[GenerateSerializer, Immutable] +internal readonly struct DisseminationKey(object? value) : IEquatable, IComparable, IComparable, ISpanFormattable +{ + public static readonly DisseminationKey Default = default; + + [Id(0)] + public readonly object? Value = value; + + public static implicit operator DisseminationKey(SiloAddress? value) => new(value); + + public static implicit operator DisseminationKey(string? value) => new(value); + + public static bool operator ==(DisseminationKey left, DisseminationKey right) => left.Equals(right); + + public static bool operator !=(DisseminationKey left, DisseminationKey right) => !left.Equals(right); + + public bool Equals(DisseminationKey other) => Equals(Value, other.Value); + + public override bool Equals(object? obj) => obj is DisseminationKey other && Equals(other); + + public override int GetHashCode() => Value?.GetHashCode() ?? 0; + + public int CompareTo(DisseminationKey other) => Compare(Value, other.Value); + + public int CompareTo(object? obj) + { + if (obj is null) + { + return 1; + } + + return obj is DisseminationKey other + ? CompareTo(other) + : throw new ArgumentException($"Object must be of type {nameof(DisseminationKey)}.", nameof(obj)); + } + + public override string ToString() => ToString(null, null); + + public string ToString(string? format, IFormatProvider? formatProvider) => Value switch + { + null => string.Empty, + IFormattable formattable => formattable.ToString(format, formatProvider), + _ => Value.ToString() ?? string.Empty, + }; + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? formatProvider) + { + if (Value is null) + { + charsWritten = 0; + return true; + } + + if (Value is ISpanFormattable spanFormattable) + { + return spanFormattable.TryFormat(destination, out charsWritten, format, formatProvider); + } + + var text = ToString(format.IsEmpty ? null : format.ToString(), formatProvider); + if (text.AsSpan().TryCopyTo(destination)) + { + charsWritten = text.Length; + return true; + } + + charsWritten = 0; + return false; + } + + private static int Compare(object? left, object? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return -1; + } + + if (right is null) + { + return 1; + } + + if (Equals(left, right)) + { + return 0; + } + + return left switch + { + string leftString when right is string rightString => StringComparer.Ordinal.Compare(leftString, rightString), + SiloAddress leftSilo when right is SiloAddress rightSilo => leftSilo.CompareTo(rightSilo), + string when right is SiloAddress => -1, + SiloAddress when right is string => 1, + _ => Comparer.Default.Compare(left, right), + }; + } +} + +[GenerateSerializer, Immutable] +internal readonly struct DigestEntry +{ + public DigestEntry(DisseminationKey key, long version, long fingerprint = 0) + { + Key = key; + Version = version; + Fingerprint = fingerprint; + } + + [Id(0)] + public DisseminationKey Key { get; } + + [Id(1)] + public long Version { get; } + + // Fingerprints distinguish meaningful same-version state, such as membership liveness. + [Id(2)] + public long Fingerprint { get; } +} + +// FromVersion zero carries full state; a positive baseline identifies a namespace-specific broadcast delta. +[GenerateSerializer, Immutable] +internal readonly struct DisseminationValue +{ + public DisseminationValue(DisseminationKey key, long fromVersion, long toVersion, ReadOnlyMemory payload) + { + Key = key; + FromVersion = fromVersion; + ToVersion = toVersion; + Payload = payload; + } + + [Id(0)] + public DisseminationKey Key { get; } + + [Id(1)] + public long FromVersion { get; } + + [Id(2)] + public long ToVersion { get; } + + [Id(3)] + public ReadOnlyMemory Payload { get; } +} + +// Lifetime is hop-local so forwarding re-materializes both the payload and its delivery window. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastValue +{ + [Id(0)] + public DisseminationValue Value { get; init; } + + [Id(1)] + public TimeSpan TimeToLive { get; init; } +} + +// Sender is the immediate hop, not the original publisher. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastBatch +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public Dictionary> Values { get; init; } = []; + + [Id(2)] + public bool SupportsCompactAcknowledgments { get; init; } +} + +// Acknowledgments report the versions the receiver actually holds after processing the batch. +[GenerateSerializer, Immutable] +internal sealed class DisseminationBroadcastResponse +{ + [Id(0)] + public Dictionary> Acknowledgments { get; init; } = []; + + [Id(1)] + public List UnsupportedNamespaces { get; init; } = []; + + // Certifies accepted values and the exact maximum transmitted version of every key. + // Namespace entries retain capability evidence. + [Id(2)] + public bool AllVersionsAcknowledged { get; init; } +} + +[GenerateSerializer, Immutable] +internal sealed class DisseminationAntiEntropyRequest +{ + [Id(0)] + public Dictionary> Digests { get; init; } = []; + + [Id(1)] + public required SiloAddress Sender { get; init; } + + [Id(2)] + public List SupportedNamespaces { get; init; } = []; + + // Older requests use the responder's own limits. + [Id(3)] + public int? MaxResponseItems { get; init; } + + [Id(4)] + public int? MaxResponseBytes { get; init; } +} + +[GenerateSerializer, Immutable] +internal sealed class DisseminationAntiEntropyResponse +{ + [Id(0)] + public required SiloAddress Sender { get; init; } + + [Id(1)] + public Dictionary> Values { get; init; } = []; + + // Truncation means at least one valid repair remains for a later round. + [Id(2)] + public bool Truncated { get; init; } + + [Id(3)] + public List SupportedNamespaces { get; init; } = []; + + [Id(4)] + public List UnsupportedNamespaces { get; init; } = []; +} diff --git a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs index 7a8b9824fb8..b7e9f58967d 100644 --- a/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/DeploymentLoadPublisherOptions.cs @@ -16,5 +16,26 @@ public class DeploymentLoadPublisherOptions /// The default value for . /// public static readonly TimeSpan DEFAULT_DEPLOYMENT_LOAD_PUBLISHER_REFRESH_TIME = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets dissemination options for deployment load statistics. + /// + /// + /// When all active peers confirm support, producers send updates directly to the aggregation root. + /// The root seals a cohort after one fresh contribution from each active silo, including itself, or after + /// elapses. Publication receipts complete at sealing and + /// align subsequent samples with the cohort's next publication boundary. Relays forward admitted cohorts + /// immediately through the parent/child tree. Batch item and byte limits can split a cohort into multiple wire messages. + /// Confirmation remains valid for that silo generation + /// until the peer explicitly rejects the namespace or leaves the eligible membership set. During bootstrap + /// or mixed-version operation, direct publication covers all active peers so that an unsupported intermediate + /// node cannot interrupt delivery. Direct publication also covers all active peers when dissemination is + /// unavailable, declines the update, throws, or exceeds the bounded publication receipt wait. + /// + public DisseminationNamespaceOptions Dissemination { get; set; } = new() + { + ExpectedUpdateCadence = TimeSpan.FromSeconds(5), + MaxPendingItemCount = 8 * 1024, + }; } } diff --git a/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs new file mode 100644 index 00000000000..771427c2517 --- /dev/null +++ b/src/Orleans.Runtime/Configuration/Options/DisseminationOptionsValidator.cs @@ -0,0 +1,116 @@ +using System; +using Microsoft.Extensions.Options; + +namespace Orleans.Configuration; + +internal sealed class DisseminationOptionsValidator : IValidateOptions +{ + private static readonly TimeSpan MaxPeriodicTimerPeriod = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + + public ValidateOptionsResult Validate(string? name, DisseminationOptions options) + { + if (options.MaxConcurrentSends <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxConcurrentSends)} must be greater than 0."); + } + + if (options.MaxBatchBytes <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxBatchBytes)} must be greater than 0."); + } + + if (options.MaxBatchItems <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOptions.MaxBatchItems)} must be greater than 0."); + } + + var overlay = options.Overlay; + if (overlay.TargetHopCount <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.TargetHopCount)} must be greater than 0."); + } + + if (overlay.MinFanOutFactor <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MinFanOutFactor)} must be greater than 0."); + } + + if (overlay.MaxFanOutFactor < overlay.MinFanOutFactor) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxFanOutFactor)} must be greater than or equal to {nameof(DisseminationOverlayOptions.MinFanOutFactor)}."); + } + + if (overlay.AggregationFanOutFactor <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AggregationFanOutFactor)} must be greater than 0."); + } + + if (overlay.AntiEntropyInterval < TimeSpan.FromMilliseconds(1) + || overlay.AntiEntropyInterval > MaxPeriodicTimerPeriod) + { + return ValidateOptionsResult.Fail( + $"{nameof(DisseminationOverlayOptions.AntiEntropyInterval)} must be between 1 millisecond and {MaxPeriodicTimerPeriod}."); + } + + if (overlay.AntiEntropyPeerCount <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.AntiEntropyPeerCount)} must be greater than 0."); + } + + if (overlay.MaxAntiEntropyBatchItems <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxAntiEntropyBatchItems)} must be greater than 0."); + } + + if (overlay.MaxAntiEntropyBatchBytes <= 0) + { + return ValidateOptionsResult.Fail($"{nameof(DisseminationOverlayOptions.MaxAntiEntropyBatchBytes)} must be greater than 0."); + } + + return ValidateOptionsResult.Success; + } +} + +internal sealed class DisseminationNamespaceOptionsValidator +{ + public static ValidateOptionsResult Validate(string owner, DisseminationNamespaceOptions options) + { + if (options.MaxPendingItemCount <= 0) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.MaxPendingItemCount)} must be greater than 0."); + } + + if (options.StaleItemTtl <= TimeSpan.Zero) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.StaleItemTtl)} must be greater than 0."); + } + + if (options.ExpectedUpdateCadence <= TimeSpan.Zero) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.ExpectedUpdateCadence)} must be greater than 0."); + } + + if (options.MaxPayloadBytes <= 0) + { + return ValidateOptionsResult.Fail($"{owner}.{nameof(DisseminationNamespaceOptions.MaxPayloadBytes)} must be greater than 0."); + } + + return ValidateOptionsResult.Success; + } +} + +internal sealed class DeploymentLoadPublisherOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, DeploymentLoadPublisherOptions options) => + DisseminationNamespaceOptionsValidator.Validate( + $"{nameof(DeploymentLoadPublisherOptions)}.{nameof(DeploymentLoadPublisherOptions.Dissemination)}", + options.Dissemination); +} + +internal sealed class ClusterMembershipOptionsDisseminationValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, ClusterMembershipOptions options) => + DisseminationNamespaceOptionsValidator.Validate( + $"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.Dissemination)}", + options.Dissemination); +} diff --git a/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs new file mode 100644 index 00000000000..a9dbb6d6c21 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DeploymentLoadStatisticsDisseminationNamespace.cs @@ -0,0 +1,157 @@ +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Serialization; + +namespace Orleans.Runtime.Dissemination; + +// A newer load sample supersedes every older sample, so this namespace never needs a version chain. +internal sealed class DeploymentLoadStatisticsDisseminationNamespace( + DeploymentLoadPublisher deploymentLoadPublisher, + IOptionsMonitor options, + Serializer serializer) : IDisseminationNamespace +{ + private readonly object _cacheLock = new(); + private readonly Dictionary _cachedValues = []; + + public DisseminationNamespace Name => DisseminationNamespaceNames.DeploymentLoad; + + public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.ActiveMembers; + + public DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.AggregationTree; + + public TimeSpan AggregationPeriod => options.CurrentValue.DeploymentLoadPublisherRefreshTime; + + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; + + public DisseminationValue CreateValue(SiloAddress origin, SiloRuntimeStatistics statistics) + { + lock (_cacheLock) + { + // Reuse serialization until this silo publishes a new timestamp. + if (_cachedValues.TryGetValue(origin, out var cached) + && cached.ToVersion == statistics.DateTime.Ticks) + { + return cached; + } + + var result = new DisseminationValue( + origin, + fromVersion: 0, + toVersion: statistics.DateTime.Ticks, + serializer.SerializeToArray(statistics)); + _cachedValues[origin] = result; + return result; + } + } + + public IEnumerable Keys + { + get + { + var activeSilos = deploymentLoadPublisher.GetActiveSiloStatusesForStatisticsDigest(); + PruneCache(activeSilos); + foreach (var siloAddress in activeSilos.Keys) + { + yield return siloAddress; + } + } + } + + public IEnumerable Digests + { + get + { + foreach (var key in Keys) + { + yield return new DigestEntry(key, GetVersion(key)); + } + } + } + + public long GetVersion(DisseminationKey key) => + key.Value is SiloAddress siloAddress + && deploymentLoadPublisher.PeriodicStatistics.TryGetValue(siloAddress, out var statistics) + && !deploymentLoadPublisher.IsRuntimeStatisticsObsolete(siloAddress, statistics.DateTime.Ticks) + ? statistics.DateTime.Ticks + : 0; + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (request.Key.Value is not SiloAddress siloAddress + || !deploymentLoadPublisher.PeriodicStatistics.TryGetValue(siloAddress, out var statistics) + || deploymentLoadPublisher.IsRuntimeStatisticsObsolete(siloAddress, statistics.DateTime.Ticks)) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + var version = statistics.DateTime.Ticks; + if (request.FromVersion is { } peerVersion && peerVersion >= version) + { + return DisseminationRepairResult.Current(version); + } + + // Every repair is a full value from zero, making the peer's exact baseline irrelevant. + var value = CreateValue(siloAddress, statistics); + return value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(value) + : DisseminationRepairResult.InsufficientCapacity(version); + } + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (value.Key.Value is not SiloAddress siloAddress) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + if (value.FromVersion != 0 + || serializer.Deserialize(value.Payload) is not { } statistics + || value.ToVersion != statistics.DateTime.Ticks) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + cancellationToken.ThrowIfCancellationRequested(); + return ApplyAndCache(siloAddress, statistics, value, cancellationToken); + } + + private async ValueTask ApplyAndCache( + SiloAddress origin, + SiloRuntimeStatistics statistics, + DisseminationValue value, + CancellationToken cancellationToken) + { + var result = await deploymentLoadPublisher.ApplyDisseminatedRuntimeStatisticsAsync(origin, statistics, cancellationToken); + if (result is DisseminationApplyResult.Applied) + { + lock (_cacheLock) + { + // The owner accepted these exact bytes. Forward them without serializing the same sample again. + if (!_cachedValues.TryGetValue(origin, out var cached) || cached.ToVersion < value.ToVersion) + { + _cachedValues[origin] = value; + } + } + } + + return result; + } + + private void PruneCache(Dictionary activeSilos) + { + lock (_cacheLock) + { + foreach (var key in _cachedValues.Keys) + { + if (!activeSilos.ContainsKey(key)) + { + _cachedValues.Remove(key); + } + } + } + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs b/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs new file mode 100644 index 00000000000..d0f1e741642 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationApplyResult.cs @@ -0,0 +1,9 @@ +namespace Orleans.Runtime.Dissemination; + +internal enum DisseminationApplyResult +{ + Applied, + Duplicate, + Obsolete, + Rejected, +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs new file mode 100644 index 00000000000..1545e88628b --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationBroadcastQueue.cs @@ -0,0 +1,1646 @@ +using System.Collections.Frozen; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.ObjectPool; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Internal; + +namespace Orleans.Runtime.Dissemination; + +// The queue remembers what each peer has acknowledged. +// Payloads are materialized from namespace state only when a peer is ready to send. +internal sealed partial class DisseminationBroadcastQueue +{ + private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan MaxTransportLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + private readonly TimeProvider _timeProvider; + private readonly SiloAddress _localSilo; + private readonly IInternalGrainFactory _grainFactory; + private readonly IOptionsMonitor _options; + private readonly FrozenDictionary _namespaces; + private readonly ILogger _logger; + private readonly Action? _responseObserver; + private readonly object _lock = new(); + private readonly Dictionary _peers = []; + private readonly DisseminationSendGate _sendGate; + private readonly ObjectPool>> _inventoryPool; + private bool _stopped; + + public DisseminationBroadcastQueue( + TimeProvider timeProvider, + SiloAddress localSilo, + IInternalGrainFactory grainFactory, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + ILogger logger, + Action? responseObserver = null) + { + _timeProvider = timeProvider; + _localSilo = localSilo; + _grainFactory = grainFactory; + _options = options; + _namespaces = disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + _logger = logger; + _responseObserver = responseObserver; + _sendGate = new(Math.Max(1, options.CurrentValue.MaxConcurrentSends)); + _inventoryPool = new DefaultObjectPool>>( + new InventoryPoolPolicy(_namespaces), maximumRetained: 1); + } + + public bool Notify( + SiloAddress peer, + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + bool force = true) + { + PeerQueuePump pump; + lock (_lock) + { + if (_stopped) + { + return false; + } + + pump = GetOrCreatePeerUnsafe(peer); + } + + // Notification diagnostics run outside the queue lock and can reenter the queue. + var version = disseminationNamespace.GetVersion(key); + return pump.Notify(disseminationNamespace, [new(key, version, force)]); + } + + public bool NotifyBatch( + SiloAddress peer, + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + PeerQueuePump pump; + lock (_lock) + { + if (_stopped) + { + return false; + } + + if (notifications.IsEmpty) + { + return true; + } + + pump = GetOrCreatePeerUnsafe(peer); + } + + return pump.Notify(disseminationNamespace, notifications); + } + + internal readonly record struct KeyNotification(DisseminationKey Key, long Version, bool Force); + + public void ObservePeerVersion( + SiloAddress peer, + DisseminationNamespace namespaceName, + DisseminationKey key, + long version) + { + if (version < 0 + || !_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + || !disseminationNamespace.Options.Enabled) + { + return; + } + + // Aggregation acknowledgments also confirm distribution processing. A peer's possession alone + // cannot suppress distribution: its descendants may still need the value. + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree) + { + return; + } + + // Passive evidence only sharpens an existing ledger; it must not allocate a timer for a peer with no outbound work. + lock (_lock) + { + if (_stopped || !_peers.TryGetValue(peer, out var pending)) + { + return; + } + + pending.ObservePeerVersion(disseminationNamespace, key, version); + } + } + + public async Task FlushPendingBroadcast(CancellationToken cancellationToken) + { + List peers; + lock (_lock) + { + peers = [.. _peers.Values.OrderBy(static peer => peer.Peer)]; + } + + await Task.WhenAll(peers.Select(peer => peer.FlushAsync(cancellationToken).AsTask())); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + List peers; + lock (_lock) + { + if (_stopped) + { + return; + } + + _stopped = true; + peers = [.. _peers.Values]; + _peers.Clear(); + } + + try + { + await Task.WhenAll(peers.Select(peer => peer.StopAsync(drain: true, cancellationToken).AsTask())); + } + finally + { + _sendGate.Stop(); + } + } + + public async Task Prune( + DisseminationMembershipSnapshots membershipSnapshots, + CancellationToken cancellationToken) + { + // Namespace identities are the authoritative inventory for retiring clean ledger entries. + var activeKeys = _inventoryPool.Get(); + try + { + foreach (var disseminationNamespace in _namespaces.Values) + { + if (disseminationNamespace.Options.Enabled) + { + activeKeys[disseminationNamespace.Name].UnionWith(disseminationNamespace.Keys); + } + } + + List? removedPeers = null; + List retainedPeers; + lock (_lock) + { + retainedPeers = new(_peers.Count); + foreach (var (peer, pending) in _peers) + { + if (!_localSilo.Equals(peer) && !membershipSnapshots.AllMembers.ContainsMember(peer)) + { + (removedPeers ??= []).Add(pending); + } + else + { + retainedPeers.Add(pending); + } + } + + if (removedPeers is not null) + { + foreach (var pending in removedPeers) + { + _peers.Remove(pending.Peer); + } + } + } + + foreach (var peer in retainedPeers) + { + peer.PruneKeys(activeKeys, membershipSnapshots); + } + + if (removedPeers is not null) + { + await Task.WhenAll(removedPeers.Select(peer => peer.StopAsync(drain: false, cancellationToken).AsTask())); + } + } + finally + { + _inventoryPool.Return(activeKeys); + } + } + + private sealed class InventoryPoolPolicy(FrozenDictionary namespaces) + : PooledObjectPolicy>> + { + public override Dictionary> Create() + { + var result = new Dictionary>(namespaces.Count); + foreach (var ns in namespaces.Values) + { + result.Add(ns.Name, []); + } + + return result; + } + + public override bool Return(Dictionary> inventory) + { + foreach (var keys in inventory.Values) + { + keys.Clear(); + } + + return true; + } + } + + private PeerQueuePump GetOrCreatePeerUnsafe(SiloAddress peer) + { + if (!_peers.TryGetValue(peer, out var result)) + { + result = new(peer, this); + _peers.Add(peer, result); + } + + return result; + } + + private TimeSpan GetRetryDelay(int attempt) + { + // Failed local attempts back off independently of the root's load collection period. + var floor = InitialRetryDelay; + var cap = _options.CurrentValue.Overlay.AntiEntropyInterval; + if (floor > cap) + { + floor = cap; + } + + var multiplier = Math.Pow(2, Math.Min(Math.Max(0, attempt - 1), 20)); + return TimeSpan.FromTicks((long)Math.Min(cap.Ticks, floor.Ticks * multiplier)); + } + + private sealed class PeerQueuePump + { + private readonly DisseminationBroadcastQueue _owner; + private readonly Action _admissionQueued; + private readonly object _lock = new(); + private readonly CancellationTokenSource _shutdownCts = new(); + private readonly WakeTimer _flushTimer; + private readonly Task _flushTask; + private readonly Dictionary _statesByNamespace = []; + private TaskCompletionSource _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource? _activeFlushCompletion; + private Exception? _pumpFailure; + private IDisseminationSystemTarget? _target; + // Notifications arriving during a send advance the epoch and remain dirty for the next pass. + private long _notificationEpoch; + private int _retryAttempt; + private bool _stopping; + private bool _draining; + + public PeerQueuePump(SiloAddress peer, DisseminationBroadcastQueue owner) + { + Peer = peer; + _owner = owner; + _admissionQueued = EmitAdmissionQueued; + _flushTimer = new(owner._timeProvider); + using var _ = new ExecutionContextSuppressor(); + _flushTask = RunScheduledFlush(_shutdownCts.Token); + } + + public SiloAddress Peer { get; } + + private int DirtyCount { get; set; } + + public bool Notify( + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + ScheduledFlush? scheduled = null; + var rejections = 0; + var accepted = true; + lock (_lock) + { + var anyChanged = false; + foreach (var notification in notifications) + { + accepted &= NotifyKeyUnsafe( + disseminationNamespace, notification.Key, notification.Version, notification.Force, + out var changed, out var rejected); + anyChanged |= changed; + rejections += rejected ? 1 : 0; + } + + if (anyChanged) + { + // Only the aggregation root collects load; peer pumps send each accepted batch immediately. + _flushTimer.Wake(); + scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + } + + EmitNotification(disseminationNamespace, rejections, scheduled); + return accepted; + } + + private bool NotifyKeyUnsafe( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + bool force, + out bool changed, + out bool admissionRejected) + { + changed = false; + admissionRejected = false; + if (_stopping) + { + return false; + } + + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); + namespaceState.Keys.TryGetValue(key, out var keyState); + if (!force) + { + // Duplicate deliveries seed unknown peers without perpetuating cycles between skewed views. + if (keyState is { Dirty: true } or { InFlight: true } && keyState.NotificationVersion >= version) + { + return true; + } + + var knownVersion = keyState?.KnownVersion; + if (knownVersion is null && namespaceState.KnownVersions.TryGetValue(key, out var recordedVersion)) + { + knownVersion = recordedVersion.Version; + } + + if (knownVersion >= version) + { + return true; + } + } + + if (keyState is null && namespaceState.Keys.Count >= disseminationNamespace.Options.MaxPendingItemCount) + { + admissionRejected = true; + return false; + } + + keyState ??= namespaceState.AddKey(key); + keyState.NotificationVersion = version; + keyState.NotificationGeneration = ++_notificationEpoch; + _retryAttempt = 0; + MarkDirtyUnsafe(keyState); + changed = true; + return true; + } + + private void EmitNotification(IDisseminationNamespace disseminationNamespace, int rejections, ScheduledFlush? scheduled) + { + for (var index = 0; index < rejections; index++) + { + try + { + DisseminationInstruments.OnQueueAdmissionRejected(disseminationNamespace.Name); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + + try + { + DisseminationEvents.EmitQueueAdmissionRejected( + _owner._localSilo, + Peer, + disseminationNamespace.Name, + disseminationNamespace.Options.MaxPendingItemCount); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + EmitScheduled(scheduled); + } + + private void EmitScheduled(ScheduledFlush? scheduled) + { + if (scheduled is { } info) + { + try + { + DisseminationInstruments.OnBroadcastScheduled(info.Reason); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + + try + { + DisseminationEvents.EmitBroadcastScheduled(_owner._localSilo, Peer, info.Reason, info.DueTime, info.Attempt, info.Epoch); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + } + + private void EmitAdmissionQueued() + { + try + { + DisseminationEvents.EmitSendGate(_owner._localSilo, Peer, kind: "broadcast", stage: "queued"); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + // Captures a scheduling decision made under the lock so the diagnostic can be emitted after the lock is released. + private readonly record struct ScheduledFlush( + DisseminationBroadcastScheduleReason Reason, + TimeSpan DueTime, + int Attempt, + long Epoch); + + public void ObservePeerVersion( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version) + { + lock (_lock) + { + if (_stopping) + { + return; + } + + // Peer knowledge is evidence-driven and monotonic across acknowledgments, pushes, and anti-entropy. + var namespaceState = GetOrCreateNamespaceStateUnsafe(disseminationNamespace); + if (namespaceState.Keys.TryGetValue(key, out var keyState)) + { + if (keyState.KnownVersion is not { } knownVersion || version > knownVersion) + { + keyState.KnownVersion = version; + } + } + else + { + namespaceState.ObserveKnownVersion(key, version); + } + } + } + + public void PruneKeys( + Dictionary> activeKeys, + DisseminationMembershipSnapshots membershipSnapshots) + { + TaskCompletionSource? droppedFlushCompletion = null; + var droppedDirtyCount = 0; + lock (_lock) + { + foreach (var (namespaceName, namespaceState) in _statesByNamespace) + { + if (!membershipSnapshots.GetSnapshot(namespaceState.Namespace.MembershipScope).ContainsMember(Peer)) + { + var removedDirtyCount = namespaceState.Keys.Values.Count(static key => key.Dirty); + droppedDirtyCount += removedDirtyCount; + DirtyCount -= removedDirtyCount; + _statesByNamespace.Remove(namespaceName); + continue; + } + + activeKeys.TryGetValue(namespaceName, out var namespaceKeys); + foreach (var (key, keyState) in namespaceState.Keys) + { + if (!keyState.Dirty + && !keyState.InFlight + && (namespaceKeys is null || !namespaceKeys.Contains(key))) + { + namespaceState.Keys.Remove(key); + } + } + + namespaceState.PruneKnownVersions(namespaceKeys); + if (namespaceState.Keys.Count == 0 && namespaceState.KnownVersions.Count == 0) + { + _statesByNamespace.Remove(namespaceName); + } + } + + if (droppedDirtyCount > 0 && DirtyCount == 0 && !_nextFlushCompletion.Task.IsCompleted) + { + droppedFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + droppedFlushCompletion?.TrySetResult(); + } + + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task? flushCompletion; + lock (_lock) + { + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + if (DirtyCount > 0) + { + flushCompletion = _nextFlushCompletion.Task; + _flushTimer.Wake(); + } + else + { + flushCompletion = _activeFlushCompletion?.Task; + } + } + + if (flushCompletion is null) + { + return; + } + + flushCompletion.Ignore(); + await flushCompletion.WaitAsync(cancellationToken); + } + + public async ValueTask StopAsync(bool drain, CancellationToken cancellationToken) + { + Task? flushCompletion; + TaskCompletionSource? droppedFlushCompletion = null; + var alreadyStopping = false; + lock (_lock) + { + if (_stopping) + { + alreadyStopping = true; + flushCompletion = null; + } + else + { + _stopping = true; + _draining = drain; + if (drain) + { + if (_pumpFailure is { } pumpFailure) + { + flushCompletion = Task.FromException(pumpFailure); + } + else if (DirtyCount > 0) + { + flushCompletion = _nextFlushCompletion.Task; + } + else + { + flushCompletion = _activeFlushCompletion?.Task; + } + } + else + { + droppedFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + ClearPendingUnsafe(); + flushCompletion = null; + } + } + } + + if (alreadyStopping) + { + await _flushTask.WaitAsync(cancellationToken); + return; + } + + droppedFlushCompletion?.TrySetResult(); + try + { + lock (_lock) + { + if (ReferenceEquals(flushCompletion, _nextFlushCompletion.Task)) + { + _flushTimer.Wake(); + } + } + + while (flushCompletion is not null) + { + flushCompletion.Ignore(); + await flushCompletion.WaitAsync(cancellationToken); + lock (_lock) + { + if (_pumpFailure is { } pumpFailure) + { + throw new InvalidOperationException($"The dissemination broadcast pump for {Peer} has failed.", pumpFailure); + } + + flushCompletion = DirtyCount > 0 ? _nextFlushCompletion.Task : _activeFlushCompletion?.Task; + } + } + } + finally + { + lock (_lock) + { + _draining = false; + } + + await _shutdownCts.CancelAsync(); + _flushTimer.Dispose(); + try + { + await _flushTask; + } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + } + + _shutdownCts.Dispose(); + } + } + + private async Task RunScheduledFlush(CancellationToken cancellationToken) + { + try + { + while (await _flushTimer.WaitAsync(cancellationToken)) + { + try + { + await RunScheduledFlushIteration(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (!TryRecoverFromUnexpectedIterationFailure(exception)) + { + return; + } + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + } + + private async Task RunScheduledFlushIteration(CancellationToken cancellationToken) + { + TaskCompletionSource flushCompletion; + List work; + long notificationEpoch; + // Move one dirty generation to in-flight atomically; a concurrent notification can mark it dirty again. + lock (_lock) + { + // All wakes issued before this drain refer to the work being consumed now. + _flushTimer.Reset(); + if (DirtyCount == 0) + { + return; + } + + flushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + _activeFlushCompletion = flushCompletion; + notificationEpoch = _notificationEpoch; + work = DrainDirtyUnsafe(); + } + + var result = default(SendWorkResult); + try + { + result = await SendValues(work, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + // Restore accepted identities before invoking diagnostics, whose callbacks can throw or reenter. + Requeue(work); + result = new(RequiresBackoff: true, MadeProgress: false); + LogDebugBroadcastFlushFailed(_owner._logger, exception); + EmitPumpFailure(DisseminationPumpFailureStatus.Recovered); + } + finally + { + ScheduledFlush? scheduled = null; + lock (_lock) + { + if (result.MadeProgress) + { + _retryAttempt = 0; + } + + // A newer notification already signaled the next pass. + if (DirtyCount > 0 && (!_stopping || _draining) && notificationEpoch == _notificationEpoch) + { + if (result.RequiresBackoff) + { + _retryAttempt++; + var delay = _owner.GetRetryDelay(_retryAttempt); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); + } + else + { + _flushTimer.Wake(); + scheduled = new(DisseminationBroadcastScheduleReason.Immediate, TimeSpan.Zero, _retryAttempt, _notificationEpoch); + } + } + + if (ReferenceEquals(_activeFlushCompletion, flushCompletion)) + { + _activeFlushCompletion = null; + } + } + + flushCompletion.TrySetResult(); + EmitScheduled(scheduled); + } + } + + private bool TryRecoverFromUnexpectedIterationFailure(Exception exception) + { + try + { + ScheduledFlush? scheduled = null; + TaskCompletionSource? activeFlushCompletion; + lock (_lock) + { + if (DirtyCount > 0 && (!_stopping || _draining)) + { + _retryAttempt++; + var delay = _owner.GetRetryDelay(_retryAttempt); + _flushTimer.Change(delay); + scheduled = new(DisseminationBroadcastScheduleReason.Retry, delay, _retryAttempt, _notificationEpoch); + } + + activeFlushCompletion = _activeFlushCompletion; + _activeFlushCompletion = null; + } + + activeFlushCompletion?.TrySetResult(); + // Diagnostics follow state recovery, but a failing logger must still fault the pump explicitly. + LogWarningBroadcastPumpIterationFailed(_owner._logger, exception, Peer, scheduled?.DueTime); + EmitPumpFailure(DisseminationPumpFailureStatus.Recovered); + EmitScheduled(scheduled); + return true; + } + catch (Exception recoveryException) + { + var failure = new AggregateException( + $"The dissemination broadcast pump for {Peer} could not recover from an iteration failure.", + exception, + recoveryException); + FailPump(failure); + LogErrorBroadcastPumpFailed(_owner._logger, failure, Peer); + EmitPumpFailure(DisseminationPumpFailureStatus.Permanent); + return false; + } + } + + private void EmitPumpFailure(DisseminationPumpFailureStatus status) + { + try + { + DisseminationInstruments.OnPumpFailure(status); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + private void FailPump(Exception exception) + { + TaskCompletionSource nextFlushCompletion; + TaskCompletionSource? activeFlushCompletion; + lock (_lock) + { + _pumpFailure = exception; + nextFlushCompletion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + activeFlushCompletion = _activeFlushCompletion; + _activeFlushCompletion = null; + ClearPendingUnsafe(); + } + + activeFlushCompletion?.TrySetException(exception); + nextFlushCompletion.TrySetException(exception); + } + + private async ValueTask SendValues( + List initialWork, + CancellationToken cancellationToken) + { + // Materialize current broadcasts after admission using the peer's acknowledged baseline. + var pending = new Queue(initialWork); + var requiresBackoff = false; + var madeProgress = false; + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + // Dropping identities needs no transport capacity, including while shutdown drains disabled work. + var enabled = _owner._options.CurrentValue.Enabled; + while (pending.TryPeek(out var nextWork)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!IsWorkActive(nextWork)) + { + pending.Dequeue(); + } + else if (!enabled || !nextWork.Namespace.Options.Enabled) + { + pending.Dequeue(); + CompleteUnsupported(nextWork.Namespace.Name, initialWork); + } + else + { + break; + } + } + + if (pending.Count == 0) + { + break; + } + + var lease = await _owner._sendGate.AcquireAsync(Peer, _admissionQueued, cancellationToken); + try + { + cancellationToken.ThrowIfCancellationRequested(); + // Admission precedes serialization; waiting destinations retain only identities. + var currentOptions = _owner._options.CurrentValue; + var batch = new Dictionary>(); + var sentKeys = new List(); + var itemCount = 0; + var byteCount = 0; + + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var work = pending.Peek(); + if (!IsWorkActive(work)) + { + pending.Dequeue(); + continue; + } + + if (!currentOptions.Enabled || !work.Namespace.Options.Enabled) + { + pending.Dequeue(); + CompleteUnsupported(work.Namespace.Name, initialWork); + continue; + } + + var (knownVersion, baseline) = GetBroadcastBaseline(work); + var request = new DisseminationRepairRequest( + work.Key, + knownVersion, + currentOptions.MaxBatchBytes - byteCount, + work.Namespace.Options.MaxPayloadBytes); + var repair = work.Namespace.CreateBroadcast(request, baseline); + if (repair.Status is DisseminationRepairStatus.Current) + { + // The namespace confirms that the peer is already current, so no RPC is needed. + pending.Dequeue(); + CompleteCurrent(work, repair.Version); + continue; + } + + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity && itemCount > 0) + { + // Give this key a fresh budget in the next batch. + break; + } + + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + { + // A value which cannot fit in an empty batch waits for a future publication to change it. + pending.Dequeue(); + CompleteUnsendable(work); + continue; + } + + if (repair.Status is DisseminationRepairStatus.Unavailable + && !IsActiveKey(work.Namespace, work.Key)) + { + // A key absent from current digests was removed, not transiently unavailable. + pending.Dequeue(); + CompleteRemoved(work); + continue; + } + + if (repair.Status is not DisseminationRepairStatus.Produced + || !ValidateBroadcast(work.Namespace, request, repair)) + { + // Invalid or temporarily unavailable repairs retain the dirty key and enter backoff. + pending.Dequeue(); + Requeue([work]); + requiresBackoff = true; + continue; + } + + pending.Dequeue(); + ref var namespaceValues = ref CollectionsMarshal.GetValueRefOrAddDefault( + batch, + work.Namespace.Name, + out _); + namespaceValues ??= []; + var value = repair.Value; + namespaceValues.Add(new DisseminationBroadcastValue + { + Value = value, + TimeToLive = work.Namespace.Options.StaleItemTtl, + }); + itemCount++; + byteCount += value.Payload.Length; + + sentKeys.Add(new(work, knownVersion, value.ToVersion, repair.BroadcastState)); + if (itemCount >= currentOptions.MaxBatchItems || byteCount >= currentOptions.MaxBatchBytes) + { + break; + } + } + + if (itemCount == 0) + { + continue; + } + + var responseTask = SendBatch(batch, cancellationToken); + batch = null!; + var response = await responseTask; + if (response is null) + { + Requeue(sentKeys.Select(static sent => sent.Work)); + Requeue(pending); + requiresBackoff = true; + break; + } + + // RPC completion is not application evidence; only the returned receiver versions advance the ledger. + _owner._responseObserver?.Invoke(Peer, response); + var acknowledgments = response.AllVersionsAcknowledged ? null : CreateAcknowledgmentLookup(response.Acknowledgments); + var unsupportedNamespaces = response.UnsupportedNamespaces.Count > 0 + ? response.UnsupportedNamespaces.ToHashSet() + : null; + foreach (var sent in sentKeys) + { + if (unsupportedNamespaces?.Contains(sent.Work.Namespace.Name) == true) + { + CompleteUnsupported(sent.Work.Namespace.Name, initialWork); + continue; + } + + var acknowledgedVersion = sent.SentVersion; + if (acknowledgments is not null && !acknowledgments.TryGetValue( + new(sent.Work.Namespace.Name, sent.Work.Key), + out acknowledgedVersion)) + { + if (!CompleteFromExistingEvidence(sent)) + { + requiresBackoff = true; + } + + continue; + } + + var completion = CompleteAcknowledged(sent, acknowledgedVersion, response.AllVersionsAcknowledged); + madeProgress |= completion.MadeProgress; + requiresBackoff |= completion.RequiresBackoff; + } + } + finally + { + lease.Dispose(); + } + } + + return new(requiresBackoff, madeProgress); + } + + private async ValueTask SendBatch( + Dictionary> valuesByNamespace, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var transportLifetime = TimeSpan.MaxValue; + foreach (var values in valuesByNamespace.Values) + { + foreach (var value in values) + { + if (value.TimeToLive < transportLifetime) + { + transportLifetime = value.TimeToLive; + } + } + } + if (transportLifetime <= TimeSpan.Zero) + { + return null; + } + + if (transportLifetime > MaxTransportLifetime) + { + transportLifetime = MaxTransportLifetime; + } + + using var lifetimeCancellation = new CancellationTokenSource(transportLifetime, _owner._timeProvider); + using var sendCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lifetimeCancellation.Token); + try + { + var batch = new DisseminationBroadcastBatch + { + Sender = _owner._localSilo, + Values = valuesByNamespace, + SupportsCompactAcknowledgments = true, + }; + sendCancellation.Token.ThrowIfCancellationRequested(); + var sendTask = GetTarget().PushBroadcast(batch, sendCancellation.Token); + try + { + var response = await sendTask.WaitAsync(sendCancellation.Token); + try + { + DisseminationInstruments.OnBroadcastSent(batch.Values, "tree"); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + + return response; + } + catch (OperationCanceledException) when (sendCancellation.IsCancellationRequested) + { + ObserveLateSend(sendTask); + throw; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException) when (lifetimeCancellation.IsCancellationRequested) + { + EmitSendFailure(DisseminationFailureReason.Timeout); + LogDebugBroadcastTransportLifetimeExpired(_owner._logger, Peer, transportLifetime); + return null; + } + catch (Exception exception) + { + EmitSendFailure(DisseminationFailureReason.Error); + LogDebugDisseminationSendFailed(_owner._logger, exception, Peer); + return null; + } + } + + private void EmitSendFailure(DisseminationFailureReason reason) + { + try + { + DisseminationInstruments.OnBroadcastSendFailure(reason); + } + catch (Exception exception) + { + LogDebugBroadcastDiagnosticFailed(_owner._logger, exception, Peer); + } + } + + private void ObserveLateSend(Task sendTask) + { + // Fault observation follows transport completion after the operation's cancellation. + sendTask.ContinueWith( + task => LogDebugDisseminationSendFailed(_owner._logger, task.Exception!, Peer), + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default).Ignore(); + } + + private IDisseminationSystemTarget GetTarget() => + _target ??= _owner._grainFactory.GetSystemTarget( + Constants.DisseminationSystemTargetType, + Peer); + + private List DrainDirtyUnsafe() + { + // Snapshot dirty work into an in-flight generation; any later notification will set Dirty again. + // Membership precedes load when a peer has both kinds of work. + var result = new List(DirtyCount); + foreach (var namespaceState in _statesByNamespace.Values + .OrderBy(static state => state.Namespace.RoutingMode == DisseminationRoutingMode.AggregationTree)) + { + foreach (var (key, keyState) in namespaceState.Keys) + { + if (!keyState.Dirty) + { + continue; + } + + keyState.Dirty = false; + keyState.InFlight = true; + DirtyCount--; + result.Add(new( + namespaceState.Namespace, + key, + keyState.NotificationGeneration, + keyState.KnownVersion, + namespaceState)); + } + } + + return result; + } + + private (long? Version, DisseminationBroadcastState? State) GetBroadcastBaseline(PendingKeyWork work) + { + lock (_lock) + { + if (!TryGetKeyStateUnsafe(work, out _, out var keyState)) + { + return (work.KnownVersion, null); + } + + var state = keyState.BroadcastState; + return (keyState.KnownVersion, state?.Version == keyState.KnownVersion ? state : null); + } + } + + private bool IsWorkActive(PendingKeyWork work) + { + lock (_lock) + { + return TryGetKeyStateUnsafe(work, out _, out var keyState) && keyState.InFlight; + } + } + + private void CompleteCurrent(PendingKeyWork work, long version) + { + lock (_lock) + { + if (!TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + return; + } + + if (keyState.KnownVersion is not { } knownVersion || version > knownVersion) + { + keyState.KnownVersion = version; + } + + keyState.InFlight = false; + if (keyState.NotificationGeneration == work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(work.Key, keyState); + } + } + } + + private SendWorkResult CompleteAcknowledged(SentKey sent, long acknowledgedVersion, bool accepted) + { + lock (_lock) + { + if (!TryGetKeyStateUnsafe(sent.Work, out var namespaceState, out var keyState)) + { + return default; + } + + var previousVersion = keyState.KnownVersion; + if (previousVersion is null || acknowledgedVersion > previousVersion) + { + keyState.KnownVersion = acknowledgedVersion; + } + + if (accepted && keyState.KnownVersion == sent.SentVersion) + { + keyState.BroadcastState = sent.BroadcastState; + } + + var madeProgress = sent.FromVersion is null + ? acknowledgedVersion > 0 + : acknowledgedVersion > sent.FromVersion; + keyState.InFlight = false; + // Retire only the generation we sent; a newer notification remains queued even if this repair reached its target. + if (keyState.NotificationGeneration != sent.Work.NotificationGeneration + || keyState.Dirty) + { + return new(RequiresBackoff: false, madeProgress); + } + + if (keyState.KnownVersion >= sent.SentVersion) + { + namespaceState.RetireKey(sent.Work.Key, keyState); + return new(RequiresBackoff: false, madeProgress); + } + + MarkDirtyUnsafe(keyState); + return new(RequiresBackoff: true, madeProgress); + } + } + + private bool CompleteFromExistingEvidence(SentKey sent) + { + // A concurrent digest observation can satisfy a response which omitted this key's acknowledgment. + lock (_lock) + { + if (!TryGetKeyStateUnsafe(sent.Work, out var namespaceState, out var keyState)) + { + return true; + } + + if (keyState.KnownVersion >= sent.SentVersion) + { + keyState.InFlight = false; + if (keyState.NotificationGeneration == sent.Work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(sent.Work.Key, keyState); + } + + return true; + } + + keyState.InFlight = false; + MarkDirtyUnsafe(keyState); + return false; + } + } + + private void CompleteUnsupported(DisseminationNamespace namespaceName, List work) + { + // Capability evidence completes this flush's generations, not publications made during the send. + lock (_lock) + { + if (!_statesByNamespace.TryGetValue(namespaceName, out var namespaceState)) + { + return; + } + + var matchesState = false; + foreach (var item in work) + { + if (item.Namespace.Name != namespaceName + || !ReferenceEquals(item.NamespaceState, namespaceState)) + { + continue; + } + + matchesState = true; + if (!namespaceState.Keys.TryGetValue(item.Key, out var keyState)) + { + continue; + } + + keyState.InFlight = false; + if (keyState.NotificationGeneration != item.NotificationGeneration) + { + MarkDirtyUnsafe(keyState); + continue; + } + + if (keyState.Dirty) + { + DirtyCount--; + } + + namespaceState.Keys.Remove(item.Key); + } + + if (!matchesState) + { + return; + } + + namespaceState.KnownVersions.Clear(); + foreach (var keyState in namespaceState.Keys.Values) + { + keyState.KnownVersion = null; + keyState.BroadcastState = null; + } + + if (namespaceState.Keys.Count == 0) + { + _statesByNamespace.Remove(namespaceName); + } + + if (DirtyCount == 0) + { + var completion = _nextFlushCompletion; + _nextFlushCompletion = new(TaskCreationOptions.RunContinuationsAsynchronously); + completion.TrySetResult(); + } + } + } + + private void CompleteRemoved(PendingKeyWork work) + { + // Removed keys leave no retry state behind unless a newer notification raced with the repair. + lock (_lock) + { + if (!TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + return; + } + + if (keyState.NotificationGeneration != work.NotificationGeneration || keyState.Dirty) + { + keyState.InFlight = false; + return; + } + + namespaceState.Keys.Remove(work.Key); + if (namespaceState.Keys.Count == 0) + { + _statesByNamespace.Remove(work.Namespace.Name); + } + } + } + + private void CompleteUnsendable(PendingKeyWork work) + { + // Release admission capacity while preserving peer knowledge for a later publication. + lock (_lock) + { + if (TryGetKeyStateUnsafe(work, out var namespaceState, out var keyState)) + { + keyState.InFlight = false; + if (keyState.NotificationGeneration == work.NotificationGeneration && !keyState.Dirty) + { + namespaceState.RetireKey(work.Key, keyState); + } + } + } + } + + private void Requeue(IEnumerable work) + { + lock (_lock) + { + if (_stopping && !_draining) + { + foreach (var item in work) + { + if (TryGetKeyStateUnsafe(item, out _, out var keyState)) + { + keyState.InFlight = false; + } + } + + return; + } + + foreach (var item in work) + { + if (TryGetKeyStateUnsafe(item, out _, out var keyState)) + { + keyState.InFlight = false; + MarkDirtyUnsafe(keyState); + } + } + } + } + + private void ClearPendingUnsafe() + { + _statesByNamespace.Clear(); + DirtyCount = 0; + _retryAttempt = 0; + } + + private PeerNamespaceState GetOrCreateNamespaceStateUnsafe(IDisseminationNamespace disseminationNamespace) + { + if (!_statesByNamespace.TryGetValue(disseminationNamespace.Name, out var result)) + { + result = new(disseminationNamespace); + _statesByNamespace.Add(disseminationNamespace.Name, result); + } + + return result; + } + + private bool TryGetKeyStateUnsafe( + PendingKeyWork work, + out PeerNamespaceState namespaceState, + out PeerKeyState keyState) + { + if (_statesByNamespace.TryGetValue(work.Namespace.Name, out namespaceState!) + && ReferenceEquals(namespaceState, work.NamespaceState) + && namespaceState.Keys.TryGetValue(work.Key, out keyState!)) + { + return true; + } + + namespaceState = null!; + keyState = null!; + return false; + } + + private void MarkDirtyUnsafe(PeerKeyState keyState) + { + if (keyState.Dirty) + { + return; + } + + keyState.Dirty = true; + DirtyCount++; + } + + private static Dictionary CreateAcknowledgmentLookup( + Dictionary> acknowledgments) + { + var result = new Dictionary(); + foreach (var (namespaceName, entries) in acknowledgments) + { + foreach (var entry in entries) + { + var key = new DigestKey(namespaceName, entry.Key); + if (!result.TryGetValue(key, out var version) || entry.Version > version) + { + result[key] = entry.Version; + } + } + } + + return result; + } + + private static bool IsActiveKey( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key) => + disseminationNamespace.Digests.Any(entry => entry.Key == key); + + private static bool ValidateBroadcast( + IDisseminationNamespace disseminationNamespace, + in DisseminationRepairRequest request, + in DisseminationRepairResult repair) + { + var value = repair.Value; + return repair.Status is DisseminationRepairStatus.Produced + && repair.Version > 0 + && value.Key == request.Key + && (disseminationNamespace.BroadcastsAreDeltas + ? value.FromVersion > 0 && value.FromVersion <= value.ToVersion + : value.FromVersion == 0) + && value.ToVersion == repair.Version + && (repair.BroadcastState is null || repair.BroadcastState.Version == value.ToVersion) + && value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes; + } + + private sealed class PeerNamespaceState(IDisseminationNamespace disseminationNamespace) + { + public IDisseminationNamespace Namespace { get; } = disseminationNamespace; + + public Dictionary Keys { get; } = []; + + public Dictionary KnownVersions { get; } = []; + + public PeerKeyState AddKey(DisseminationKey key) + { + var result = new PeerKeyState(); + if (KnownVersions.TryGetValue(key, out var knownVersion)) + { + result.KnownVersion = knownVersion.Version; + result.BroadcastState = knownVersion.State; + } + + Keys.Add(key, result); + return result; + } + + public void ObserveKnownVersion( + DisseminationKey key, + long version, + DisseminationBroadcastState? state = null) + { + state = state?.Version == version ? state : null; + if (!KnownVersions.TryGetValue(key, out var knownVersion) || version > knownVersion.Version + || version == knownVersion.Version && state is not null) + { + KnownVersions[key] = new(version, state); + } + } + + public void RetireKey(DisseminationKey key, PeerKeyState keyState) + { + if (keyState.KnownVersion is { } knownVersion) + { + ObserveKnownVersion(key, knownVersion, keyState.BroadcastState); + } + + Keys.Remove(key); + } + + public void PruneKnownVersions(HashSet? activeKeys) + { + foreach (var key in KnownVersions.Keys) + { + if (activeKeys is null || !activeKeys.Contains(key)) + { + KnownVersions.Remove(key); + } + } + } + } + + private sealed class PeerKeyState + { + // A null version means no known baseline. Dirty can coexist with InFlight when a newer notification arrives mid-send. + public long? KnownVersion { get; set; } + + public DisseminationBroadcastState? BroadcastState { get; set; } + + public long NotificationGeneration { get; set; } + + public long NotificationVersion { get; set; } + + public bool Dirty { get; set; } + + public bool InFlight { get; set; } + } + + private readonly record struct PendingKeyWork( + IDisseminationNamespace Namespace, + DisseminationKey Key, + long NotificationGeneration, + long? KnownVersion, + PeerNamespaceState NamespaceState); + + private readonly record struct SentKey( + PendingKeyWork Work, + long? FromVersion, + long SentVersion, + DisseminationBroadcastState? BroadcastState); + + private readonly record struct PeerKnowledge(long Version, DisseminationBroadcastState? State); + + private readonly record struct DigestKey( + DisseminationNamespace Namespace, + DisseminationKey Key); + + private readonly record struct SendWorkResult(bool RequiresBackoff, bool MadeProgress); + } + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination send to {Peer} failed.")] + private static partial void LogDebugDisseminationSendFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast batch flush failed.")] + private static partial void LogDebugBroadcastFlushFailed( + ILogger logger, + Exception exception); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Dissemination broadcast pump for {Peer} encountered an unexpected iteration failure and will retry after {RetryDelay}.")] + private static partial void LogWarningBroadcastPumpIterationFailed( + ILogger logger, + Exception exception, + SiloAddress peer, + TimeSpan? retryDelay); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Dissemination broadcast pump for {Peer} failed permanently. Pending flush and drain waiters will fail explicitly.")] + private static partial void LogErrorBroadcastPumpFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast diagnostic for {Peer} failed.")] + private static partial void LogDebugBroadcastDiagnosticFailed( + ILogger logger, + Exception exception, + SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination broadcast transport to {Peer} exceeded its remaining hop lifetime of {Lifetime}.")] + private static partial void LogDebugBroadcastTransportLifetimeExpired( + ILogger logger, + SiloAddress peer, + TimeSpan lifetime); +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs new file mode 100644 index 00000000000..7907a6674a5 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationEvents.cs @@ -0,0 +1,186 @@ +using System; +using System.Diagnostics; + +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationEvents +{ + public const string ListenerName = "Microsoft.Orleans.Dissemination"; + public static readonly DiagnosticListener Listener = new(ListenerName); + + public static void EmitValue(DisseminationNamespace namespaceName, DisseminationValue value, SiloAddress localSilo, SiloAddress? peer, DisseminationApplyResult result, int payloadBytes) + { + if (Listener.IsEnabled("Dissemination.ValueApply")) + { + Listener.Write("Dissemination.ValueApply", new DisseminationValueEvent + { + Namespace = namespaceName, + LocalSilo = localSilo, + Peer = peer, + Key = value.Key, + FromVersion = value.FromVersion, + ToVersion = value.ToVersion, + Result = result.ToString(), + PayloadBytes = payloadBytes, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitPayloadDrop(DisseminationNamespace namespaceName, DisseminationValue value, SiloAddress localSilo, string reason, int payloadBytes) + { + if (Listener.IsEnabled("Dissemination.PayloadDrop")) + { + Listener.Write("Dissemination.PayloadDrop", new DisseminationValueEvent + { + Namespace = namespaceName, + LocalSilo = localSilo, + Key = value.Key, + FromVersion = value.FromVersion, + ToVersion = value.ToVersion, + Result = reason, + PayloadBytes = payloadBytes, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public const string BroadcastScheduledEventName = "Dissemination.BroadcastScheduled"; + public const string QueueAdmissionRejectedEventName = "Dissemination.QueueAdmissionRejected"; + public const string SendGateEventName = "Dissemination.SendGate"; + public const string NamespacePendingLimitReason = "namespace_pending_limit"; + + public static void EmitSendGate(SiloAddress localSilo, SiloAddress peer, string kind, string stage) + { + if (Listener.IsEnabled(SendGateEventName)) + { + Listener.Write(SendGateEventName, new DisseminationSendGateEvent + { + LocalSilo = localSilo, + Peer = peer, + Kind = kind, + Stage = stage, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitBroadcastScheduled( + SiloAddress localSilo, + SiloAddress peer, + DisseminationBroadcastScheduleReason reason, + TimeSpan dueTime, + int attempt, + long epoch) + { + if (Listener.IsEnabled(BroadcastScheduledEventName)) + { + Listener.Write(BroadcastScheduledEventName, new DisseminationBroadcastScheduledEvent + { + LocalSilo = localSilo, + Peer = peer, + Reason = reason, + DueTime = dueTime, + Attempt = attempt, + Epoch = epoch, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } + + public static void EmitQueueAdmissionRejected( + SiloAddress localSilo, + SiloAddress peer, + DisseminationNamespace namespaceName, + int limit) + { + if (Listener.IsEnabled(QueueAdmissionRejectedEventName)) + { + Listener.Write(QueueAdmissionRejectedEventName, new DisseminationQueueAdmissionRejectedEvent + { + LocalSilo = localSilo, + Peer = peer, + Namespace = namespaceName, + Limit = limit, + Reason = NamespacePendingLimitReason, + Timestamp = DateTimeOffset.UtcNow, + }); + } + } +} + +// Why a peer pump (re)armed its flush timer, exposed for deterministic tests and diagnostics. +internal enum DisseminationBroadcastScheduleReason +{ + // New work is ready for admission. + Immediate, + + // A prior send failed and the pump re-armed after backoff. + Retry, +} + +internal sealed class DisseminationBroadcastScheduledEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public DisseminationBroadcastScheduleReason Reason { get; init; } + + public TimeSpan DueTime { get; init; } + + public int Attempt { get; init; } + + public long Epoch { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationValueEvent +{ + public DisseminationNamespace Namespace { get; init; } + + public required SiloAddress LocalSilo { get; init; } + + public SiloAddress? Peer { get; init; } + + public DisseminationKey Key { get; init; } + + public long FromVersion { get; init; } + + public long ToVersion { get; init; } + + public string Result { get; init; } = string.Empty; + + public int PayloadBytes { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationQueueAdmissionRejectedEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public DisseminationNamespace Namespace { get; init; } + + public int Limit { get; init; } + + public string Reason { get; init; } = string.Empty; + + public DateTimeOffset Timestamp { get; init; } +} + +internal sealed class DisseminationSendGateEvent +{ + public required SiloAddress LocalSilo { get; init; } + + public required SiloAddress Peer { get; init; } + + public required string Kind { get; init; } + + public required string Stage { get; init; } + + public DateTimeOffset Timestamp { get; init; } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs new file mode 100644 index 00000000000..5d3beb640da --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationInstruments.cs @@ -0,0 +1,201 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationInstruments +{ + internal const string MeterName = "Microsoft.Orleans"; + internal const string BroadcastSendFailuresName = "orleans-dissemination-broadcast-send-failures"; + internal const string BroadcastScheduledName = "orleans-dissemination-broadcast-scheduled"; + internal const string AntiEntropyFailuresName = "orleans-dissemination-anti-entropy-failures"; + internal const string PumpFailuresName = "orleans-dissemination-pump-failures"; + internal const string PublicationsName = "orleans-dissemination-publications"; + internal const string QueueAdmissionRejectedName = "orleans-dissemination-queue-admission-rejected"; + internal const string ValuesReceivedName = "orleans-dissemination-values-received"; + + private static readonly Meter Meter = new(MeterName); + private static readonly Counter BroadcastSent = Meter.CreateCounter("orleans-dissemination-broadcast-sent", "messages"); + private static readonly Counter BroadcastReceived = Meter.CreateCounter("orleans-dissemination-broadcast-received", "messages"); + private static readonly Counter ValuesSent = Meter.CreateCounter("orleans-dissemination-values-sent", "values"); + private static readonly Counter ValuesReceived = Meter.CreateCounter(ValuesReceivedName, "values"); + private static readonly Counter ValuesApplied = Meter.CreateCounter("orleans-dissemination-values-applied", "values"); + private static readonly Counter BytesSent = Meter.CreateCounter("orleans-dissemination-bytes-sent", "bytes"); + private static readonly Counter BroadcastSendFailures = Meter.CreateCounter(BroadcastSendFailuresName, "attempts"); + private static readonly Counter BroadcastScheduled = Meter.CreateCounter(BroadcastScheduledName, "schedules"); + private static readonly Counter AntiEntropyExchanges = Meter.CreateCounter("orleans-dissemination-anti-entropy-exchanges", "operations"); + private static readonly Counter AntiEntropyDigests = Meter.CreateCounter("orleans-dissemination-anti-entropy-digests", "digests"); + private static readonly Counter AntiEntropyValues = Meter.CreateCounter("orleans-dissemination-anti-entropy-values", "values"); + private static readonly Counter AntiEntropyFailures = Meter.CreateCounter(AntiEntropyFailuresName, "operations"); + private static readonly Counter PumpFailures = Meter.CreateCounter(PumpFailuresName, "failures"); + private static readonly Counter Publications = Meter.CreateCounter(PublicationsName, "operations"); + private static readonly Counter PayloadDropped = Meter.CreateCounter("orleans-dissemination-payload-dropped", "values"); + private static readonly Counter QueueAdmissionRejected = Meter.CreateCounter( + QueueAdmissionRejectedName, + "keys"); + + public static void OnBroadcastSent(DisseminationNamespace namespaceName, string kind, int itemCount, int byteCount) + { + if (!BroadcastSent.Enabled && !ValuesSent.Enabled && !BytesSent.Enabled) + { + return; + } + + var namespaceTag = Tag("namespace", namespaceName); + var kindTag = Tag("kind", kind); + BroadcastSent.Add(1, namespaceTag, kindTag); + ValuesSent.Add(itemCount, namespaceTag, kindTag); + BytesSent.Add(byteCount, namespaceTag, kindTag); + } + + public static void OnBroadcastSent(Dictionary> valuesByNamespace, string kind) + { + if (!BroadcastSent.Enabled && !ValuesSent.Enabled && !BytesSent.Enabled) + { + return; + } + + foreach (var (namespaceName, values) in valuesByNamespace) + { + OnBroadcastSent(namespaceName, kind, values.Count, values.Sum(static item => item.Value.Payload.Length)); + } + } + + public static void OnBroadcastReceived(DisseminationNamespace namespaceName, string kind, int itemCount) + { + if (!BroadcastReceived.Enabled && !ValuesReceived.Enabled) + { + return; + } + + var namespaceTag = Tag("namespace", namespaceName); + var kindTag = Tag("kind", kind); + BroadcastReceived.Add(1, namespaceTag, kindTag); + ValuesReceived.Add(itemCount, namespaceTag, kindTag); + } + + public static void OnBroadcastSendFailure(DisseminationFailureReason reason) + { + if (BroadcastSendFailures.Enabled) + { + BroadcastSendFailures.Add(1, Tag("reason", GetFailureReason(reason))); + } + } + + public static void OnBroadcastScheduled(DisseminationBroadcastScheduleReason reason) + { + if (BroadcastScheduled.Enabled) + { + BroadcastScheduled.Add(1, Tag("reason", GetScheduleReason(reason))); + } + } + + public static void OnValueApplied(DisseminationNamespace namespaceName, DisseminationApplyResult result) + { + if (!ValuesApplied.Enabled) + { + return; + } + + ValuesApplied.Add(1, Tag("namespace", namespaceName), Tag("result", GetApplyResult(result))); + } + + public static void OnAntiEntropyExchange(string direction, int digestCount, int itemCount, bool truncated) + { + if (!AntiEntropyExchanges.Enabled && !AntiEntropyDigests.Enabled && !AntiEntropyValues.Enabled) + { + return; + } + + var directionTag = Tag("direction", direction); + if (AntiEntropyExchanges.Enabled) + { + AntiEntropyExchanges.Add(1, directionTag, Tag("truncated", truncated)); + } + + AntiEntropyDigests.Add(digestCount, directionTag); + AntiEntropyValues.Add(itemCount, directionTag); + } + + public static void OnAntiEntropyFailure(DisseminationFailureReason reason, int count = 1) + { + if (AntiEntropyFailures.Enabled && count > 0) + { + AntiEntropyFailures.Add(count, Tag("reason", GetFailureReason(reason))); + } + } + + public static void OnPumpFailure(DisseminationPumpFailureStatus status) + { + if (PumpFailures.Enabled) + { + PumpFailures.Add( + 1, + Tag("status", status == DisseminationPumpFailureStatus.Recovered ? "recovered" : "permanent")); + } + } + + public static void OnPublication(DisseminationNamespace namespaceName, bool accepted, string reason) + { + if (Publications.Enabled) + { + Publications.Add( + 1, + Tag("namespace", namespaceName), + Tag("result", accepted ? "accepted" : "rejected"), + Tag("reason", reason)); + } + } + + public static void OnPayloadDropped(DisseminationNamespace namespaceName, string reason) + { + if (PayloadDropped.Enabled) + { + PayloadDropped.Add(1, Tag("namespace", namespaceName), Tag("reason", reason)); + } + } + + public static void OnQueueAdmissionRejected(DisseminationNamespace namespaceName) + { + if (QueueAdmissionRejected.Enabled) + { + QueueAdmissionRejected.Add( + 1, + Tag("namespace", namespaceName), + Tag("reason", DisseminationEvents.NamespacePendingLimitReason)); + } + } + + private static string GetFailureReason(DisseminationFailureReason reason) => + reason == DisseminationFailureReason.Timeout ? "timeout" : "error"; + + private static string GetApplyResult(DisseminationApplyResult result) => result switch + { + DisseminationApplyResult.Applied => "applied", + DisseminationApplyResult.Duplicate => "duplicate", + DisseminationApplyResult.Obsolete => "obsolete", + DisseminationApplyResult.Rejected => "rejected", + _ => "unknown", + }; + + private static string GetScheduleReason(DisseminationBroadcastScheduleReason reason) => reason switch + { + DisseminationBroadcastScheduleReason.Immediate => "immediate", + DisseminationBroadcastScheduleReason.Retry => "retry", + _ => "unknown", + }; + + private static KeyValuePair Tag(string name, object value) => new(name, value); +} + +internal enum DisseminationFailureReason +{ + Timeout, + Error, +} + +internal enum DisseminationPumpFailureStatus +{ + Recovered, + Permanent, +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs new file mode 100644 index 00000000000..0ab930bd2d0 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembership.cs @@ -0,0 +1,167 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationMembership( + IMembershipManager membershipManager, + ILocalSiloDetails localSiloDetails, + IOptions options) +{ + private readonly object _membershipLock = new(); + private CachedMembership? _current; + + public DisseminationMembershipSnapshots CurrentSnapshots + { + get + { + var membershipSnapshot = membershipManager.CurrentSnapshot; + var current = Volatile.Read(ref _current); + if (current is not null && ReferenceEquals(current.Source, membershipSnapshot)) + { + return current.Snapshots; + } + + lock (_membershipLock) + { + current = Volatile.Read(ref _current); + if (current is not null) + { + // Re-read the owner after acquiring the lock instead of imposing a second version policy. + membershipSnapshot = membershipManager.CurrentSnapshot; + if (ReferenceEquals(current.Source, membershipSnapshot)) + { + return current.Snapshots; + } + } + + // Same-version heartbeats and removal of non-participants retain the existing topology. + var snapshots = current is { } previous && HasSameTopology(previous, membershipSnapshot) + ? previous.Snapshots + : ComputeMembership(membershipSnapshot, localSiloDetails.SiloAddress, options.Value.Overlay, current?.Snapshots); + Volatile.Write(ref _current, new(membershipSnapshot, snapshots)); + return snapshots; + } + } + } + + public DisseminationMembershipSnapshot CurrentSnapshot => CurrentSnapshots.AllMembers; + + public DisseminationMembershipSnapshot GetSnapshot(DisseminationMembershipScope scope) => + CurrentSnapshots.GetSnapshot(scope); + + public Task RefreshMembership(CancellationToken cancellationToken) => + membershipManager.Refresh(targetVersion: null, cancellationToken: cancellationToken); + + public async ValueTask GetSnapshotsContainingMember( + SiloAddress member, + DisseminationMembershipScope scope, + CancellationToken cancellationToken) + { + var snapshots = CurrentSnapshots; + if (snapshots.GetSnapshot(scope).ContainsMember(member)) + { + return snapshots; + } + + await RefreshMembership(cancellationToken); + snapshots = CurrentSnapshots; + return snapshots.GetSnapshot(scope).ContainsMember(member) ? snapshots : null; + } + + private static bool HasSameTopology(CachedMembership current, MembershipTableSnapshot source) + { + if (current.Source.Version != source.Version) + { + return false; + } + + foreach (var member in current.Snapshots.AllMembers.Members) + { + if (!source.Entries.TryGetValue(member, out var entry) + || entry.Status != current.Source.Entries[member].Status) + { + return false; + } + } + + foreach (var (member, entry) in source.Entries) + { + if (IsDisseminationMember(entry.Status) && !current.Snapshots.AllMembers.ContainsMember(member)) + { + return false; + } + } + + return true; + } + + private static DisseminationMembershipSnapshots ComputeMembership( + MembershipTableSnapshot snapshot, + SiloAddress localSilo, + DisseminationOverlayOptions overlayOptions, + DisseminationMembershipSnapshots? previous) + { + var members = snapshot.Entries.Values + .Where(static entry => IsDisseminationMember(entry.Status)) + .OrderBy(static entry => GetStatusRank(entry.Status)) + // Storage providers can round StartTime differently from the originating silo's local snapshot. + // SiloAddress includes the generation and provides a stable oldest-first order everywhere. + .ThenBy(static entry => entry.SiloAddress) + .ToArray(); + var allMembers = ImmutableArray.CreateBuilder(members.Length); + var activeMembers = ImmutableArray.CreateBuilder(members.Length); + foreach (var member in members) + { + allMembers.Add(member.SiloAddress); + if (member.Status == SiloStatus.Active) + { + activeMembers.Add(member.SiloAddress); + } + } + + return new( + new DisseminationMembershipSnapshot( + snapshot.Version, + localSilo, + activeMembers.ToImmutable(), + overlayOptions, + previous?.ActiveMembers), + new DisseminationMembershipSnapshot( + snapshot.Version, + localSilo, + allMembers.MoveToImmutable(), + overlayOptions, + previous?.AllMembers)); + } + + private static bool IsDisseminationMember(SiloStatus status) => + status is SiloStatus.Joining or SiloStatus.Active or SiloStatus.ShuttingDown or SiloStatus.Stopping; + + private static int GetStatusRank(SiloStatus status) => status switch + { + SiloStatus.Active => 0, + SiloStatus.Joining => 1, + SiloStatus.ShuttingDown => 2, + SiloStatus.Stopping => 3, + _ => 4, + }; + + private sealed record CachedMembership(MembershipTableSnapshot Source, DisseminationMembershipSnapshots Snapshots); +} + +internal sealed class DisseminationMembershipSnapshots( + DisseminationMembershipSnapshot activeMembers, + DisseminationMembershipSnapshot allMembers) +{ + public MembershipVersion MembershipVersion => AllMembers.MembershipVersion; + + public DisseminationMembershipSnapshot ActiveMembers { get; } = activeMembers; + + public DisseminationMembershipSnapshot AllMembers { get; } = allMembers; + + public DisseminationMembershipSnapshot GetSnapshot(DisseminationMembershipScope scope) => + scope == DisseminationMembershipScope.ActiveMembers ? ActiveMembers : AllMembers; +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs new file mode 100644 index 00000000000..7b80a93a592 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationMembershipSnapshot.cs @@ -0,0 +1,173 @@ +using System.Collections.Frozen; +using System.Collections.Immutable; +using System.Runtime.InteropServices; +using Orleans.Configuration; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationMembershipSnapshot +{ + private readonly FrozenSet _set; + private readonly SiloAddress[] _antiEntropyPeers; + private readonly AntiEntropyPeerSelection _antiEntropySelection; + private readonly SiloAddress _localSilo; + private readonly ImmutableArray _aggregationOriginatorTargets; + + public DisseminationMembershipSnapshot( + MembershipVersion membershipVersion, + SiloAddress localSilo, + ImmutableArray members, + DisseminationOverlayOptions overlayOptions, + DisseminationMembershipSnapshot? previous = null) + { + MembershipVersion = membershipVersion; + _localSilo = localSilo; + Members = members.IsDefault ? [] : members; + // Membership versions also advance without changing the eligible peers. Preserve progress so that + // frequent updates cannot keep repair rounds confined to the first few members. + _antiEntropySelection = previous?._antiEntropySelection ?? new(); + var fanout = Members.Length <= 1 ? 1 : overlayOptions.GetFanOutFactor(Members.Length); + var aggregationFanout = Math.Clamp(overlayOptions.AggregationFanOutFactor, 1, Math.Max(1, Members.Length)); + var memberSet = new HashSet(Members.Length); + var localIndex = -1; + for (var i = 0; i < Members.Length; i++) + { + var member = Members[i]; + if (!memberSet.Add(member)) + { + throw new ArgumentException("Membership snapshot members must be unique.", nameof(members)); + } + + if (Equals(member, localSilo)) + { + localIndex = i; + } + } + + _set = memberSet.ToFrozenSet(); + ForwardingTreeTargets = localIndex < 0 ? [] : ComputeChildren((long)fanout * (localIndex + 1), fanout); + AggregationChildren = localIndex < 0 ? [] : ComputeChildren((long)aggregationFanout * localIndex + 1, aggregationFanout); + + if (localIndex < 0) + { + _antiEntropyPeers = []; + } + else + { + _antiEntropyPeers = new SiloAddress[Members.Length - 1]; + var candidateIndex = 0; + for (var i = 0; i < Members.Length; i++) + { + if (i != localIndex) + { + _antiEntropyPeers[candidateIndex++] = Members[i]; + } + } + } + + OriginatorTreeTargets = ComputeOriginatorTreeTargets(localSilo, localIndex, fanout); + IsAggregationRoot = localIndex == 0; + if (localIndex < 0) + { + _aggregationOriginatorTargets = []; + } + else + { + _aggregationOriginatorTargets = IsAggregationRoot ? AggregationChildren : [Members[0]]; + } + } + + public MembershipVersion MembershipVersion { get; } + + public ImmutableArray Members { get; } + + public ImmutableArray OriginatorTreeTargets { get; } + + public ImmutableArray ForwardingTreeTargets { get; } + + public ImmutableArray AggregationChildren { get; } + + public bool IsAggregationRoot { get; } + + public ImmutableArray GetOriginatorTargets(DisseminationRoutingMode mode) => + mode == DisseminationRoutingMode.AggregationTree ? _aggregationOriginatorTargets : OriginatorTreeTargets; + + public ImmutableArray GetForwardingTargets(DisseminationRoutingMode mode) => + mode == DisseminationRoutingMode.AggregationTree ? AggregationChildren : ForwardingTreeTargets; + + // Active members are address-ordered: ingest moves toward a smaller root, distribution toward larger children. + public ImmutableArray GetForwardingTargets(DisseminationRoutingMode mode, SiloAddress sender) => + mode == DisseminationRoutingMode.AggregationTree && !IsAggregationRoot && sender.CompareTo(_localSilo) > 0 + ? _aggregationOriginatorTargets + : GetForwardingTargets(mode); + + public bool ContainsMember(SiloAddress silo) => _set.Contains(silo); + + public ImmutableArray SelectAntiEntropyPeers(int peerCount) + { + ArgumentOutOfRangeException.ThrowIfNegative(peerCount); + var candidates = _antiEntropyPeers; + var count = Math.Min(peerCount, candidates.Length); + if (count <= 0) + { + return []; + } + + var peers = new SiloAddress[count]; + lock (_antiEntropySelection) + { + for (var i = 0; i < count; i++) + { + peers[i] = candidates[(_antiEntropySelection.Cursor + i) % candidates.Length]; + } + + _antiEntropySelection.Cursor = (_antiEntropySelection.Cursor + count) % candidates.Length; + } + + return ImmutableCollectionsMarshal.AsImmutableArray(peers); + } + + private ImmutableArray ComputeOriginatorTreeTargets(SiloAddress localSilo, int localIndex, int fanout) + { + if (localIndex < 0) + { + return []; + } + + var result = ImmutableArray.CreateBuilder(Math.Min(fanout * 2, Members.Length)); + var count = Math.Min(fanout, Members.Length); + for (var i = 0; i < count; i++) + { + var member = Members[i]; + if (!Equals(member, localSilo)) + { + result.Add(member); + } + } + + result.AddRange(ForwardingTreeTargets); + return result.ToImmutable(); + } + + private ImmutableArray ComputeChildren(long firstChild, int fanout) + { + if (firstChild >= Members.Length) + { + return []; + } + + var count = (int)Math.Min(fanout, Members.Length - firstChild); + var result = ImmutableArray.CreateBuilder(count); + for (var child = firstChild; child < firstChild + count; child++) + { + result.Add(Members[(int)child]); + } + + return result.MoveToImmutable(); + } + + private sealed class AntiEntropyPeerSelection + { + public int Cursor; + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs b/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs new file mode 100644 index 00000000000..f2d8279b717 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationNamespaceNames.cs @@ -0,0 +1,7 @@ +namespace Orleans.Runtime.Dissemination; + +internal static class DisseminationNamespaceNames +{ + public static readonly DisseminationNamespace DeploymentLoad = new("load"); + public static readonly DisseminationNamespace Membership = new("membership"); +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs new file mode 100644 index 00000000000..55f585ae847 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.Cohorts.cs @@ -0,0 +1,165 @@ +using Orleans.Internal; + +namespace Orleans.Runtime.Dissemination; + +internal sealed partial class DisseminationProtocol +{ + private readonly DisseminationSendGate _publicationSendGate = new(1); + private readonly CancellationTokenSource _publicationShutdown = new(); + + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + return Reject("stopping"); + } + + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _publicationShutdown.Token); + try + { + // Held load receipts have independent admission, keeping membership's peer pumps available. + using var lease = await _publicationSendGate.AcquireAsync(_localSilo, cancellation.Token); + var options = _options.CurrentValue; + if (!options.Enabled || !disseminationNamespace.Options.Enabled) + { + return Reject("disabled"); + } + + if (disseminationNamespace.RoutingMode != DisseminationRoutingMode.AggregationTree + || !Equals(key.Value, _localSilo) + || version <= 0 + || disseminationNamespace.GetVersion(key) < version) + { + return Reject("invalid-contribution"); + } + + var membership = await GetMembershipSnapshotForRouting( + disseminationNamespace.MembershipScope, _localSilo, cancellation.Token); + if (membership is null) + { + return Reject("membership-unavailable"); + } + + var repairRequest = new DisseminationRepairRequest( + key, fromVersion: null, + options.MaxBatchBytes, disseminationNamespace.Options.MaxPayloadBytes); + var repair = disseminationNamespace.CreateRepair(repairRequest); + if (repair.Status != DisseminationRepairStatus.Produced + || repair.Version < version + || !ValidateRepair(disseminationNamespace, repairRequest, repair, options)) + { + return Reject("invalid-repair"); + } + + var value = repair.Value; + RecordValueUpdate(disseminationNamespace.Name, key, value.ToVersion); + DisseminationPublicationReceipt receipt; + if (membership.IsAggregationRoot) + { + var root = GetRootBatcher(disseminationNamespace); + if (root is null) + { + return Reject("stopping"); + } + + receipt = await root.PublishAsync(new(key, value.ToVersion, true), cancellation.Token); + } + else + { + var rootAddress = membership.Members[0]; + var target = _grainFactory.GetSystemTarget( + Constants.DisseminationSystemTargetType, rootAddress); + var request = new DisseminationPublicationRequest + { + Sender = _localSilo, + Namespace = disseminationNamespace.Name, + Value = new() { Value = value, TimeToLive = disseminationNamespace.Options.StaleItemTtl }, + }; + var response = target.PublishAggregated(request, cancellation.Token); + response.Ignore(); + receipt = await response.WaitAsync(cancellation.Token); + if (receipt.Accepted) + { + ConfirmPeerNamespaces(rootAddress, [disseminationNamespace.Name]); + } + } + + EmitPublication(disseminationNamespace.Name, receipt.Accepted, + receipt.Accepted ? "none" : "cohort-rejected"); + return receipt; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + + DisseminationPublicationReceipt Reject(string reason) + { + EmitPublication(disseminationNamespace.Name, accepted: false, reason); + return default; + } + } + + public async Task ReceivePublication( + DisseminationPublicationRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + var receivedTimestamp = _timeProvider.GetTimestamp(); + var options = _options.CurrentValue; + if (!options.Enabled || !TryGetEnabledNamespace(request.Namespace, out var ns)) + { + return Reject("disabled"); + } + + if (ns.RoutingMode != DisseminationRoutingMode.AggregationTree + || !Equals(request.Value.Value.Key.Value, request.Sender) + || request.Value.Value.FromVersion != 0 + || options.MaxBatchItems < 1 + || !ValidatePayloadSize(ns, request.Value.Value, options.MaxBatchBytes)) + { + return Reject("invalid-contribution"); + } + + var membership = await GetMembershipSnapshotForRouting(ns.MembershipScope, request.Sender, cancellationToken); + if (membership is null || !membership.IsAggregationRoot) + { + return Reject("root-changed"); + } + + var result = await ApplyReceivedValue( + ns, request.Value, request.Sender, options, receivedTimestamp, allowDelta: false, cancellationToken); + if (result is not (DisseminationApplyResult.Applied or DisseminationApplyResult.Duplicate)) + { + return Reject("application-rejected"); + } + + ConfirmPeerNamespaces(request.Sender, [ns.Name]); + var root = GetRootBatcher(ns); + if (root is null) + { + return Reject("stopping"); + } + + var receipt = await root.PublishAsync( + new(request.Value.Value.Key, request.Value.Value.ToVersion, result == DisseminationApplyResult.Applied), + cancellationToken); + EmitPublication(ns.Name, receipt.Accepted, receipt.Accepted ? "cohort-sealed" : "cohort-rejected"); + return receipt; + + DisseminationPublicationReceipt Reject(string reason) + { + EmitPublication(request.Namespace, accepted: false, reason); + return default; + } + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs new file mode 100644 index 00000000000..bf5cac0e1df --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationProtocol.cs @@ -0,0 +1,1753 @@ +using System.Collections.Frozen; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Internal; + +namespace Orleans.Runtime.Dissemination; + +// The protocol coordinates routing and application while namespaces remain authoritative for values and repair history. +internal sealed partial class DisseminationProtocol +{ + private const int MaxRetainedNonMemberResponseCursors = 64; + private static readonly TimeSpan MaxAntiEntropyRoundLifetime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + private static readonly TimeSpan InventoryMaintenanceInterval = TimeSpan.FromSeconds(1); + private readonly SiloAddress _localSilo; + private readonly IInternalGrainFactory _grainFactory; + private readonly DisseminationMembership _membership; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly DisseminationBroadcastQueue _broadcastQueue; + private readonly object _rootBatcherLock = new(); + private readonly Dictionary _rootBatchers = []; + private bool _rootBatchersStopped; + private readonly object _maintenanceLock = new(); + private MaintenanceStamp? _lastMaintenance; + private readonly AdmissionGate _admission = new(); + private readonly DisseminationSendGate _antiEntropySendGate; + private readonly CancellationTokenSource _antiEntropyShutdown = new(); + private readonly object _antiEntropyResponseCursorLock = new(); + private readonly Dictionary _antiEntropyResponseCursors = []; + private long _antiEntropyResponseCursorAccess; + private readonly object _receivedBatchCursorLock = new(); + private readonly Dictionary<(SiloAddress Peer, bool AntiEntropy), ReceivedBatchCursor> _receivedBatchCursors = []; + private long _receivedBatchCursorAccess; + private readonly object _valueUpdateLock = new(); + private readonly Dictionary _lastValueUpdates = []; + private readonly object _peerSupportLock = new(); + private readonly Dictionary> _confirmedPeerNamespaces = []; + private DisseminationMembershipSnapshot? _confirmedMembership; + private readonly FrozenDictionary _namespaces; + + public DisseminationProtocol( + ILocalSiloDetails localSiloDetails, + IInternalGrainFactory grainFactory, + DisseminationMembership membership, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + TimeProvider timeProvider, + ILogger logger, + ILogger broadcastQueueLogger) + { + _localSilo = localSiloDetails.SiloAddress; + _grainFactory = grainFactory; + _membership = membership; + _options = options; + _timeProvider = timeProvider; + _logger = logger; + _namespaces = disseminationNamespaces.ToFrozenDictionary(static ns => ns.Name); + foreach (var ns in _namespaces.Values) + { + if (ns.RoutingMode == DisseminationRoutingMode.AggregationTree + && ns.MembershipScope != DisseminationMembershipScope.ActiveMembers) + { + throw new ArgumentException("Root aggregation requires an ActiveMembers membership scope.", nameof(disseminationNamespaces)); + } + } + + _antiEntropySendGate = new(Math.Max(1, options.CurrentValue.Overlay.AntiEntropyPeerCount)); + + _broadcastQueue = new DisseminationBroadcastQueue( + _timeProvider, + _localSilo, + _grainFactory, + _options, + _namespaces.Values, + broadcastQueueLogger, + ObserveBroadcastResponse); + } + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + EmitPublication(disseminationNamespace.Name, accepted: false, reason: "stopping"); + return false; + } + + var options = _options.CurrentValue; + if (!options.Enabled || !disseminationNamespace.Options.Enabled) + { + EmitPublication( + disseminationNamespace.Name, + accepted: false, + reason: "disabled"); + return false; + } + + // Before replacing the legacy fallback, prove that an unknown peer can receive a complete, bounded repair. + if (!TryValidatePublish( + disseminationNamespace, + key, + version, + options, + out var publishedVersion, + out var reason)) + { + EmitPublication( + disseminationNamespace.Name, + accepted: false, + reason: reason); + return false; + } + + var membership = await GetMembershipSnapshotForRouting( + disseminationNamespace.MembershipScope, + _localSilo, + cancellationToken); + if (membership is null) + { + EmitPublication( + disseminationNamespace.Name, + accepted: false, + "membership-unavailable"); + return false; + } + + // Notifications carry identity only; each peer pump asks the namespace for the latest repair at send time. + RecordValueUpdate(disseminationNamespace.Name, key, publishedVersion); + var accepted = true; + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && membership.IsAggregationRoot) + { + accepted = NotifyRoot(disseminationNamespace, [new(key, publishedVersion, true)]); + } + else + { + foreach (var peer in membership.GetOriginatorTargets(disseminationNamespace.RoutingMode)) + { + accepted &= _broadcastQueue.Notify(peer, disseminationNamespace, key); + } + } + + EmitPublication( + disseminationNamespace.Name, + accepted, + reason: accepted ? "none" : "queue-rejected"); + return accepted; + } + + public async Task ReceiveBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + + var receivedTimestamp = _timeProvider.GetTimestamp(); + var options = _options.CurrentValue; + if (!options.Enabled) + { + return new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [.. batch.Values.Keys], + }; + } + + var receivedKeys = new Dictionary>(); + var unsupportedNamespaces = new List(); + foreach (var namespaceName in batch.Values.Keys) + { + if (!TryGetEnabledNamespace(namespaceName, out _)) + { + unsupportedNamespaces.Add(namespaceName); + } + } + + var selectedValues = SelectReceivedValues(batch.Sender, batch.Values, options, antiEntropy: false, out var isUnfiltered); + foreach (var (namespaceName, values) in selectedValues) + { + var disseminationNamespace = _namespaces[namespaceName]; + try + { + DisseminationInstruments.OnBroadcastReceived(disseminationNamespace.Name, "tree", values.Count); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, batch.Sender, "broadcast-receive"); + } + + ConfirmPeerNamespaces(batch.Sender, [namespaceName]); + var namespaceKeys = new Dictionary(); + receivedKeys.Add(disseminationNamespace, namespaceKeys); + foreach (var item in values) + { + cancellationToken.ThrowIfCancellationRequested(); + var existing = namespaceKeys.TryGetValue(item.Value.Key, out var keyState); + var sentVersion = existing ? Math.Max(keyState.SentVersion, item.Value.ToVersion) : item.Value.ToVersion; + // The sender necessarily owns this version; use that fact only if an outbound ledger already exists. + _broadcastQueue.ObservePeerVersion( + batch.Sender, + namespaceName, + item.Value.Key, + item.Value.ToVersion); + var result = await ApplyReceivedValue( + disseminationNamespace, + item, + batch.Sender, + options, + receivedTimestamp, + allowDelta: disseminationNamespace.BroadcastsAreDeltas, + cancellationToken); + var accepted = result is DisseminationApplyResult.Applied or DisseminationApplyResult.Duplicate; + namespaceKeys[item.Value.Key] = new( + sentVersion, + keyState.Applied || result is DisseminationApplyResult.Applied, + accepted && (!existing || keyState.Accepted)); + if (disseminationNamespace.BroadcastsAreDeltas + && item.Value.FromVersion > 0 + && result == DisseminationApplyResult.Rejected) + { + // A missing broadcast baseline should participate in the next repair round. + lock (_valueUpdateLock) + { + _lastValueUpdates.Remove(new(namespaceName, item.Value.Key)); + } + } + } + } + + // Membership may be part of this batch, so apply everything before deriving the forwarding tree. + await MaintainInventories(cancellationToken); + var membershipSnapshots = _membership.CurrentSnapshots; + // Changed state always wakes children, including same-version liveness updates. Duplicate deliveries + // wake only children which still need this version and have no equivalent queued work. + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + var membership = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope); + var notifications = new DisseminationBroadcastQueue.KeyNotification[keys.Count]; + var notificationCount = 0; + foreach (var (key, state) in keys) + { + var version = disseminationNamespace.GetVersion(key); + if (version > 0) + { + notifications[notificationCount++] = new(key, version, state.Applied); + } + } + + if (disseminationNamespace.RoutingMode == DisseminationRoutingMode.AggregationTree && membership.IsAggregationRoot) + { + NotifyRoot(disseminationNamespace, notifications.AsSpan(0, notificationCount)); + continue; + } + + foreach (var peer in membership.GetForwardingTargets(disseminationNamespace.RoutingMode, batch.Sender)) + { + if (!Equals(peer, batch.Sender)) + { + _broadcastQueue.NotifyBatch(peer, disseminationNamespace, notifications.AsSpan(0, notificationCount)); + } + } + } + + // Once downstream work is queued, report the versions this receiver actually holds. + var compact = batch.SupportsCompactAcknowledgments + && isUnfiltered + && CanAcknowledgeTransmittedVersions(receivedKeys); + var acknowledgments = new Dictionary>(receivedKeys.Count); + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + var namespaceAcknowledgments = new List(compact ? 0 : keys.Count); + if (!compact) + { + foreach (var key in keys.Keys) + { + namespaceAcknowledgments.Add(new DigestEntry(key, disseminationNamespace.GetVersion(key))); + } + } + + acknowledgments.Add(disseminationNamespace.Name, namespaceAcknowledgments); + } + + return new DisseminationBroadcastResponse + { + Acknowledgments = acknowledgments, + UnsupportedNamespaces = unsupportedNamespaces, + AllVersionsAcknowledged = compact, + }; + } + + private static bool CanAcknowledgeTransmittedVersions( + Dictionary> receivedKeys) + { + foreach (var (disseminationNamespace, keys) in receivedKeys) + { + foreach (var (key, state) in keys) + { + // A different version needs an explicit acknowledgment to preserve the peer's exact repair baseline. + if (!state.Accepted || disseminationNamespace.GetVersion(key) != state.SentVersion) + { + return false; + } + } + } + + return true; + } + + private readonly record struct ReceivedKeyState(long SentVersion, bool Applied, bool Accepted); + + public async Task RunAntiEntropyRound(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + if (!admission.Entered) + { + return; + } + + var options = _options.CurrentValue; + if (!options.Enabled) + { + return; + } + + await MaintainInventories(cancellationToken); + var membershipSnapshots = _membership.CurrentSnapshots; + + // A round never queues behind prior rounds: busy destinations and slots wait for a future rotation. + var leases = AcquireAntiEntropyPeers(membershipSnapshots, options.Overlay.AntiEntropyPeerCount); + try + { + if (leases.Count == 0) + { + return; + } + + using var roundCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + _antiEntropyShutdown.Token); + var roundCancellationToken = roundCancellation.Token; + roundCancellationToken.ThrowIfCancellationRequested(); + // Push traffic suppresses redundant checks; admitted peers share this round's digest snapshot. + var requestDigests = CreateAntiEntropyRequestDigests(_timeProvider.GetTimestamp()); + var requests = CreateAntiEntropyRequests(membershipSnapshots, requestDigests, leases.Keys, options); + if (requests.Count == 0) + { + return; + } + + var roundLifetime = GetAntiEntropyRoundLifetime(requests.Values); + using var lifetimeCancellation = new CancellationTokenSource(roundLifetime, _timeProvider); + using var exchangeCancellation = CancellationTokenSource.CreateLinkedTokenSource( + roundCancellationToken, + lifetimeCancellation.Token); + var responseTasks = new List>(requests.Count); + DisseminationAntiEntropyResponse?[] responses; + try + { + foreach (var (peer, request) in requests) + { + var responseTask = ExchangeAntiEntropyRequest( + peer, + request, + leases[peer], + GetDigestCount(request.Digests), + exchangeCancellation.Token, + roundCancellationToken, + lifetimeCancellation.Token); + leases.Remove(peer); + responseTasks.Add(responseTask); + } + + // Each exchange bounds its local wait, so all leases are released before the round completes. + responses = await Task.WhenAll(responseTasks); + } + finally + { + // The local wait can win cancellation before the linked transport source is signaled. + await exchangeCancellation.CancelAsync(); + } + + roundCancellationToken.ThrowIfCancellationRequested(); + await ApplyAntiEntropyResponses(responses, options, roundCancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + finally + { + foreach (var lease in leases.Values) + { + lease.Dispose(); + } + } + } + + private Dictionary AcquireAntiEntropyPeers( + DisseminationMembershipSnapshots membershipSnapshots, + int peerCount) + { + // AllMembers is a superset of ActiveMembers, preserving a single budget across namespace scopes. + var selectionScope = _namespaces.Values.Any(static disseminationNamespace => + disseminationNamespace.Options.Enabled + && disseminationNamespace.MembershipScope == DisseminationMembershipScope.AllMembers) + ? DisseminationMembershipScope.AllMembers + : DisseminationMembershipScope.ActiveMembers; + var result = new Dictionary(); + foreach (var peer in membershipSnapshots.GetSnapshot(selectionScope).SelectAntiEntropyPeers(peerCount)) + { + if (_antiEntropySendGate.TryAcquire(peer, out var lease)) + { + result.Add(peer, lease); + } + } + + return result; + } + + private Dictionary CreateAntiEntropyRequests( + DisseminationMembershipSnapshots membershipSnapshots, + Dictionary> requestDigests, + IEnumerable peers, + DisseminationOptions options) + { + var result = new Dictionary(); + var enabledNamespaces = _namespaces.Values.Where(static ns => ns.Options.Enabled).ToArray(); + foreach (var peer in peers) + { + var peerDigests = new Dictionary>(); + var supportedNamespaces = new List(); + foreach (var disseminationNamespace in enabledNamespaces) + { + if (!membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope).ContainsMember(peer)) + { + continue; + } + + supportedNamespaces.Add(disseminationNamespace.Name); + if (requestDigests.TryGetValue(disseminationNamespace.Name, out var digest)) + { + peerDigests.Add(disseminationNamespace.Name, digest); + } + } + + if (supportedNamespaces.Count > 0) + { + result.Add(peer, new DisseminationAntiEntropyRequest + { + Sender = _localSilo, + Digests = peerDigests, + SupportedNamespaces = supportedNamespaces, + MaxResponseItems = Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems), + MaxResponseBytes = Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes), + }); + } + } + + return result; + } + + private TimeSpan GetAntiEntropyRoundLifetime( + IEnumerable requests) + { + var result = MaxAntiEntropyRoundLifetime; + foreach (var request in requests) + { + foreach (var namespaceName in request.SupportedNamespaces) + { + if (_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + && disseminationNamespace.Options.StaleItemTtl < result) + { + result = disseminationNamespace.Options.StaleItemTtl; + } + } + } + + return result; + } + + private Dictionary> CreateAntiEntropyRequestDigests(long now) + { + // New or quiet streams need periodic repair; this pass also forgets streams which disappeared. + Dictionary lastValueUpdates; + lock (_valueUpdateLock) + { + lastValueUpdates = new(_lastValueUpdates); + } + + Dictionary>? digestsByNamespace = null; + var currentValueStreams = new HashSet(); + + foreach (var disseminationNamespace in _namespaces.Values) + { + var namespaceOptions = disseminationNamespace.Options; + if (!namespaceOptions.Enabled) + { + continue; + } + + List? digestEntries = null; + foreach (var digest in disseminationNamespace.Digests) + { + var digestKey = new DigestKey(disseminationNamespace.Name, digest.Key); + currentValueStreams.Add(digestKey); + + if (!lastValueUpdates.TryGetValue(digestKey, out var lastUpdate) + || lastUpdate.Version != digest.Version + || _timeProvider.GetElapsedTime(lastUpdate.Timestamp, now) >= namespaceOptions.ExpectedUpdateCadence) + { + (digestEntries ??= []).Add(digest); + } + } + + if (digestEntries is not null) + { + (digestsByNamespace ??= [])[disseminationNamespace.Name] = digestEntries; + } + } + + lock (_valueUpdateLock) + { + foreach (var key in _lastValueUpdates.Keys) + { + if (!currentValueStreams.Contains(key)) + { + _lastValueUpdates.Remove(key); + } + } + } + + return digestsByNamespace ?? []; + } + + private async Task ExchangeAntiEntropyRequest( + SiloAddress peer, + DisseminationAntiEntropyRequest request, + DisseminationSendGate.Lease lease, + int requestDigestCount, + CancellationToken cancellationToken, + CancellationToken callerCancellationToken, + CancellationToken lifetimeCancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + var exchangeTask = _grainFactory.GetSystemTarget(Constants.DisseminationSystemTargetType, peer) + .ExchangeAntiEntropy(request, cancellationToken); + exchangeTask.Ignore(); + var response = await exchangeTask.WaitAsync(cancellationToken).ConfigureAwait(false); + // Record exchange metrics after the peer returns so truncation and repair counts reflect the response. + EmitAntiEntropyExchange( + peer, + "out", + requestDigestCount, + GetValueCount(response.Values), + response.Truncated); + return response; + } + catch (OperationCanceledException) when (callerCancellationToken.IsCancellationRequested) + { + return null; + } + catch (OperationCanceledException) when (lifetimeCancellationToken.IsCancellationRequested) + { + EmitAntiEntropyFailure(peer, DisseminationFailureReason.Timeout); + return null; + } + catch (Exception exception) + { + // Anti-entropy transport failures are isolated to the peer; random peer selection naturally spreads retries. + EmitAntiEntropyFailure(peer, DisseminationFailureReason.Error); + LogDebugDisseminationSendFailed(_logger, exception, peer); + return null; + } + finally + { + lease.Dispose(); + EmitAntiEntropyAdmissionReleased(peer); + } + } + + private void EmitAntiEntropyAdmissionReleased(SiloAddress peer) + { + try + { + DisseminationEvents.EmitSendGate(_localSilo, peer, kind: "repair", stage: "released"); + } + catch (Exception exception) + { + LogDebugDisseminationAdmissionDiagnosticFailed(_logger, exception, peer); + } + } + + private async Task ApplyAntiEntropyResponses( + DisseminationAntiEntropyResponse?[] responses, + DisseminationOptions options, + CancellationToken cancellationToken) + { + // Completed exchanges get a separate local application window, even if another peer used its whole + // transport budget. No remote clock or transmission delay is inferred from a relative wire lifetime. + var receivedTimestamp = _timeProvider.GetTimestamp(); + foreach (var response in responses) + { + if (response is null) + { + continue; + } + + ConfirmPeerNamespaces(response.Sender, response.SupportedNamespaces); + RevokePeerNamespaces(response.Sender, response.UnsupportedNamespaces); + // A response only includes namespaces which produced repairs. Absence is not evidence that an + // up-to-date or unrelated namespace is unsupported, so confirmations are additive here. + ConfirmPeerNamespaces(response.Sender, response.Values.Keys); + foreach (var (namespaceName, values) in SelectReceivedValues(response.Sender, response.Values, options, antiEntropy: true, out _)) + { + cancellationToken.ThrowIfCancellationRequested(); + var disseminationNamespace = _namespaces[namespaceName]; + + foreach (var item in values) + { + _broadcastQueue.ObservePeerVersion( + response.Sender, + namespaceName, + item.Value.Key, + item.Value.ToVersion); + + // Equal-version membership snapshots can contain complementary heartbeat advances. + await ApplyReceivedValue( + disseminationNamespace, + item, + response.Sender, + options, + receivedTimestamp, + allowDelta: false, + cancellationToken); + } + } + } + } + + private Dictionary> SelectReceivedValues( + SiloAddress peer, + Dictionary> values, + DisseminationOptions options, + bool antiEntropy, + out bool isUnfiltered) + { + isUnfiltered = false; + var maxItems = antiEntropy ? Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems) : options.MaxBatchItems; + var maxBytes = antiEntropy ? Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes) : options.MaxBatchBytes; + long totalCount = 0; + foreach (var entries in values.Values) + { + totalCount += entries.Count; + } + var cursorKey = (peer, antiEntropy); + long position; + lock (_receivedBatchCursorLock) + { + position = _receivedBatchCursors.TryGetValue(cursorKey, out var cursor) + && cursor.TotalCount == totalCount && cursor.Position < totalCount + ? cursor.Position + : 0; + } + + if (position == 0 && FitsReceiveBudget(values, totalCount, maxItems, maxBytes)) + { + var fastPathMembers = _membership.CurrentSnapshots.AllMembers; + lock (_receivedBatchCursorLock) + { + _receivedBatchCursors.Remove(cursorKey); + PruneReceivedBatchCursors(fastPathMembers); + } + + // Compact acknowledgments require this whole-batch validation, independently of collection reuse. + isUnfiltered = true; + return values; + } + + var result = new Dictionary>(); + var skip = position; + var examined = 0; + var byteCount = 0; + foreach (var (namespaceName, entries) in values) + { + if (skip >= entries.Count) + { + skip -= entries.Count; + continue; + } + + if (!TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + position += entries.Count - skip; + skip = 0; + continue; + } + + for (var index = (int)skip; index < entries.Count; index++) + { + if (examined >= maxItems || byteCount >= maxBytes) + { + goto Complete; + } + + var item = entries[index]; + var payloadBytes = item.Value.Payload.Length; + if (!ValidatePayloadSize(disseminationNamespace, item.Value, maxBytes)) + { + // Oversized entries consume inspection capacity, but must not pin the receive cursor. + examined++; + position++; + continue; + } + + if (payloadBytes > maxBytes - byteCount) + { + // Preserve this candidate for a fresh budget instead of discarding it. + goto Complete; + } + + if (!result.TryGetValue(namespaceName, out var selected)) + { + selected = []; + result.Add(namespaceName, selected); + } + + selected.Add(item); + examined++; + byteCount += payloadBytes; + position++; + } + + skip = 0; + } + +Complete: + var members = _membership.CurrentSnapshots.AllMembers; + lock (_receivedBatchCursorLock) + { + // Inspect each item at most once per delivery. Repeated oversized batches resume later, + // allowing cold keys past a repeatedly rejected or hot prefix. + if (position < totalCount) + { + _receivedBatchCursors[cursorKey] = new(position, totalCount, ++_receivedBatchCursorAccess); + } + else + { + _receivedBatchCursors.Remove(cursorKey); + } + + PruneReceivedBatchCursors(members); + } + + return result; + } + + private bool FitsReceiveBudget( + Dictionary> values, + long itemCount, + int maxItems, + int maxBytes) + { + if (itemCount > maxItems) + { + return false; + } + + var remainingBytes = maxBytes; + foreach (var (namespaceName, entries) in values) + { + if (entries.Count == 0 || !TryGetEnabledNamespace(namespaceName, out var disseminationNamespace)) + { + return false; + } + + var maxPayloadBytes = disseminationNamespace.Options.MaxPayloadBytes; + foreach (var entry in entries) + { + var length = entry.Value.Payload.Length; + if (remainingBytes == 0 || length > maxPayloadBytes || length > remainingBytes) + { + return false; + } + + remainingBytes -= length; + } + } + + return true; + } + + // Called under _receivedBatchCursorLock. + private void PruneReceivedBatchCursors(DisseminationMembershipSnapshot members) + { + if (_receivedBatchCursors.Count > MaxRetainedNonMemberResponseCursors) + { + var nonMembers = _receivedBatchCursors + .Where(entry => !members.ContainsMember(entry.Key.Peer)) + .OrderByDescending(static entry => entry.Value.LastAccess) + .Skip(MaxRetainedNonMemberResponseCursors) + .Select(static entry => entry.Key) + .ToArray(); + foreach (var key in nonMembers) + { + _receivedBatchCursors.Remove(key); + } + } + } + + public ValueTask ReceiveAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var admission = _admission.TryEnter(); + ObjectDisposedException.ThrowIf(!admission.Entered, this); + + if (request.MaxResponseItems is { } maxResponseItems) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResponseItems, nameof(request.MaxResponseItems)); + } + + if (request.MaxResponseBytes is { } maxResponseBytes) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResponseBytes, nameof(request.MaxResponseBytes)); + } + + ConfirmPeerNamespaces(request.Sender, request.SupportedNamespaces); + ConfirmPeerNamespaces(request.Sender, request.Digests.Keys); + // Incoming digests are passive evidence for existing peer pumps, not a reason to create new ones. + foreach (var (namespaceName, entries) in request.Digests) + { + foreach (var entry in entries) + { + _broadcastQueue.ObservePeerVersion( + request.Sender, + namespaceName, + entry.Key, + entry.Version); + } + } + + var options = _options.CurrentValue; + var response = CreateAntiEntropyResponse(request, options, cancellationToken); + PruneAntiEntropyResponseCursors(_membership.CurrentSnapshots.AllMembers, request.Sender); + return new(response); + } + + private DisseminationAntiEntropyResponse CreateAntiEntropyResponse( + DisseminationAntiEntropyRequest request, + DisseminationOptions options, + CancellationToken cancellationToken) + { + if (!options.Enabled) + { + return new DisseminationAntiEntropyResponse + { + Sender = _localSilo, + Values = [], + Truncated = false, + UnsupportedNamespaces = [.. request.SupportedNamespaces], + }; + } + + var supportedNamespaces = new List(); + var unsupportedNamespaces = new List(); + foreach (var namespaceName in request.SupportedNamespaces) + { + if (TryGetEnabledNamespace(namespaceName, out _)) + { + supportedNamespaces.Add(namespaceName); + } + else + { + unsupportedNamespaces.Add(namespaceName); + } + } + + // Honor the recipient's budget at the responder's existing fair cursor. Independently truncating + // rotating responses at the receiver can repeatedly omit the same keys when the limits differ. + var maxResponseItems = Math.Min(Math.Min(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems), + request.MaxResponseItems ?? int.MaxValue); + var maxResponseBytes = Math.Min(Math.Min(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes), + request.MaxResponseBytes ?? int.MaxValue); + var valueCount = 0; + var byteCount = 0; + var truncated = false; + var valuesByNamespace = new Dictionary>(); + var candidates = new List(); + foreach (var (namespaceName, remoteDigest) in request.Digests) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!TryGetEnabledNamespace(namespaceName, out var requestedNamespace)) + { + continue; + } + + if (remoteDigest.Count == 0) + { + continue; + } + + var remoteVersions = CreateDigestLookup(remoteDigest); + foreach (var localDigest in requestedNamespace.Digests) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!remoteVersions.TryGetValue(localDigest.Key, out var peerDigest)) + { + continue; + } + + if (localDigest.Version < peerDigest.Version + || localDigest.Version == peerDigest.Version + && localDigest.Fingerprint == peerDigest.Fingerprint) + { + continue; + } + + candidates.Add(new(requestedNamespace, localDigest, peerDigest)); + } + } + + var start = GetAntiEntropyResponseStart(request.Sender, candidates.Count); + var examined = 0; + for (var i = 0; i < candidates.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + if (valueCount >= maxResponseItems) + { + truncated = true; + break; + } + + var candidate = candidates[(start + i) % candidates.Count]; + examined++; + var requestedNamespace = candidate.Namespace; + var localDigest = candidate.LocalDigest; + var peerDigest = candidate.PeerDigest; + var repairRequest = new DisseminationRepairRequest( + localDigest.Key, + peerDigest.Version, + maxResponseBytes, + requestedNamespace.Options.MaxPayloadBytes); + var repair = requestedNamespace.CreateRepair(repairRequest); + if (repair.Status is not DisseminationRepairStatus.Produced + || !ValidateRepair(requestedNamespace, repairRequest, repair, options)) + { + continue; + } + + var value = repair.Value; + if (value.Payload.Length > maxResponseBytes - byteCount) + { + // This value fits a fresh response, so resume here instead of discarding it. + examined--; + truncated = true; + break; + } + + if (!valuesByNamespace.TryGetValue(requestedNamespace.Name, out var namespaceValues)) + { + namespaceValues = []; + valuesByNamespace.Add(requestedNamespace.Name, namespaceValues); + } + + namespaceValues.Add(CreateBroadcastValue(requestedNamespace, value)); + ++valueCount; + byteCount += value.Payload.Length; + } + + if (truncated) + { + AdvanceAntiEntropyResponseCursor(request.Sender, start, examined, candidates.Count); + } + else + { + ClearAntiEntropyResponseCursor(request.Sender); + } + + EmitAntiEntropyExchange(request.Sender, "in", GetDigestCount(request.Digests), valueCount, truncated); + return new DisseminationAntiEntropyResponse + { + Sender = _localSilo, + Values = valuesByNamespace, + Truncated = truncated, + SupportedNamespaces = supportedNamespaces, + UnsupportedNamespaces = unsupportedNamespaces, + }; + } + + private int GetAntiEntropyResponseStart(SiloAddress peer, int candidateCount) + { + if (candidateCount == 0) + { + return 0; + } + + lock (_antiEntropyResponseCursorLock) + { + if (!_antiEntropyResponseCursors.TryGetValue(peer, out var cursor)) + { + return 0; + } + + _antiEntropyResponseCursors[peer] = cursor with + { + LastAccess = ++_antiEntropyResponseCursorAccess, + }; + return cursor.Position % candidateCount; + } + } + + private void AdvanceAntiEntropyResponseCursor( + SiloAddress peer, + int start, + int examined, + int candidateCount) + { + lock (_antiEntropyResponseCursorLock) + { + if (candidateCount == 0) + { + _antiEntropyResponseCursors.Remove(peer); + } + else + { + _antiEntropyResponseCursors[peer] = new( + (start + Math.Max(1, examined)) % candidateCount, + ++_antiEntropyResponseCursorAccess); + } + } + } + + private void ClearAntiEntropyResponseCursor(SiloAddress peer) + { + lock (_antiEntropyResponseCursorLock) + { + _antiEntropyResponseCursors.Remove(peer); + } + } + + private void PruneAntiEntropyResponseCursors( + DisseminationMembershipSnapshot membership, + SiloAddress currentRequester) + { + lock (_antiEntropyResponseCursorLock) + { + var nonMemberCount = 0; + foreach (var peer in _antiEntropyResponseCursors.Keys) + { + if (!membership.ContainsMember(peer)) + { + nonMemberCount++; + } + } + + if (nonMemberCount <= MaxRetainedNonMemberResponseCursors) + { + return; + } + + foreach (var cursor in _antiEntropyResponseCursors + .Where(entry => !Equals(entry.Key, currentRequester) && !membership.ContainsMember(entry.Key)) + .OrderBy(static entry => entry.Value.LastAccess) + .ToArray()) + { + _antiEntropyResponseCursors.Remove(cursor.Key); + if (--nonMemberCount <= MaxRetainedNonMemberResponseCursors) + { + break; + } + } + } + } + + private async ValueTask ApplyReceivedValue( + IDisseminationNamespace disseminationNamespace, + DisseminationBroadcastValue item, + SiloAddress sender, + DisseminationOptions options, + long receivedTimestamp, + bool allowDelta, + CancellationToken cancellationToken) + { + try + { + return await ApplyReceivedValueCore( + disseminationNamespace, + item, + sender, + options, + receivedTimestamp, + allowDelta, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + catch (Exception exception) + { + LogDebugDisseminationValueApplyFailed( + _logger, + exception, + sender, + disseminationNamespace.Name, + item.Value.Key, + item.Value.ToVersion); + EmitApplyResult(disseminationNamespace.Name, item, sender, DisseminationApplyResult.Rejected); + return DisseminationApplyResult.Rejected; + } + } + + private async ValueTask ApplyReceivedValueCore( + IDisseminationNamespace disseminationNamespace, + DisseminationBroadcastValue item, + SiloAddress sender, + DisseminationOptions options, + long receivedTimestamp, + bool allowDelta, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var namespaceName = disseminationNamespace.Name; + if (!ValidatePayloadSize(disseminationNamespace, item.Value, options.MaxBatchBytes)) + { + return DisseminationApplyResult.Rejected; + } + + var lifetime = item.TimeToLive < disseminationNamespace.Options.StaleItemTtl + ? item.TimeToLive + : disseminationNamespace.Options.StaleItemTtl; + var remainingLifetime = lifetime - _timeProvider.GetElapsedTime(receivedTimestamp); + if (remainingLifetime <= TimeSpan.Zero) + { + EmitApplyResult(namespaceName, item, sender, DisseminationApplyResult.Obsolete); + return DisseminationApplyResult.Obsolete; + } + + if (TryGetTerminalApplyResult(disseminationNamespace, item.Value, allowDelta, out var terminalResult)) + { + EmitApplyResult(namespaceName, item, sender, terminalResult); + return terminalResult; + } + + using var lifetimeCancellation = new CancellationTokenSource( + remainingLifetime < MaxAntiEntropyRoundLifetime ? remainingLifetime : MaxAntiEntropyRoundLifetime, + _timeProvider); + using var applicationCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + lifetimeCancellation.Token); + DisseminationApplyResult result; + Task? applicationTask = null; + try + { + var application = disseminationNamespace.ApplyValueAsync(item.Value, applicationCancellation.Token); + if (application.IsCompletedSuccessfully) + { + result = application.Result; + } + else + { + // Only this local wait is bounded. Namespace owners must observe cancellation before mutating + // queued state; arbitrary implementations which ignore the token cannot be forcibly stopped. + applicationTask = application.AsTask(); + applicationTask.Ignore(); + result = await applicationTask.WaitAsync(applicationCancellation.Token); + } + } + catch (OperationCanceledException) when ( + lifetimeCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + result = applicationTask is { IsCompletedSuccessfully: true } + ? applicationTask.Result + : DisseminationApplyResult.Obsolete; + } + + cancellationToken.ThrowIfCancellationRequested(); + EmitApplyResult(namespaceName, item, sender, result); + if (result is DisseminationApplyResult.Applied) + { + RecordValueUpdate(namespaceName, item.Value.Key, item.Value.ToVersion); + } + + return result; + } + + internal async Task FlushPendingBroadcast(CancellationToken cancellationToken) + { + DisseminationRootBatcher[] roots; + lock (_rootBatcherLock) + { + roots = [.. _rootBatchers.Values]; + } + + await Task.WhenAll(roots.Select(root => root.FlushAsync(cancellationToken))); + await _broadcastQueue.FlushPendingBroadcast(cancellationToken); + } + + internal async Task StopAsync(CancellationToken cancellationToken) + { + var admittedOperations = _admission.CloseAsync(); + _antiEntropySendGate.Stop(); + _publicationSendGate.Stop(); + DisseminationRootBatcher[] roots; + lock (_rootBatcherLock) + { + _rootBatchersStopped = true; + roots = [.. _rootBatchers.Values]; + _rootBatchers.Clear(); + } + + // Sealing releases held ingress receipts, so it precedes waiting for their protocol admissions. + var rootStops = Task.WhenAll(roots.Select(root => root.StopAsync(cancellationToken))); + try + { + await Task.WhenAll(_antiEntropyShutdown.CancelAsync(), _publicationShutdown.CancelAsync()); + await Task.WhenAll(rootStops, admittedOperations.WaitAsync(cancellationToken)); + } + finally + { + rootStops.Ignore(); + await _broadcastQueue.StopAsync(cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + + internal IReadOnlyList GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) + { + var membershipSnapshots = _membership.CurrentSnapshots; + PrunePeerNamespaceConfirmations(membershipSnapshots.AllMembers); + var participants = membershipSnapshots.GetSnapshot(disseminationNamespace.MembershipScope).Members; + lock (_peerSupportLock) + { + List? result = null; + foreach (var peer in participants) + { + if (Equals(peer, _localSilo)) + { + continue; + } + + if (!_confirmedPeerNamespaces.TryGetValue(peer, out var namespaces) + || !namespaces.Contains(disseminationNamespace.Name)) + { + (result ??= []).Add(peer); + } + } + + return result ?? []; + } + } + + private void ObserveBroadcastResponse( + SiloAddress peer, + DisseminationBroadcastResponse response) + { + ConfirmPeerNamespaces(peer, response.Acknowledgments.Keys); + RevokePeerNamespaces(peer, response.UnsupportedNamespaces); + } + + private void ConfirmPeerNamespaces( + SiloAddress peer, + IEnumerable namespaceNames) + { + if (Equals(peer, _localSilo) + || !_membership.CurrentSnapshots.AllMembers.ContainsMember(peer)) + { + return; + } + + lock (_peerSupportLock) + { + HashSet? confirmedNamespaces = null; + foreach (var namespaceName in namespaceNames) + { + if (!_namespaces.TryGetValue(namespaceName, out var disseminationNamespace) + || !disseminationNamespace.Options.Enabled) + { + continue; + } + + if (confirmedNamespaces is null) + { + if (!_confirmedPeerNamespaces.TryGetValue(peer, out confirmedNamespaces)) + { + confirmedNamespaces = []; + _confirmedPeerNamespaces.Add(peer, confirmedNamespaces); + } + } + + confirmedNamespaces.Add(namespaceName); + } + } + } + + private void RevokePeerNamespaces( + SiloAddress peer, + IEnumerable namespaceNames) + { + lock (_peerSupportLock) + { + if (!_confirmedPeerNamespaces.TryGetValue(peer, out var confirmedNamespaces)) + { + return; + } + + foreach (var namespaceName in namespaceNames) + { + confirmedNamespaces.Remove(namespaceName); + } + + if (confirmedNamespaces.Count == 0) + { + _confirmedPeerNamespaces.Remove(peer); + } + } + } + + private void PrunePeerNamespaceConfirmations(DisseminationMembershipSnapshot membership) + { + lock (_peerSupportLock) + { + if (ReferenceEquals(_confirmedMembership, membership)) + { + return; + } + + membership = _membership.CurrentSnapshots.AllMembers; + foreach (var peer in _confirmedPeerNamespaces.Keys) + { + if (!membership.ContainsMember(peer)) + { + _confirmedPeerNamespaces.Remove(peer); + } + } + + _confirmedMembership = membership; + } + } + + private void RecordValueUpdate(DisseminationNamespace namespaceName, DisseminationKey key, long version) + { + // Recent successful updates suppress anti-entropy until the namespace's expected cadence elapses. + var digestKey = new DigestKey(namespaceName, key); + var update = new ValueUpdate(version, _timeProvider.GetTimestamp()); + lock (_valueUpdateLock) + { + if (!_lastValueUpdates.TryGetValue(digestKey, out var previous) || version > previous.Version) + { + _lastValueUpdates[digestKey] = update; + } + } + } + + private async ValueTask GetMembershipSnapshotForRouting( + DisseminationMembershipScope scope, + SiloAddress member, + CancellationToken cancellationToken) + { + // Prune peer ledgers against the same membership view used to choose tree targets. + await _membership.GetSnapshotsContainingMember(member, scope, cancellationToken); + await MaintainInventories(cancellationToken); + var snapshot = _membership.GetSnapshot(scope); + return snapshot.ContainsMember(member) ? snapshot : null; + } + + private bool NotifyRoot( + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + return notifications.IsEmpty || GetRootBatcher(disseminationNamespace)?.Notify(notifications) == true; + } + + private DisseminationRootBatcher? GetRootBatcher(IDisseminationNamespace disseminationNamespace) + { + lock (_rootBatcherLock) + { + if (_rootBatchersStopped) + { + return null; + } + + if (!_rootBatchers.TryGetValue(disseminationNamespace.Name, out var root)) + { + root = new( + _timeProvider, disseminationNamespace, _options, + () => _membership.GetSnapshot(disseminationNamespace.MembershipScope), + values => DispatchRootBatch(disseminationNamespace, values), _logger); + _rootBatchers.Add(disseminationNamespace.Name, root); + } + + return root; + } + } + + private bool DispatchRootBatch( + IDisseminationNamespace disseminationNamespace, + ReadOnlySpan notifications) + { + var membership = _membership.GetSnapshot(disseminationNamespace.MembershipScope); + if (!membership.IsAggregationRoot) + { + // A departing root hands accepted state to the new root, even after leaving Active membership. + return membership.Members.IsEmpty + || _broadcastQueue.NotifyBatch(membership.Members[0], disseminationNamespace, notifications); + } + + var accepted = true; + foreach (var child in membership.AggregationChildren) + { + // Include a child producer's own value so it reaches that child's descendants. + accepted &= _broadcastQueue.NotifyBatch(child, disseminationNamespace, notifications); + } + + return accepted; + } + + private async ValueTask MaintainInventories(CancellationToken cancellationToken) + { + MaintenanceStamp reservation; + bool membershipChanged; + lock (_maintenanceLock) + { + var membership = _membership.CurrentSnapshots; + var now = _timeProvider.GetTimestamp(); + membershipChanged = !ReferenceEquals(_lastMaintenance?.Membership, membership); + if (!membershipChanged + && _lastMaintenance is { } previous + && _timeProvider.GetElapsedTime(previous.Timestamp, now) < InventoryMaintenanceInterval) + { + return; + } + + // Reserve the pass before enumerating namespace state, keeping callbacks outside this lock. + reservation = new(membership, now); + _lastMaintenance = reservation; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + PrunePeerNamespaceConfirmations(reservation.Membership.AllMembers); + await _broadcastQueue.Prune(reservation.Membership, cancellationToken); + KeyValuePair[] roots; + lock (_rootBatcherLock) + { + roots = [.. _rootBatchers]; + } + + foreach (var (name, root) in roots) + { + cancellationToken.ThrowIfCancellationRequested(); + var ns = _namespaces[name]; + var activeKeys = ns.Options.Enabled ? ns.Keys.ToHashSet() : new HashSet(); + root.Prune(activeKeys); + if (membershipChanged && !reservation.Membership.GetSnapshot(ns.MembershipScope).IsAggregationRoot) + { + root.WakeForMembershipChange(); + } + } + } + catch + { + lock (_maintenanceLock) + { + if (ReferenceEquals(_lastMaintenance, reservation)) + { + _lastMaintenance = null; + } + } + + throw; + } + } + + private sealed record MaintenanceStamp(DisseminationMembershipSnapshots Membership, long Timestamp); + + private bool TryGetEnabledNamespace(DisseminationNamespace namespaceName, [NotNullWhen(true)] out IDisseminationNamespace? disseminationNamespace) + { + if (_namespaces.TryGetValue(namespaceName, out disseminationNamespace) + && disseminationNamespace.Options.Enabled) + { + return true; + } + + disseminationNamespace = null; + return false; + } + + private bool ValidatePayloadSize( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value, + int maxBatchBytes) + { + if (value.Payload.Length <= disseminationNamespace.Options.MaxPayloadBytes + && value.Payload.Length <= maxBatchBytes) + { + return true; + } + + var namespaceName = disseminationNamespace.Name; + try + { + DisseminationEvents.EmitPayloadDrop(namespaceName, value, _localSilo, "oversize", value.Payload.Length); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, value.Key); + } + + try + { + DisseminationInstruments.OnPayloadDropped(namespaceName, "oversize"); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, value.Key); + } + + return false; + } + + private bool TryValidatePublish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + DisseminationOptions options, + out long publishedVersion, + [NotNullWhen(false)] out string? failureReason) + { + if (version <= 0) + { + publishedVersion = 0; + failureReason = "invalid-version"; + return false; + } + + if (disseminationNamespace.GetVersion(key) < version) + { + publishedVersion = 0; + failureReason = "unavailable"; + return false; + } + + // Validate a complete current value for an unknown peer before accepting the publication. + var request = new DisseminationRepairRequest( + key, + fromVersion: null, + options.MaxBatchBytes, + disseminationNamespace.Options.MaxPayloadBytes); + var repair = disseminationNamespace.CreateRepair(request); + if (repair.Status is DisseminationRepairStatus.InsufficientCapacity) + { + publishedVersion = 0; + failureReason = "oversize"; + return false; + } + + if (repair.Status is not DisseminationRepairStatus.Produced + || repair.Version < version + || !ValidateRepair(disseminationNamespace, request, repair, options)) + { + publishedVersion = 0; + failureReason = "invalid-repair"; + return false; + } + + publishedVersion = repair.Version; + failureReason = null; + return true; + } + + private bool ValidateRepair( + IDisseminationNamespace disseminationNamespace, + in DisseminationRepairRequest request, + in DisseminationRepairResult repair, + DisseminationOptions options) + { + var value = repair.Value; + return repair.Status is DisseminationRepairStatus.Produced + && repair.Version > 0 + && value.Key == request.Key + && value.FromVersion == 0 + && value.ToVersion == repair.Version + && ValidatePayloadSize(disseminationNamespace, value, options.MaxBatchBytes) + && value.Payload.Length <= request.MaxPayloadBytes + && value.Payload.Length <= request.MaxBatchBytes; + } + + private static bool TryGetTerminalApplyResult( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value, + bool allowDelta, + out DisseminationApplyResult result) + { + if (value.ToVersion <= 0 || value.FromVersion < 0 + || value.FromVersion > 0 && (!allowDelta || value.FromVersion > value.ToVersion)) + { + result = DisseminationApplyResult.Rejected; + return true; + } + + var localVersion = disseminationNamespace.GetVersion(value.Key); + if (value.ToVersion < localVersion) + { + result = DisseminationApplyResult.Obsolete; + return true; + } + + result = default; + return false; + } + + private DisseminationBroadcastValue CreateBroadcastValue( + IDisseminationNamespace disseminationNamespace, + DisseminationValue value) => + new() + { + Value = value, + TimeToLive = disseminationNamespace.Options.StaleItemTtl, + }; + + private void EmitApplyResult(DisseminationNamespace namespaceName, DisseminationBroadcastValue item, SiloAddress sender, DisseminationApplyResult result) + { + try + { + DisseminationEvents.EmitValue(namespaceName, item.Value, _localSilo, sender, result, item.Value.Payload.Length); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, item.Value.Key); + } + + try + { + DisseminationInstruments.OnValueApplied(namespaceName, result); + } + catch (Exception exception) + { + LogDebugDisseminationDiagnosticFailed(_logger, exception, namespaceName, item.Value.Key); + } + } + + private static int GetDigestCount(Dictionary> digest) => digest.Values.Sum(entries => entries.Count); + + private void EmitPublication(DisseminationNamespace namespaceName, bool accepted, string reason) + { + try + { + DisseminationInstruments.OnPublication(namespaceName, accepted, reason); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, _localSilo, "publication"); + } + } + + private void EmitAntiEntropyExchange(SiloAddress peer, string direction, int digests, int values, bool truncated) + { + try + { + DisseminationInstruments.OnAntiEntropyExchange(direction, digests, values, truncated); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, peer, "anti-entropy-exchange"); + } + } + + private void EmitAntiEntropyFailure(SiloAddress peer, DisseminationFailureReason reason) + { + try + { + DisseminationInstruments.OnAntiEntropyFailure(reason); + } + catch (Exception exception) + { + LogDebugProtocolDiagnosticFailed(_logger, exception, peer, "anti-entropy-failure"); + } + } + + private static Dictionary CreateDigestLookup(List digest) + { + var result = new Dictionary(digest.Count); + foreach (var entry in digest) + { + result[entry.Key] = entry; + } + + return result; + } + + private static int GetValueCount(Dictionary> valuesByNamespace) => valuesByNamespace.Values.Sum(values => values.Count); + + private readonly record struct DigestKey(DisseminationNamespace Namespace, DisseminationKey Key); + + private readonly record struct ValueUpdate(long Version, long Timestamp); + + private readonly record struct AntiEntropyResponseCursor(int Position, long LastAccess); + + private readonly record struct ReceivedBatchCursor(long Position, long TotalCount, long LastAccess); + + private readonly record struct AntiEntropyResponseCandidate( + IDisseminationNamespace Namespace, + DigestEntry LocalDigest, + DigestEntry PeerDigest); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination protocol diagnostic for {Peer} during {Operation} failed.")] + private static partial void LogDebugProtocolDiagnosticFailed(ILogger logger, Exception exception, SiloAddress peer, string operation); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination send to {Peer} failed.")] + private static partial void LogDebugDisseminationSendFailed(ILogger logger, Exception exception, SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination admission diagnostic for {Peer} failed.")] + private static partial void LogDebugDisseminationAdmissionDiagnosticFailed(ILogger logger, Exception exception, SiloAddress peer); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Failed to apply dissemination value from {Sender} for namespace {Namespace}, key {Key}, version {Version}.")] + private static partial void LogDebugDisseminationValueApplyFailed(ILogger logger, Exception exception, SiloAddress sender, DisseminationNamespace @namespace, DisseminationKey key, long version); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination value diagnostic failed for namespace {Namespace}, key {Key}.")] + private static partial void LogDebugDisseminationDiagnosticFailed( + ILogger logger, + Exception exception, + DisseminationNamespace @namespace, + DisseminationKey key); + +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs new file mode 100644 index 00000000000..5ab61a7faae --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationRootBatcher.cs @@ -0,0 +1,812 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Internal; +using KeyNotification = Orleans.Runtime.Dissemination.DisseminationBroadcastQueue.KeyNotification; + +namespace Orleans.Runtime.Dissemination; + +// Cohorts contain identities, not values. Peer queues still own payload materialization, byte +// limits, splitting, and RPC lifetime. A receipt acknowledges admission there, not delivery. +internal sealed partial class DisseminationRootBatcher +{ + internal delegate bool DispatchCallback(ReadOnlySpan notifications); + + private readonly object _lock = new(); + private readonly TimeProvider _timeProvider; + private readonly IDisseminationNamespace _namespace; + private readonly DisseminationNamespace _namespaceName; + private readonly IOptionsMonitor _options; + private readonly Func _getMembership; + private readonly DispatchCallback _dispatch; + private readonly ILogger _logger; + private readonly WakeTimer _timer; + private readonly Task _worker; + private readonly Dictionary _pending = []; + private readonly LinkedList _ready = []; + private readonly Dictionary _producers = []; + private DisseminationMembershipSnapshot? _membership; + private Cohort? _cohort; + private TaskCompletionSource? _drainCompletion; + private Exception? _failure; + private long? _retryTimestamp; + private TimeSpan _retryPeriod; + private bool _dispatching; + private bool _workerActive; + private bool _handoff; + private bool _stopping; + private bool _aborted; + private CancellationToken _abortToken; + + public DisseminationRootBatcher( + TimeProvider timeProvider, + IDisseminationNamespace disseminationNamespace, + IOptionsMonitor options, + Func getMembership, + DispatchCallback dispatch, + ILogger logger) + { + _timeProvider = timeProvider; + _namespace = disseminationNamespace; + _namespaceName = disseminationNamespace.Name; + _options = options; + _getMembership = getMembership; + _dispatch = dispatch; + _logger = logger; + _timer = new(timeProvider); + using var _ = new ExecutionContextSuppressor(); + _worker = RunAsync(); + } + + // Hints (including owner inventory and forwarded values) never count as fresh contributions. + public bool Notify(ReadOnlySpan notifications) + { + var settings = GetSettings(); + var membership = _getMembership(); + var rejected = 0; + var wake = false; + lock (_lock) + { + ThrowIfFailedUnsafe(); + if (_stopping) + { + return false; + } + + membership = RefreshMembershipUnsafe(membership); + if (!settings.Enabled) + { + ClearPendingUnsafe(); + wake = true; + rejected = notifications.Length; + } + else + { + var wasEmpty = _ready.Count == 0; + foreach (var notification in notifications) + { + if (!IsEligible(notification, membership) + || !TryAddUnsafe(notification, settings, membership, out _)) + { + rejected++; + } + } + + wake = _ready.Count > 0 && wasEmpty; + } + } + + if (wake) + { + Wake(); + } + + if (rejected > 0) + { + ReportRejection(rejected, settings.MaxPendingItemCount); + } + + return settings.Enabled && rejected == 0; + } + + public ValueTask PublishAsync( + KeyNotification notification, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var settings = GetSettings(); + var membership = _getMembership(); + Task completion; + var wake = false; + lock (_lock) + { + ThrowIfFailedUnsafe(); + membership = RefreshMembershipUnsafe(membership); + if (_stopping || !settings.Enabled || !membership.IsAggregationRoot + || notification.Version <= 0 + || notification.Key.Value is not SiloAddress silo || !membership.ContainsMember(silo)) + { + return new(new DisseminationPublicationReceipt(false, settings.Period)); + } + + if (_producers.TryGetValue(silo, out var producer) && notification.Version <= producer.Version) + { + // A retry must neither contribute again nor start another publication period. + // Force on a retry is not a new invalidation; independent invalidations use Notify. + if (_pending.TryGetValue(notification.Key, out var retained)) + { + retained.AdmissionGeneration++; + } + + if (notification.Version <= producer.CompletedVersion) + { + return new(CreateReceipt(producer.Outcome)); + } + + completion = notification.Version <= producer.InFlightVersion && producer.InFlightReceipt is { } inFlight + ? inFlight.Task : retained!.Receipt!.Task; + } + else + { + var wasEmpty = _ready.Count == 0; + if (!TryAddUnsafe(notification, settings, membership, out var pending, contribution: true)) + { + return new(new DisseminationPublicationReceipt(false, settings.Period)); + } + + producer ??= new(); + _producers[silo] = producer; + producer.Version = notification.Version; + // All newer versions before sealing share one signal and occupy one producer slot. + pending.Receipt ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + pending.Producer = producer; + if (_cohort!.Membership.ContainsMember(silo)) + { + _cohort.Contributors.Add(silo); + } + + completion = pending.Receipt.Task; + wake = wasEmpty + || _cohort.Contributors.Count == _cohort.Membership.Members.Length; + } + } + + if (wake) + { + Wake(); + } + + return AwaitReceiptAsync(completion, cancellationToken); + } + + private async ValueTask AwaitReceiptAsync( + Task completion, CancellationToken cancellationToken) + { + // Cancellation detaches this RPC only. Other callers and the admitted contribution survive. + var outcome = await completion.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_lock) + { + ThrowIfFailedUnsafe(); + } + + return CreateReceipt(outcome); + } + + private DisseminationPublicationReceipt CreateReceipt(ReceiptOutcome outcome) + { + var delay = outcome.Period - _timeProvider.GetElapsedTime(outcome.Started); + return new(outcome.Accepted, delay > TimeSpan.Zero ? delay : TimeSpan.Zero); + } + + public async Task FlushAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task completion; + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + if (_pending.Count == 0 && !_dispatching) + { + return; + } + + _drainCompletion ??= new(TaskCreationOptions.RunContinuationsAsynchronously); + completion = _drainCompletion.Task; + } + + Wake(); + await completion.WaitAsync(cancellationToken).ConfigureAwait(false); + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + lock (_lock) + { + _stopping = true; + } + + try + { + cancellationToken.ThrowIfCancellationRequested(); + Wake(); + await _worker.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + lock (_lock) + { + _aborted = true; + _abortToken = cancellationToken; + ClearPendingUnsafe(); + CompleteDrainUnsafe(); + } + + DisposeTimer(); + throw; + } + + lock (_lock) + { + ThrowIfFailedUnsafe(); + ThrowIfAbortedUnsafe(); + } + } + + public void Prune(IReadOnlySet activeKeys) + { + List<(DisseminationKey Key, PendingKey Pending, long Generation)> candidates = []; + lock (_lock) + { + foreach (var (key, pending) in _pending) + { + if (!activeKeys.Contains(key)) + { + candidates.Add((key, pending, pending.AdmissionGeneration)); + } + } + } + + // Namespace callbacks may reenter and may block. Reconcile against admission generations. + foreach (var candidate in candidates) + { + if (_namespace.GetVersion(candidate.Key) > 0) + { + continue; + } + + lock (_lock) + { + if (_pending.TryGetValue(candidate.Key, out var pending) + && ReferenceEquals(pending, candidate.Pending) + && pending.AdmissionGeneration == candidate.Generation) + { + RemoveUnsafe(pending); + } + } + } + + Wake(); + } + + public void WakeForMembershipChange() + { + var membership = _getMembership(); + lock (_lock) + { + RefreshMembershipUnsafe(membership); + // A new root is a different admission destination, so bypass an old child's retry floor. + _handoff |= _ready.Count > 0; + } + + Wake(); + } + + private bool TryAddUnsafe( + KeyNotification notification, in Settings settings, + DisseminationMembershipSnapshot membership, out PendingKey pending, bool contribution = false) + { + if (_pending.TryGetValue(notification.Key, out pending!)) + { + pending.AdmissionGeneration++; + if (!notification.Force && notification.Version <= pending.Notification.Version + && (!contribution || pending.Node.List is not null)) + { + return true; + } + + pending.Notification = new(notification.Key, + Math.Max(notification.Version, pending.Notification.Version), + pending.Notification.Force || notification.Force); + pending.Generation = pending.AdmissionGeneration; + } + else + { + if (_pending.Count >= settings.MaxPendingItemCount) + { + return false; + } + + pending = new(notification); + _pending.Add(notification.Key, pending); + } + + _cohort ??= new(membership, _timeProvider.GetTimestamp(), settings.Period); + if (pending.Node.List is null) + { + _ready.AddLast(pending.Node); + } + + return true; + } + + private async Task RunAsync() + { + try + { + while (await _timer.WaitAsync(CancellationToken.None).ConfigureAwait(false)) + { + while (true) + { + var settings = GetSettings(); + var membership = _getMembership(); + PendingDispatch[] work; + Cohort cohort; + lock (_lock) + { + _workerActive = true; + if (_failure is not null || _aborted) + { + return; + } + + membership = RefreshMembershipUnsafe(membership); + if (!settings.Enabled) + { + ClearPendingUnsafe(); + } + + if (_ready.Count == 0) + { + _cohort = null; + CompleteDrainUnsafe(); + if (_stopping) + { + return; + } + + _workerActive = false; + break; + } + + var delay = GetDelayUnsafe(); + if (delay > TimeSpan.Zero) + { + _timer.Change(delay); + _workerActive = false; + break; + } + + cohort = _cohort!; + work = new PendingDispatch[_ready.Count]; + for (var i = 0; i < work.Length; i++) + { + var pending = _ready.First!.Value; + _ready.RemoveFirst(); + work[i] = new(pending, pending.Generation, pending.Notification, + pending.Receipt, pending.Producer, pending.Producer?.Version ?? 0); + if (pending.Producer is { } producer) + { + producer.InFlightVersion = producer.Version; + producer.InFlightReceipt = pending.Receipt; + } + + pending.InFlight = pending.Receipt; + pending.Receipt = null; + pending.Producer = null; + pending.Notification = pending.Notification with { Force = false }; + } + + _cohort = null; + _handoff = false; + _retryTimestamp = null; + _dispatching = true; + } + + Dispatch(work, cohort, settings); + } + } + } + catch (Exception exception) + { + Fail(exception); + } + finally + { + DisposeTimer(); + lock (_lock) + { + ClearPendingUnsafe(); + CompleteDrainUnsafe(); + } + } + } + + private TimeSpan GetDelayUnsafe() + { + var cohort = _cohort!; + if (_handoff) + { + return TimeSpan.Zero; + } + + var retryDelay = _retryTimestamp is { } retry + ? _retryPeriod - _timeProvider.GetElapsedTime(retry) : TimeSpan.Zero; + var complete = cohort.Membership.Members.Length > 0 + && cohort.Contributors.Count == cohort.Membership.Members.Length; + var collectionDelay = _stopping || _drainCompletion is not null || complete + ? TimeSpan.Zero : cohort.Period - _timeProvider.GetElapsedTime(cohort.Started); + return collectionDelay > retryDelay ? collectionDelay : retryDelay; + } + + private void Dispatch(PendingDispatch[] work, Cohort cohort, in Settings settings) + { + var notifications = new KeyNotification[Math.Min(work.Length, settings.MaxBatchItems)]; + for (var offset = 0; offset < work.Length; offset += notifications.Length) + { + // Revalidate before every chunk: a callback can change membership or exhaust Stop's budget. + var membership = _getMembership(); + var count = Math.Min(notifications.Length, work.Length - offset); + var sendCount = 0; + lock (_lock) + { + membership = RefreshMembershipUnsafe(membership); + if (_aborted || _failure is not null) + { + break; + } + + for (var i = offset; i < offset + count; i++) + { + var item = work[i]; + if (IsCurrentUnsafe(item.Pending) && IsEligible(item.Notification, membership)) + { + notifications[sendCount++] = item.Notification; + } + } + } + + var accepted = false; + try + { + accepted = sendCount > 0 && _dispatch(notifications.AsSpan(0, sendCount)); + } + catch (Exception exception) + { + try + { + LogDispatchFailed(_logger, exception, _namespaceName); + } + catch (Exception loggingFailure) + { + Fail(new AggregateException("The root dispatch recovery diagnostic failed.", exception, loggingFailure)); + } + } + + lock (_lock) + { + for (var i = offset; i < offset + count; i++) + { + var item = work[i]; + var pending = item.Pending; + var admitted = accepted && IsEligible(item.Notification, membership) + && !_aborted && _failure is null && IsCurrentUnsafe(pending); + CompleteReceiptUnsafe(item, new(admitted, cohort.Started, cohort.Period)); + pending.InFlight = null; + if (!IsCurrentUnsafe(pending)) + { + continue; + } + + if (admitted && pending.Generation == item.Generation) + { + _pending.Remove(pending.Notification.Key); + } + else if (!admitted && !_aborted && _failure is null) + { + // Receipts are final even after partial child admission. Only bounded hints + // retry; publishers can use direct fallback without waiting on a congested child. + if (pending.Notification.Version == item.Notification.Version) + { + pending.Notification = pending.Notification with + { + Force = pending.Notification.Force || item.Notification.Force, + }; + } + + _cohort ??= new(membership, _timeProvider.GetTimestamp(), settings.Period); + if (pending.Node.List is null) + { + _ready.AddLast(pending.Node); + } + + _retryTimestamp = _timeProvider.GetTimestamp(); + _retryPeriod = settings.Period; + } + } + } + } + + lock (_lock) + { + _dispatching = false; + if (_pending.Count == 0) + { + CompleteDrainUnsafe(); + } + } + } + + private void CompleteReceiptUnsafe(PendingDispatch item, ReceiptOutcome outcome) + { + if (item.Receipt is null) + { + return; + } + + if (item.Producer is { } producer) + { + if (item.ProducerVersion >= producer.CompletedVersion) + { + producer.CompletedVersion = item.ProducerVersion; + producer.Outcome = outcome; + } + + if (ReferenceEquals(producer.InFlightReceipt, item.Receipt)) + { + producer.InFlightReceipt = null; + } + } + + item.Receipt.TrySetResult(outcome); + } + + private DisseminationMembershipSnapshot RefreshMembershipUnsafe(DisseminationMembershipSnapshot membership) + { + if (_membership is { } current + && (ReferenceEquals(current, membership) || membership.MembershipVersion.Value < current.MembershipVersion.Value)) + { + return current; + } + + _membership = membership; + foreach (var silo in _producers.Keys.Where(silo => !membership.ContainsMember(silo)).ToArray()) + { + _producers.Remove(silo); + } + + foreach (var pending in _pending.Values.Where(pending => !IsEligible(pending.Notification, membership)).ToArray()) + { + RemoveUnsafe(pending); + } + + return membership; + } + + private static bool IsEligible(KeyNotification notification, DisseminationMembershipSnapshot membership) => + notification.Key.Value is not SiloAddress silo || membership.ContainsMember(silo); + + private bool IsCurrentUnsafe(PendingKey pending) => + _pending.TryGetValue(pending.Notification.Key, out var current) && ReferenceEquals(current, pending); + + private void RemoveUnsafe(PendingKey pending) + { + var outcome = new ReceiptOutcome(false, _cohort?.Started ?? _timeProvider.GetTimestamp(), _cohort?.Period ?? TimeSpan.Zero); + CompleteReceiptUnsafe(new(pending, pending.Generation, pending.Notification, + pending.Receipt, pending.Producer, pending.Producer?.Version ?? 0), outcome); + pending.InFlight?.TrySetResult(outcome); + if (pending.Receipt is not null && pending.Notification.Key.Value is SiloAddress silo) + { + _cohort?.Contributors.Remove(silo); + } + + _pending.Remove(pending.Notification.Key); + if (pending.Node.List is not null) + { + _ready.Remove(pending.Node); + } + + if (_ready.Count == 0) + { + _cohort = null; + } + } + + private Settings GetSettings() + { + var options = _options.CurrentValue; + var namespaceOptions = _namespace.Options; + var period = _namespace.AggregationPeriod; + return new( + options.Enabled && namespaceOptions.Enabled, + Math.Max(1, namespaceOptions.MaxPendingItemCount), + Math.Max(1, options.MaxBatchItems), + TimeSpan.FromMilliseconds(Math.Clamp(period.TotalMilliseconds, 1, uint.MaxValue - 1))); + } + + private void ClearPendingUnsafe() + { + foreach (var pending in _pending.Values.ToArray()) + { + RemoveUnsafe(pending); + } + + _cohort = null; + _retryTimestamp = null; + _handoff = false; + } + + private void CompleteDrainUnsafe() + { + _retryTimestamp = null; + _handoff = false; + _drainCompletion?.TrySetResult(); + _drainCompletion = null; + } + + private void ThrowIfFailedUnsafe() + { + if (_failure is { } failure) + { + throw new InvalidOperationException($"The dissemination root batcher for {_namespaceName} has failed.", failure); + } + } + + private void ThrowIfAbortedUnsafe() + { + if (_aborted) + { + throw new OperationCanceledException("The dissemination root drain was canceled.", _abortToken); + } + } + + private void Wake() + { + lock (_lock) + { + if (_workerActive || _failure is not null || _aborted) + { + return; + } + + _workerActive = true; + } + + try + { + _timer.Wake(); + } + catch (Exception exception) + { + Fail(exception); + DisposeTimer(); + } + } + + private void DisposeTimer() + { + try + { + _timer.Dispose(); + } + catch (Exception exception) + { + Fail(exception); + } + } + + private void ReportRejection(int count, int limit) + { + try + { + LogAdmissionRejected(_logger, _namespaceName, count, limit); + for (var i = 0; i < count; i++) + { + try + { + DisseminationInstruments.OnQueueAdmissionRejected(_namespaceName); + } + catch (Exception exception) + { + LogDiagnosticFailed(_logger, exception, _namespaceName); + } + } + } + catch (Exception exception) + { + Fail(exception); + DisposeTimer(); + } + } + + private void Fail(Exception exception) + { + lock (_lock) + { + if (_failure is not null) + { + return; + } + + _failure = exception; + // Signals complete normally: a canceled caller must not leave an unobserved task fault. + ClearPendingUnsafe(); + _drainCompletion?.TrySetResult(); + } + + try + { + LogWorkerFailed(_logger, exception, _namespaceName); + } + catch + { + // Preserve the original failure for admission and drain callers even if logging fails. + } + } + + private sealed class PendingKey + { + public KeyNotification Notification; + public long Generation = 1; + public long AdmissionGeneration = 1; + public Producer? Producer; + public TaskCompletionSource? Receipt; + public TaskCompletionSource? InFlight; + public LinkedListNode Node { get; } + + public PendingKey(KeyNotification notification) + { + Notification = notification; + Node = new(this); + } + } + + private sealed class Producer + { + public long Version; + public long CompletedVersion; + public long InFlightVersion; + public ReceiptOutcome Outcome; + public TaskCompletionSource? InFlightReceipt; + } + + private sealed class Cohort(DisseminationMembershipSnapshot membership, long started, TimeSpan period) + { + public DisseminationMembershipSnapshot Membership { get; } = membership; + public long Started { get; } = started; + public TimeSpan Period { get; } = period; + public HashSet Contributors { get; } = []; + } + + private readonly record struct ReceiptOutcome(bool Accepted, long Started, TimeSpan Period); + private readonly record struct PendingDispatch( + PendingKey Pending, long Generation, KeyNotification Notification, + TaskCompletionSource? Receipt, Producer? Producer, long ProducerVersion); + private readonly record struct Settings(bool Enabled, int MaxPendingItemCount, int MaxBatchItems, TimeSpan Period); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Dissemination root dispatch for {Namespace} failed. Receipts are rejected and retained hints will retry after the publication period.")] + private static partial void LogDispatchFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Dissemination root for {Namespace} rejected {Count} new keys at its pending-key limit of {Limit}.")] + private static partial void LogAdmissionRejected(ILogger logger, DisseminationNamespace @namespace, int count, int limit); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Dissemination root diagnostic for {Namespace} failed.")] + private static partial void LogDiagnosticFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); + + [LoggerMessage(Level = LogLevel.Error, Message = "Dissemination root batcher for {Namespace} failed. Admission and drain callers will observe this failure.")] + private static partial void LogWorkerFailed(ILogger logger, Exception exception, DisseminationNamespace @namespace); +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs b/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs new file mode 100644 index 00000000000..fa143c95a95 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationSendGate.cs @@ -0,0 +1,151 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Dissemination; + +internal sealed class DisseminationSendGate(int maxConcurrency) +{ + private readonly object _lock = new(); + private readonly HashSet _activePeers = []; + private readonly LinkedList _requests = []; + private bool _stopped; + + public ValueTask AcquireAsync(SiloAddress peer, CancellationToken cancellationToken) => + AcquireAsync(peer, onQueued: null, cancellationToken); + + public ValueTask AcquireAsync(SiloAddress peer, Action? onQueued, CancellationToken cancellationToken) + { + Request request; + lock (_lock) + { + cancellationToken.ThrowIfCancellationRequested(); + ObjectDisposedException.ThrowIf(_stopped, this); + // Release fills available slots before unlocking, so any remaining queued peers are busy. + if (_activePeers.Count < maxConcurrency && _activePeers.Add(peer)) + { + return ValueTask.FromResult(new Lease(this, peer)); + } + + request = new(this, peer); + request.Node = _requests.AddLast(request); + } + + return WaitAsync(request, onQueued, cancellationToken); + } + + public bool TryAcquire(SiloAddress peer, [NotNullWhen(true)] out Lease? lease) + { + lock (_lock) + { + if (!_stopped && _activePeers.Count < maxConcurrency && _activePeers.Add(peer)) + { + lease = new(this, peer); + return true; + } + } + + lease = null; + return false; + } + + public void Stop() + { + lock (_lock) + { + _stopped = true; + while (_requests.First is { } node) + { + _requests.Remove(node); + node.Value.Completion.TrySetCanceled(); + } + } + } + + private static async ValueTask WaitAsync(Request request, Action? onQueued, CancellationToken cancellationToken) + { + using var registration = cancellationToken.UnsafeRegister( + static (state, token) => + { + var request = (Request)state!; + request.Owner.Cancel(request, token); + }, + request); + try + { + onQueued?.Invoke(); + } + catch + { + // A diagnostic failure must not leave an orphaned request or a concurrently granted lease. + request.Owner.Cancel(request, cancellationToken); + if (request.Completion.Task.IsCompletedSuccessfully) + { + request.Completion.Task.Result.Dispose(); + } + + throw; + } + + var lease = await request.Completion.Task.ConfigureAwait(false); + // A grant can win the lock just before cancellation removes its request. + if (cancellationToken.IsCancellationRequested) + { + lease.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + } + + return lease; + } + + private void Cancel(Request request, CancellationToken cancellationToken) + { + lock (_lock) + { + if (request.Node is { List: not null } node) + { + _requests.Remove(node); + request.Completion.TrySetCanceled(cancellationToken); + } + } + } + + private void Release(SiloAddress peer) + { + lock (_lock) + { + _activePeers.Remove(peer); + if (_stopped) + { + return; + } + + // FIFO among ready destinations: skip peers which already have an active local attempt. + var node = _requests.First; + while (node is not null && _activePeers.Count < maxConcurrency) + { + var next = node.Next; + if (_activePeers.Add(node.Value.Peer)) + { + _requests.Remove(node); + node.Value.Completion.SetResult(new Lease(this, node.Value.Peer)); + } + + node = next; + } + } + } + + private sealed class Request(DisseminationSendGate owner, SiloAddress peer) + { + public DisseminationSendGate Owner { get; } = owner; + public SiloAddress Peer { get; } = peer; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public LinkedListNode? Node { get; set; } + } + + internal sealed class Lease(DisseminationSendGate owner, SiloAddress peer) : IDisposable + { + private DisseminationSendGate? _owner = owner; + + public void Dispose() => Interlocked.Exchange(ref _owner, null)?.Release(peer); + } +} diff --git a/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs new file mode 100644 index 00000000000..ff2818f8a0d --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/DisseminationSystemTarget.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.Scheduler; + +namespace Orleans.Runtime.Dissemination; + +internal sealed partial class DisseminationSystemTarget : SystemTarget, IDisseminationSystemTarget, IDisseminationService, ILifecycleParticipant +{ + private readonly DisseminationProtocol _protocol; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly CancellationTokenSource _shutdownCts = new(); + private Task? _antiEntropyTask; + + public DisseminationSystemTarget( + ILocalSiloDetails localSiloDetails, + IInternalGrainFactory grainFactory, + DisseminationMembership membership, + IOptionsMonitor options, + IEnumerable disseminationNamespaces, + TimeProvider timeProvider, + ILogger logger, + ILogger broadcastQueueLogger, + SystemTargetShared shared) + : base(Constants.DisseminationSystemTargetType, shared) + { + _options = options; + _timeProvider = timeProvider; + _logger = logger; + _protocol = new DisseminationProtocol(localSiloDetails, grainFactory, membership, options, disseminationNamespaces, timeProvider, logger, broadcastQueueLogger); + shared.ActivationDirectory.RecordNewTarget(this); + } + + async ValueTask IDisseminationService.Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) => + await this.RunOrQueueTask(token => _protocol.Publish( + disseminationNamespace, + key, + version, + token).AsTask(), cancellationToken); + + async ValueTask IDisseminationService.PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) => + await this.RunOrQueueTask(token => _protocol.PublishAggregated( + disseminationNamespace, key, version, token).AsTask(), cancellationToken); + + IReadOnlyList IDisseminationService.GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) => + _protocol.GetUnconfirmedPeers(disseminationNamespace); + + Task IDisseminationSystemTarget.PushBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) => + _protocol.ReceiveBroadcast(batch, cancellationToken); + + Task IDisseminationSystemTarget.PublishAggregated( + DisseminationPublicationRequest request, + CancellationToken cancellationToken) => + _protocol.ReceivePublication(request, cancellationToken); + + async Task IDisseminationSystemTarget.ExchangeAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) => + await _protocol.ReceiveAntiEntropy(request, cancellationToken); + + void ILifecycleParticipant.Participate(ISiloLifecycle observer) + { + observer.Subscribe( + nameof(DisseminationSystemTarget), + ServiceLifecycleStage.RuntimeServices, + StartAsync, + StopAsync); + } + + private Task StartAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_antiEntropyTask is { IsCompleted: false }) + { + return Task.CompletedTask; + } + + var shutdownToken = _shutdownCts.Token; + _antiEntropyTask = this.RunOrQueueTask(() => RunAntiEntropyLoop(shutdownToken)); + _antiEntropyTask.Ignore(); + return Task.CompletedTask; + } + + private async Task StopAsync(CancellationToken cancellationToken) + { + await _shutdownCts.CancelAsync(); + try + { + await this.RunOrQueueTask(() => _protocol.StopAsync(cancellationToken)); + } + finally + { + // Always observe the loop and release the cancellation source, even if draining the protocol threw. + if (_antiEntropyTask is not null) + { + try + { + await _antiEntropyTask.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + // The loop observes owned shutdown; caller cancellation is reported after cleanup. + } + catch (Exception exception) + { + LogDebugAntiEntropyLoopFailed(_logger, exception); + } + + _antiEntropyTask = null; + } + + _shutdownCts.Dispose(); + } + + cancellationToken.ThrowIfCancellationRequested(); + } + + private async Task RunAntiEntropyLoop(CancellationToken cancellationToken) + { + using var wakeTimer = new WakeTimer(_timeProvider); + using var subscription = _options.OnChange((_, _) => wakeTimer.Wake()); + try + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var options = _options.CurrentValue; + wakeTimer.Change(options.Enabled ? options.Overlay.AntiEntropyInterval : Timeout.InfiniteTimeSpan); + if (!await wakeTimer.WaitAsync(cancellationToken)) + { + return; + } + + if (!_options.CurrentValue.Enabled) + { + continue; + } + + try + { + await _protocol.RunAntiEntropyRound(cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + LogDebugAntiEntropyLoopFailed(_logger, exception); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Expected during silo shutdown. + } + } + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Dissemination anti-entropy loop iteration failed.")] + private static partial void LogDebugAntiEntropyLoopFailed(ILogger logger, Exception exception); +} diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs new file mode 100644 index 00000000000..8d91dab466c --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/IDisseminationNamespace.cs @@ -0,0 +1,121 @@ +using Orleans.Configuration; + +namespace Orleans.Runtime.Dissemination; + +// A namespace owns current state, serialized payload caching, and repair construction. +internal interface IDisseminationNamespace +{ + DisseminationNamespace Name { get; } + + DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + + DisseminationRoutingMode RoutingMode => DisseminationRoutingMode.BroadcastTree; + + bool BroadcastsAreDeltas => false; + + TimeSpan AggregationPeriod => TimeSpan.FromSeconds(1); + + DisseminationNamespaceOptions Options { get; } + + IEnumerable Digests { get; } + + // Inventory maintenance needs identities, independently of versions and anti-entropy fingerprints. + IEnumerable Keys + { + get + { + foreach (var digest in Digests) + { + yield return digest.Key; + } + } + } + + long GetVersion(DisseminationKey key); + + DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request); + + DisseminationRepairResult CreateBroadcast( + in DisseminationRepairRequest request, + DisseminationBroadcastState? baseline) => CreateRepair(request); + + ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken); +} + +// Owned by the namespace and retained with an acknowledged peer/key until that knowledge is pruned. +internal abstract class DisseminationBroadcastState(long version) +{ + public long Version { get; } = version; +} + +internal enum DisseminationMembershipScope +{ + ActiveMembers, + AllMembers, +} + +internal enum DisseminationRoutingMode +{ + BroadcastTree, + AggregationTree, +} + +// A null FromVersion means no known peer baseline. Repairs materialize the current full value. +internal readonly struct DisseminationRepairRequest( + DisseminationKey key, + long? fromVersion, + int maxBatchBytes, + int maxPayloadBytes) +{ + public DisseminationKey Key { get; } = key; + + public long? FromVersion { get; } = fromVersion; + + public int MaxBatchBytes { get; } = maxBatchBytes; + + public int MaxPayloadBytes { get; } = maxPayloadBytes; +} + +// A produced result carries one full repair or broadcast update at Version. +internal readonly struct DisseminationRepairResult( + DisseminationRepairStatus status, + long version, + DisseminationValue value, + DisseminationBroadcastState? broadcastState = null) +{ + public DisseminationRepairStatus Status { get; } = status; + + public long Version { get; } = version; + + public DisseminationValue Value { get; } = value; + + public DisseminationBroadcastState? BroadcastState { get; } = broadcastState; + + public static DisseminationRepairResult Current(long version) => + new(DisseminationRepairStatus.Current, version, default); + + public static DisseminationRepairResult Produced( + DisseminationValue value, + DisseminationBroadcastState? broadcastState = null) => + new(DisseminationRepairStatus.Produced, value.ToVersion, value, broadcastState); + + public static DisseminationRepairResult Unavailable(long version) => + new(DisseminationRepairStatus.Unavailable, version, default); + + public static DisseminationRepairResult InsufficientCapacity(long version) => + new(DisseminationRepairStatus.InsufficientCapacity, version, default); +} + +internal enum DisseminationRepairStatus +{ + // The peer is already at or beyond the resolved version. + Current, + // Value contains the current repair or broadcast update. + Produced, + // The key has no current value. + Unavailable, + // The current value exceeds the supplied byte budget. + InsufficientCapacity, +} diff --git a/src/Orleans.Runtime/Dissemination/IDisseminationService.cs b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs new file mode 100644 index 00000000000..59b89290e28 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/IDisseminationService.cs @@ -0,0 +1,19 @@ +namespace Orleans.Runtime.Dissemination; + +// Publishing announces that a version is repairable; serialized payload ownership stays with the namespace. +internal interface IDisseminationService +{ + ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken); + + ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken); + + IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace); +} diff --git a/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs new file mode 100644 index 00000000000..0b6700c6dcc --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/MembershipDisseminationNamespace.cs @@ -0,0 +1,451 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; + +namespace Orleans.Runtime.Dissemination; + +// Broadcasts compare immutable snapshots; repair carries the complete current inventory. +internal sealed class MembershipDisseminationNamespace( + IMembershipManager membershipManager, + IOptionsMonitor options, + Serializer serializer) : IDisseminationNamespace +{ + private static readonly DisseminationKey[] MembershipKeys = [DisseminationKey.Default]; + private readonly object _cacheLock = new(); + private BroadcastState? _cachedState; + private byte[]? _cachedPayload; + private BroadcastState? _cachedBroadcastBase; + private DisseminationValue _cachedBroadcast; + + public DisseminationNamespace Name => DisseminationNamespaceNames.Membership; + + public DisseminationMembershipScope MembershipScope => DisseminationMembershipScope.AllMembers; + + public bool BroadcastsAreDeltas => true; + + public DisseminationNamespaceOptions Options => options.CurrentValue.Dissemination; + + public IEnumerable Keys => MembershipKeys; + + public ValueTask PublishAsync( + IDisseminationService disseminationService, + MembershipTableSnapshot snapshot, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return disseminationService.Publish( + this, + DisseminationKey.Default, + snapshot.Version.Value, + cancellationToken); + } + + public IEnumerable Digests + { + get + { + var snapshot = membershipManager.CurrentSnapshot; + // Version alone misses same-version liveness advances, so the digest fingerprints heartbeat state too. + yield return new DigestEntry( + DisseminationKey.Default, + snapshot.Version.Value, + GetFingerprint(snapshot)); + } + } + + public long GetVersion(DisseminationKey key) => + key == DisseminationKey.Default + ? membershipManager.CurrentSnapshot.Version.Value + : 0; + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (request.Key != DisseminationKey.Default) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + lock (_cacheLock) + { + // Re-read the owner under the cache lock: publication arguments can outlive the state they describe. + var state = GetCurrentState(); + var version = state.Version; + _cachedPayload ??= serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = state.Snapshot }); + return _cachedPayload.Length <= request.MaxPayloadBytes && _cachedPayload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(new DisseminationValue(DisseminationKey.Default, 0, version, _cachedPayload)) + : DisseminationRepairResult.InsufficientCapacity(version); + } + } + + public DisseminationRepairResult CreateBroadcast( + in DisseminationRepairRequest request, + DisseminationBroadcastState? baseline) + { + if (request.Key != DisseminationKey.Default) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + lock (_cacheLock) + { + var current = GetCurrentState(); + if (current.Version <= 0) + { + return DisseminationRepairResult.Unavailable(current.Version); + } + + if (request.FromVersion > current.Version) + { + return DisseminationRepairResult.Current(current.Version); + } + + // An empty current-view delta establishes a comparison baseline. Missing views use full repair. + var previous = baseline is null ? current : (BroadcastState)baseline; + if (previous.Version > current.Version) + { + return DisseminationRepairResult.Current(current.Version); + } + + if (!ReferenceEquals(_cachedBroadcastBase, previous)) + { + var updated = ImmutableArray.CreateBuilder(); + foreach (var (silo, entry) in current.Snapshot.Entries) + { + if (!previous.Snapshot.Entries.TryGetValue(silo, out var old) + || !MembershipEntriesEqual(old, entry)) + { + updated.Add(entry); + } + } + + var removed = ImmutableArray.CreateBuilder(); + foreach (var silo in previous.Snapshot.Entries.Keys) + { + if (!current.Snapshot.Entries.ContainsKey(silo)) + { + removed.Add(silo); + } + } + + var delta = new MembershipTableSnapshotDelta( + previous.Snapshot.Version, current.Snapshot.Version, + updated.ToImmutable(), removed.ToImmutable()); + _cachedBroadcast = new( + DisseminationKey.Default, previous.Version, current.Version, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Delta = delta })); + _cachedBroadcastBase = previous; + } + + return _cachedBroadcast.Payload.Length <= request.MaxPayloadBytes + && _cachedBroadcast.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(_cachedBroadcast, current) + : DisseminationRepairResult.InsufficientCapacity(current.Version); + } + } + + private BroadcastState GetCurrentState() + { + var snapshot = membershipManager.CurrentSnapshot; + if (_cachedState is null || !MembershipSnapshotsEqual(_cachedState.Snapshot, snapshot)) + { + _cachedState = new(snapshot); + _cachedPayload = null; + _cachedBroadcastBase = null; + _cachedBroadcast = default; + } + + return _cachedState; + } + + public async ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (value.Key != DisseminationKey.Default) + { + return DisseminationApplyResult.Rejected; + } + + if (serializer.Deserialize(value.Payload) is not { } update) + { + return DisseminationApplyResult.Rejected; + } + + var previous = membershipManager.CurrentSnapshot; + MembershipTableSnapshot? snapshot; + IEnumerable heartbeats; + IEnumerable removed; + if (update.Delta is { } delta) + { + if (update.Snapshot is not null + || value.FromVersion != delta.BaseVersion.Value + || value.ToVersion != delta.Version.Value + || delta.BaseVersion.Value <= 0 + || delta.Version < delta.BaseVersion + || delta.UpdatedEntries.IsDefault || delta.RemovedSilos.IsDefault) + { + return DisseminationApplyResult.Rejected; + } + + if (delta.Version < previous.Version) + { + return DisseminationApplyResult.Obsolete; + } + + if (previous.Version != delta.BaseVersion && previous.Version != delta.Version) + { + return DisseminationApplyResult.Rejected; + } + + snapshot = ApplyDelta(previous, delta); + heartbeats = delta.UpdatedEntries; + removed = delta.RemovedSilos; + } + else if (update.Snapshot is { } full + && value.FromVersion == 0 + && value.ToVersion == full.Version.Value) + { + snapshot = full.Version == previous.Version ? MergeSameVersion(previous, full) : full; + heartbeats = full.Entries.Values; + removed = full.Version == previous.Version + ? previous.Entries.Keys.Where(silo => !full.Entries.ContainsKey(silo)) + : []; + } + else + { + return DisseminationApplyResult.Rejected; + } + + if (snapshot is null) + { + return DisseminationApplyResult.Rejected; + } + + if (MembershipSnapshotsEqual(previous, snapshot) && CoversChanges(previous, heartbeats, removed)) + { + return DisseminationApplyResult.Duplicate; + } + + await membershipManager.ProcessGossipSnapshot(snapshot, cancellationToken); + var current = membershipManager.CurrentSnapshot; + if (snapshot.Version < current.Version) + { + return DisseminationApplyResult.Obsolete; + } + + // A concurrent authority refresh can publish while the gossip call waits. Confirm the requested + // effects, rather than treating any unrelated owner change as acceptance of this update. + if (current.Version != snapshot.Version || !CoversChanges(current, heartbeats, removed)) + { + return DisseminationApplyResult.Rejected; + } + + return MembershipSnapshotsEqual(previous, current) + ? DisseminationApplyResult.Duplicate + : DisseminationApplyResult.Applied; + } + + private static MembershipTableSnapshot? ApplyDelta( + MembershipTableSnapshot current, + MembershipTableSnapshotDelta delta) + { + var entries = current.Entries.ToBuilder(); + var touched = new HashSet(); + var sameVersion = current.Version == delta.Version; + foreach (var entry in delta.UpdatedEntries) + { + if (entry?.SiloAddress is not { } silo || !touched.Add(silo)) + { + return null; + } + + if (sameVersion) + { + if (entries.TryGetValue(silo, out var existing)) + { + entries[silo] = MergeHeartbeat(existing, entry); + } + else if (entry.Status != SiloStatus.Dead) + { + return null; + } + } + else + { + entries[silo] = entries.TryGetValue(silo, out var existing) + ? MergeHeartbeat(entry, existing) + : entry; + } + } + + foreach (var silo in delta.RemovedSilos) + { + if (silo is null || !touched.Add(silo) + || sameVersion && entries.TryGetValue(silo, out var existing) && existing.Status != SiloStatus.Dead) + { + return null; + } + + entries.Remove(silo); + } + + return new(delta.Version, entries.ToImmutable()); + } + + private static MembershipTableSnapshot? MergeSameVersion( + MembershipTableSnapshot current, + MembershipTableSnapshot incoming) + { + var entries = current.Entries.ToBuilder(); + foreach (var (silo, entry) in current.Entries) + { + if (incoming.Entries.TryGetValue(silo, out var update)) + { + entries[silo] = MergeHeartbeat(entry, update); + } + else if (entry.Status != SiloStatus.Dead) + { + return null; + } + else + { + entries.Remove(silo); + } + } + + foreach (var (silo, entry) in incoming.Entries) + { + if (entry.Status != SiloStatus.Dead && !current.Entries.ContainsKey(silo)) + { + return null; + } + } + + return new(current.Version, entries.ToImmutable()); + } + + private static MembershipEntry MergeHeartbeat(MembershipEntry current, MembershipEntry incoming) => + incoming.IAmAliveTime > current.IAmAliveTime + ? current.WithIAmAliveTime(incoming.IAmAliveTime) + : current; + + private static bool CoversChanges( + MembershipTableSnapshot current, + IEnumerable updated, + IEnumerable removed) + { + foreach (var entry in updated) + { + if (current.Entries.TryGetValue(entry.SiloAddress, out var existing)) + { + if (existing.IAmAliveTime < entry.IAmAliveTime) + { + return false; + } + } + else if (entry.Status != SiloStatus.Dead) + { + return false; + } + } + + return !removed.Any(current.Entries.ContainsKey); + } + + private sealed class BroadcastState(MembershipTableSnapshot snapshot) + : DisseminationBroadcastState(snapshot.Version.Value) + { + public MembershipTableSnapshot Snapshot { get; } = snapshot; + } + + private static long GetFingerprint(MembershipTableSnapshot snapshot) + { + // Keep the hash deterministic across hosts and focused on state which can change without a version bump. + const ulong offset = 14695981039346656037; + const ulong prime = 1099511628211; + var hash = offset; + foreach (var entry in snapshot.Entries.OrderBy(static entry => entry.Key)) + { + hash = unchecked((hash ^ (uint)entry.Key.GetConsistentHashCode()) * prime); + hash = unchecked((hash ^ (ulong)entry.Value.IAmAliveTime.Ticks) * prime); + } + + return unchecked((long)hash); + } + + // Comparing cross-version snapshots identifies the entries to include in a sparse broadcast. + private static bool MembershipEntriesEqual(MembershipEntry left, MembershipEntry right) => + ReferenceEquals(left, right) + || left.SiloAddress.Equals(right.SiloAddress) + && left.Status == right.Status + && left.ProxyPort == right.ProxyPort + && string.Equals(left.HostName, right.HostName, StringComparison.Ordinal) + && string.Equals(left.SiloName, right.SiloName, StringComparison.Ordinal) + && string.Equals(left.RoleName, right.RoleName, StringComparison.Ordinal) + && left.UpdateZone == right.UpdateZone + && left.FaultZone == right.FaultZone + && left.StartTime == right.StartTime + && left.IAmAliveTime == right.IAmAliveTime + && (ReferenceEquals(left.SuspectTimes, right.SuspectTimes) + || left.SuspectTimes is not null && right.SuspectTimes is not null + && left.SuspectTimes.SequenceEqual(right.SuspectTimes)); + + private static bool MembershipSnapshotsEqual( + MembershipTableSnapshot left, + MembershipTableSnapshot right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Version != right.Version || left.Entries.Count != right.Entries.Count) + { + return false; + } + + foreach (var (siloAddress, entry) in left.Entries) + { + if (!right.Entries.TryGetValue(siloAddress, out var other) + || entry.IAmAliveTime != other.IAmAliveTime) + { + return false; + } + } + + return true; + } +} + +[GenerateSerializer, Immutable] +internal sealed class MembershipTableSnapshotUpdate +{ + [Id(0)] + public MembershipTableSnapshot? Snapshot { get; init; } + + [Id(1)] + public MembershipTableSnapshotDelta? Delta { get; init; } +} + +[GenerateSerializer, Immutable] +internal sealed class MembershipTableSnapshotDelta( + MembershipVersion baseVersion, + MembershipVersion version, + ImmutableArray updatedEntries, + ImmutableArray removedSilos) +{ + [Id(0)] + public MembershipVersion BaseVersion { get; } = baseVersion; + + [Id(1)] + public MembershipVersion Version { get; } = version; + + [Id(2)] + public ImmutableArray UpdatedEntries { get; } = updatedEntries; + + [Id(3)] + public ImmutableArray RemovedSilos { get; } = removedSilos; +} diff --git a/src/Orleans.Runtime/Dissemination/WakeTimer.cs b/src/Orleans.Runtime/Dissemination/WakeTimer.cs new file mode 100644 index 00000000000..51912cb4fd3 --- /dev/null +++ b/src/Orleans.Runtime/Dissemination/WakeTimer.cs @@ -0,0 +1,220 @@ +namespace Orleans.Runtime.Dissemination; + +/// +/// A thread-safe, wakeable, one-shot timer. +/// +/// +/// The timer has one reusable underlying . Call to arm or re-arm it +/// with a due time, call to complete the current or next wait immediately, and call +/// to wait until either the due time elapses or the timer is explicitly woken. +/// +internal sealed class WakeTimer : IDisposable +{ + private readonly object _lock = new(); + private readonly TimeProvider _timeProvider; + private readonly ITimer _timer; + private TaskCompletionSource? _waiter; + private long _armedAtTimestamp; + private TimeSpan _dueTime; + private bool _armed; + private bool _signaled; + private bool _disposed; + + /// + /// Initializes a new instance of the class. + /// + /// The time provider used to create the underlying timer. + public WakeTimer(TimeProvider timeProvider) + { + _timeProvider = timeProvider; + _timer = timeProvider.CreateTimer( + static state => ((WakeTimer)state!).OnTimer(), + this, + Timeout.InfiniteTimeSpan, + Timeout.InfiniteTimeSpan); + } + + /// + /// Arms or re-arms the timer to fire once after the provided due time. + /// + /// + /// The delay before the timer fires. schedules an immediate wake, and + /// prevents the timer from firing until changed or woken. + /// + /// + /// The timer always uses an infinite period, so every call is a one-shot schedule. Calling this method while + /// another caller is waiting causes that waiter to observe the new due time rather than completing immediately. + /// + public void Change(TimeSpan dueTime) + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _armed = dueTime != Timeout.InfiniteTimeSpan; + _armedAtTimestamp = _armed ? _timeProvider.GetTimestamp() : 0; + _dueTime = dueTime; + _timer.Change(dueTime, Timeout.InfiniteTimeSpan); + } + } + + /// + /// Disarms the timer and consumes any pending wake, leaving an active waiter in place. + /// + public void Reset() + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _armed = false; + _signaled = false; + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + } + } + + /// + /// Immediately completes the current wait, or causes the next wait to complete immediately. + /// + public void Wake() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed) + { + return; + } + + waiter = CompleteWaitUnsafe(); + } + + waiter?.TrySetResult(true); + } + + /// + /// Waits until the timer fires, is explicitly woken, or is disposed. + /// + /// A token which cancels this wait without disarming the timer. + /// + /// when the due time elapsed or was called; + /// when the timer was disposed. + /// + /// + /// Only one waiter is supported at a time. If the timer has already fired or been woken before this method is + /// called, the method returns immediately. Cancelling a wait does not disarm the timer, + /// so a later waiter can still observe the scheduled wakeup. + /// + public async ValueTask WaitAsync(CancellationToken cancellationToken) + { + TaskCompletionSource waiter; + lock (_lock) + { + if (_disposed) + { + return false; + } + + if (_signaled) + { + _signaled = false; + return true; + } + + if (_waiter is not null) + { + throw new InvalidOperationException("Only one waiter can wait on a WakeTimer at a time."); + } + + waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiter = waiter; + } + + using var registration = cancellationToken.UnsafeRegister( + static state => + { + var (timer, waiter, token) = + ((WakeTimer Timer, TaskCompletionSource Waiter, CancellationToken Token))state!; + timer.CancelWait(waiter, token); + }, + (this, waiter, cancellationToken)); + + return await waiter.Task; + } + + /// + /// Disposes the timer, completing any current waiter with . + /// + public void Dispose() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + waiter = _waiter; + _waiter = null; + _armed = false; + } + + waiter?.TrySetResult(false); + _timer.Dispose(); + } + + private TaskCompletionSource? CompleteWaitUnsafe() + { + var waiter = _waiter; + _waiter = null; + _armed = false; + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _signaled = waiter is null; + return waiter; + } + + private void OnTimer() + { + TaskCompletionSource? waiter; + lock (_lock) + { + if (_disposed || !_armed) + { + return; + } + + var remaining = _dueTime - _timeProvider.GetElapsedTime(_armedAtTimestamp); + if (remaining > TimeSpan.Zero) + { + _timer.Change(remaining, Timeout.InfiniteTimeSpan); + return; + } + + waiter = CompleteWaitUnsafe(); + } + + waiter?.TrySetResult(true); + } + + private void CancelWait(TaskCompletionSource waiter, CancellationToken cancellationToken) + { + lock (_lock) + { + if (!ReferenceEquals(_waiter, waiter)) + { + return; + } + + _waiter = null; + waiter.TrySetCanceled(cancellationToken); + } + } +} diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index df1bc1a6e4b..b9387444b38 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -20,6 +20,7 @@ using Orleans.Runtime; using Orleans.Runtime.Configuration; using Orleans.Runtime.ConsistentRing; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.MembershipService; using Orleans.Runtime.Messaging; @@ -172,9 +173,16 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddSingleton(); services.AddFromExisting, DeploymentLoadPublisher>(); + services.AddSingleton(); + services.AddFromExisting(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, DisseminationSystemTarget>(); + services.TryAddSingleton(); services.AddFromExisting(); services.AddFromExisting, IMembershipManager>(); @@ -182,6 +190,8 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddFromExisting(); services.AddFromExisting, MembershipSystemTarget>(); services.AddSingleton(); + services.AddSingleton(); + services.AddFromExisting(); services.AddSingleton(); services.AddSingleton(); services.TryAddFromExisting(); @@ -350,6 +360,7 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.ConfigureFormatter(); services.ConfigureFormatter(); services.ConfigureFormatter(); + services.ConfigureFormatter(); // This validator needs to construct the IMembershipOracle and the IMembershipTable // so move it in the end so other validator are called first @@ -358,6 +369,9 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.AddTransient(); services.AddTransient(); services.AddTransient, SiloMessagingOptionsValidator>(); + services.AddTransient, DisseminationOptionsValidator>(); + services.AddTransient, DeploymentLoadPublisherOptionsValidator>(); + services.AddTransient, ClusterMembershipOptionsDisseminationValidator>(); services.AddTransient>(static sp => sp.GetRequiredService>()); // Enable hosted client. diff --git a/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs b/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs index 8d6c3749deb..2b39c0535a6 100644 --- a/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs +++ b/src/Orleans.Runtime/MembershipService/MembershipGossiper.cs @@ -4,14 +4,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Orleans.Runtime.Dissemination; namespace Orleans.Runtime.MembershipService; -internal partial class MembershipGossiper(IServiceProvider serviceProvider, ILogger logger) : IMembershipGossiper +internal partial class MembershipGossiper( + IServiceProvider serviceProvider, + ILocalSiloDetails localSiloDetails, + ILogger logger) : IMembershipGossiper { private MembershipSystemTarget? _membershipSystemTarget; - public Task GossipToRemoteSilos( + public async Task GossipToRemoteSilos( List gossipPartners, MembershipTableSnapshot snapshot, SiloAddress updatedSilo, @@ -19,12 +23,44 @@ public Task GossipToRemoteSilos( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (gossipPartners.Count == 0) return Task.CompletedTask; + if (gossipPartners.Count == 0) return; LogDebugGossipingStatusToPartners(logger, updatedSilo, updatedStatus, gossipPartners.Count); + // Direct gossip starts first and owns shutdown-critical delivery. var systemTarget = _membershipSystemTarget ??= serviceProvider.GetRequiredService(); - return systemTarget.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); + var directGossip = systemTarget.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); + if (snapshot.Entries.TryGetValue(localSiloDetails.SiloAddress, out var localEntry) + && localEntry.Status is SiloStatus.Joining or SiloStatus.Active or SiloStatus.ShuttingDown or SiloStatus.Stopping) + { + await Task.WhenAll(directGossip, TryGossipViaDissemination(snapshot, cancellationToken)); + } + else + { + await directGossip; + } + } + + private async Task TryGossipViaDissemination(MembershipTableSnapshot snapshot, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var disseminationNamespace = serviceProvider.GetRequiredService(); + if (disseminationNamespace.Options.Enabled) + { + var dissemination = serviceProvider.GetRequiredService(); + await disseminationNamespace.PublishAsync(dissemination, snapshot, cancellationToken); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + LogDebugMembershipDisseminationFailed(logger, exception); + } } [LoggerMessage( @@ -32,4 +68,9 @@ public Task GossipToRemoteSilos( Message = "Gossiping {Silo} status {Status} to {NumPartners} partners" )] private static partial void LogDebugGossipingStatusToPartners(ILogger logger, SiloAddress silo, SiloStatus status, int numPartners); + + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Membership dissemination failed. Direct membership gossip continues delivery.")] + private static partial void LogDebugMembershipDisseminationFailed(ILogger logger, Exception exception); } diff --git a/src/Orleans.Runtime/OrleansContracts.txt b/src/Orleans.Runtime/OrleansContracts.txt index 3dd8082bb7d..073b71e1154 100644 --- a/src/Orleans.Runtime/OrleansContracts.txt +++ b/src/Orleans.Runtime/OrleansContracts.txt @@ -23,6 +23,8 @@ class [GrainType("developmentleaseprovider")] Orleans.Runtime.Development.Develo interface [GrainInterfaceType("Orleans.Runtime.Development.IDevelopmentLeaseProviderGrain")] Orleans.Runtime.Development.IDevelopmentLeaseProviderGrain [Version(0)] 847FCE12: Reset(System.Threading.CancellationToken) -> Task +class [GrainType("disseminationsystemtarget")] Orleans.Runtime.Dissemination.DisseminationSystemTarget + class [GrainType("graincallcancellationmanager")] Orleans.Runtime.GrainCallCancellationManager class [GrainType("clientdirectory")] Orleans.Runtime.GrainDirectory.ClientDirectory diff --git a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs index a3223cc86b7..b49f969b139 100644 --- a/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs +++ b/src/Orleans.Runtime/Placement/DeploymentLoadPublisher.cs @@ -4,10 +4,12 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Runtime.Diagnostics; +using Orleans.Runtime.Dissemination; using Orleans.Internal; using Orleans.Runtime.Scheduler; using Orleans.Statistics; @@ -26,6 +28,7 @@ internal sealed partial class DeploymentLoadPublisher : SystemTarget, IDeploymen private readonly IActivationWorkingSet _activationWorkingSet; private readonly IEnvironmentStatisticsProvider _environmentStatisticsProvider; private readonly IOptions _loadSheddingOptions; + private readonly IServiceProvider _serviceProvider; private readonly ConcurrentDictionary _periodicStats; private readonly TimeSpan _statisticsRefreshTime; private readonly List _siloStatisticsChangeListeners; @@ -33,6 +36,7 @@ internal sealed partial class DeploymentLoadPublisher : SystemTarget, IDeploymen private long _lastUpdateDateTimeTicks; private IGrainTimer? _publishTimer; + private Task? _publicationTask; public ConcurrentDictionary PeriodicStatistics => _periodicStats; @@ -48,6 +52,7 @@ public DeploymentLoadPublisher( IActivationWorkingSet activationWorkingSet, IEnvironmentStatisticsProvider environmentStatisticsProvider, IOptions loadSheddingOptions, + IServiceProvider serviceProvider, SystemTargetShared shared) : base(Constants.DeploymentLoadPublisherSystemTargetType, shared) { @@ -59,6 +64,7 @@ public DeploymentLoadPublisher( _activationWorkingSet = activationWorkingSet; _environmentStatisticsProvider = environmentStatisticsProvider; _loadSheddingOptions = loadSheddingOptions; + _serviceProvider = serviceProvider; _statisticsRefreshTime = options.Value.DeploymentLoadPublisherRefreshTime; _periodicStats = new ConcurrentDictionary(); _siloStatisticsChangeListeners = new List(); @@ -79,19 +85,58 @@ private async Task StartAsync(CancellationToken cancellationToken) await this.RunOrQueueTask(() => { cancellationToken.ThrowIfCancellationRequested(); - _publishTimer = RegisterGrainTimer(PublishStatistics, randomTimerOffset, _statisticsRefreshTime); + _publishTimer = RegisterGrainTimer(PublishStatisticsOnTimer, randomTimerOffset, _statisticsRefreshTime); return Task.CompletedTask; }); } await RefreshClusterStatistics(cancellationToken); - await PublishStatistics(cancellationToken); + await this.RunOrQueueTask(async token => + { + await PublishStatistics(token); + return true; + }, cancellationToken); LogDebugStartedDeploymentLoadPublisher(_logger); } - internal async Task PublishStatistics(CancellationToken cancellationToken) + internal Task PublishStatistics(CancellationToken cancellationToken) => + PublishStatisticsAndGetReceipt(cancellationToken); + + private async Task PublishStatisticsOnTimer(CancellationToken cancellationToken) + { + var publication = await PublishStatisticsAndGetReceipt(cancellationToken); + if (publication.Receipt.Accepted && !cancellationToken.IsCancellationRequested) + { + // Change during a grain timer callback applies its due time after the callback + // completes. Account for direct delivery and notifications since receipt arrival, + // rather than adding another full period after the root's cohort wait. + // Startup and explicit publications must not change a paused/disposed timer. + var remaining = publication.Receipt.NextPublicationDelay + - publication.TimeProvider!.GetElapsedTime(publication.ReceiptTimestamp); + _publishTimer?.Change( + remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero, + _statisticsRefreshTime); + } + } + + private async Task PublishStatisticsAndGetReceipt(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + // Startup and the interleaving timer can overlap. Join the pending sample instead of + // submitting another source publication while its cohort is still open. + if (_publicationTask is { IsCompleted: false } pending) + { + return await pending.WaitAsync(cancellationToken); + } + + // Native operations own this caller's cancellation and may complete successfully + // after observing it. Only joining callers need an independently cancellable wait. + return await (_publicationTask = PublishStatisticsCore(cancellationToken)); + } + + private async Task PublishStatisticsCore(CancellationToken cancellationToken) + { + StatisticsPublication publication = default; try { LogTracePublishStatistics(_logger); @@ -112,24 +157,30 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) DeploymentLoadPublisherEvents.EmitPublished(_siloDetails.SiloAddress, myStats); // Inform other cluster members about our refreshed statistics. - var members = _siloStatusOracle.GetApproximateSiloStatuses(true).Keys; - var tasks = new List(members.Count); - foreach (var siloAddress in members) + var members = _siloStatusOracle.GetApproximateSiloStatuses(true).Keys.ToArray(); + IReadOnlyCollection directRecipients = members; + publication = await PublishStatisticsViaDissemination(myStats, cancellationToken); + if (publication.Receipt.Accepted) { - // No need to make a grain call to ourselves. - if (siloAddress.Equals(_siloDetails.SiloAddress)) - { - continue; - } - try { - var deploymentLoadPublisher = _grainFactory.GetSystemTarget(Constants.DeploymentLoadPublisherSystemTargetType, siloAddress); - tasks.Add(deploymentLoadPublisher.UpdateRuntimeStatistics(_siloDetails.SiloAddress, myStats, cancellationToken)); + var dissemination = _serviceProvider.GetRequiredService(); + var disseminationNamespace = _serviceProvider.GetRequiredService(); + var unconfirmedPeers = dissemination.GetUnconfirmedPeers(disseminationNamespace); + if (unconfirmedPeers.Count == 0) + { + directRecipients = []; + } + else + { + // An unsupported intermediate node can separate otherwise capable tree participants. + var unconfirmed = unconfirmedPeers.ToHashSet(); + directRecipients = members.Any(unconfirmed.Contains) ? members : []; + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - tasks.Add(Task.FromCanceled(cancellationToken)); + throw; } catch (Exception exception) { @@ -137,17 +188,20 @@ internal async Task PublishStatistics(CancellationToken cancellationToken) } } - await Task.WhenAll(tasks); + await PublishStatisticsDirectly(myStats, directRecipients, cancellationToken); DeploymentLoadPublisherEvents.EmitClusterRefreshed(_siloDetails.SiloAddress, _periodicStats); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + cancellationToken.ThrowIfCancellationRequested(); throw; } catch (Exception exc) { LogWarningRuntimeStatisticsUpdateFailure2(_logger, exc); } + + return publication; } public Task UpdateRuntimeStatistics( @@ -160,23 +214,144 @@ public Task UpdateRuntimeStatistics( return Task.CompletedTask; } - private void UpdateRuntimeStatisticsInternal(SiloAddress siloAddress, SiloRuntimeStatistics siloStats) + internal Task ApplyDisseminatedRuntimeStatisticsAsync( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + CancellationToken cancellationToken) => + this.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(UpdateRuntimeStatisticsInternal(siloAddress, siloStats, isDisseminated: true)); + }, + cancellationToken); + + internal bool IsRuntimeStatisticsObsolete(SiloAddress siloAddress, long timestampTicks) => + _siloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active + || (_periodicStats.TryGetValue(siloAddress, out var old) && old.DateTime.Ticks > timestampTicks); + + internal Dictionary GetActiveSiloStatusesForStatisticsDigest() => + _siloStatusOracle.GetApproximateSiloStatuses(onlyActive: true); + + private async Task PublishStatisticsViaDissemination( + SiloRuntimeStatistics myStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_statisticsRefreshTime <= TimeSpan.Zero) + { + return default; + } + + var disseminationNamespace = _serviceProvider.GetRequiredService(); + if (!disseminationNamespace.Options.Enabled) + { + return default; + } + + var timeProvider = _serviceProvider.GetRequiredService(); + // A cohort may intentionally stay open for P. Allow another P for transport and + // distribution admission; the receipt's delay, not this budget, controls sampling. + // For long periods, the ordinary RPC timeout may expire first and take the same + // logged direct-publication fallback without changing cluster-wide RPC settings. + var receiptTimeout = TimeSpan.FromMilliseconds(Math.Min(disseminationNamespace.AggregationPeriod.TotalMilliseconds * 2, uint.MaxValue - 1)); + using var timeoutCancellation = new CancellationTokenSource(receiptTimeout, timeProvider); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellation.Token); + try + { + var dissemination = _serviceProvider.GetRequiredService(); + var receipt = await dissemination.PublishAggregated( + disseminationNamespace, + _siloDetails.SiloAddress, + myStats.DateTime.Ticks, + cancellation.Token); + var receiptTimestamp = timeProvider.GetTimestamp(); + return new(receipt, timeProvider, receiptTimestamp); + } + catch (OperationCanceledException) when ( + timeoutCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + LogDebugRuntimeStatisticsDisseminationTimedOut(_logger, receiptTimeout); + return default; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + catch (Exception exception) + { + LogWarningRuntimeStatisticsUpdateFailure1(_logger, exception); + return default; + } + } + + private readonly record struct StatisticsPublication( + DisseminationPublicationReceipt Receipt, + TimeProvider? TimeProvider, + long ReceiptTimestamp); + + private async Task PublishStatisticsDirectly( + SiloRuntimeStatistics myStats, + IReadOnlyCollection members, + CancellationToken cancellationToken) + { + var tasks = new List(members.Count); + foreach (var siloAddress in members) + { + if (siloAddress.Equals(_siloDetails.SiloAddress)) + { + continue; + } + + try + { + var deploymentLoadPublisher = _grainFactory.GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, siloAddress); + tasks.Add(deploymentLoadPublisher.UpdateRuntimeStatistics(_siloDetails.SiloAddress, myStats, cancellationToken)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + tasks.Add(Task.FromCanceled(cancellationToken)); + } + catch (Exception exception) + { + LogWarningRuntimeStatisticsUpdateFailure1(_logger, exception); + } + } + + await Task.WhenAll(tasks); + } + + private DisseminationApplyResult UpdateRuntimeStatisticsInternal( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + bool isDisseminated = false) { LogTraceUpdateRuntimeStatistics(_logger, siloAddress); if (_siloStatusOracle.GetApproximateSiloStatus(siloAddress) != SiloStatus.Active) { - return; + return DisseminationApplyResult.Rejected; } // Take only if newer. - if (_periodicStats.TryGetValue(siloAddress, out var old) && old.DateTime > siloStats.DateTime) + if (_periodicStats.TryGetValue(siloAddress, out var old)) { - return; + if (old.DateTime > siloStats.DateTime) + { + return DisseminationApplyResult.Obsolete; + } + + if (isDisseminated && old.DateTime == siloStats.DateTime) + { + return DisseminationApplyResult.Duplicate; + } } _periodicStats[siloAddress] = siloStats; NotifyAllStatisticsChangeEventsSubscribers(siloAddress, siloStats); DeploymentLoadPublisherEvents.EmitReceived(siloAddress, _siloDetails.SiloAddress, siloStats); + return DisseminationApplyResult.Applied; } internal async Task RefreshClusterStatistics(CancellationToken cancellationToken) @@ -309,6 +484,11 @@ Task DisposePublishTimer(CancellationToken ct) )] private static partial void LogDebugStartedDeploymentLoadPublisher(ILogger logger); + [LoggerMessage( + Level = LogLevel.Debug, + Message = "Deployment load dissemination did not accept the update within {Timeout}. Publishing directly.")] + private static partial void LogDebugRuntimeStatisticsDisseminationTimedOut(ILogger logger, TimeSpan timeout); + [LoggerMessage( Level = LogLevel.Trace, Message = "PublishStatistics" diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index bc9c86ea2a8..d2906789dfa 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -467,6 +467,8 @@ public partial class ClusterMembershipOptions public System.TimeSpan DefunctSiloExpiration { get { throw null; } set { } } + public DisseminationNamespaceOptions Dissemination { get { throw null; } set { } } + public bool EnableConnectionLivenessCheck { get { throw null; } set { } } public bool EnableIndirectProbes { get { throw null; } set { } } @@ -532,6 +534,53 @@ public partial class ConnectionOptions public Runtime.Messaging.NetworkProtocolVersion ProtocolVersion { get { throw null; } set { } } } + public sealed partial class DisseminationNamespaceOptions + { + public bool Enabled { get { throw null; } set { } } + + public System.TimeSpan ExpectedUpdateCadence { get { throw null; } set { } } + + public int MaxPayloadBytes { get { throw null; } set { } } + + public int MaxPendingItemCount { get { throw null; } set { } } + + public System.TimeSpan StaleItemTtl { get { throw null; } set { } } + } + + public sealed partial class DisseminationOptions + { + public bool Enabled { get { throw null; } set { } } + + public int MaxBatchBytes { get { throw null; } set { } } + + public int MaxBatchItems { get { throw null; } set { } } + + public int MaxConcurrentSends { get { throw null; } set { } } + + public DisseminationOverlayOptions Overlay { get { throw null; } set { } } + } + + public sealed partial class DisseminationOverlayOptions + { + public int AggregationFanOutFactor { get { throw null; } set { } } + + public System.TimeSpan AntiEntropyInterval { get { throw null; } set { } } + + public int AntiEntropyPeerCount { get { throw null; } set { } } + + public System.Func? FanOutFactor { get { throw null; } set { } } + + public int MaxAntiEntropyBatchBytes { get { throw null; } set { } } + + public int MaxAntiEntropyBatchItems { get { throw null; } set { } } + + public int MaxFanOutFactor { get { throw null; } set { } } + + public int MinFanOutFactor { get { throw null; } set { } } + + public int TargetHopCount { get { throw null; } set { } } + } + public partial class GatewayOptions { public const int DEFAULT_PREFERED_GATEWAY_INDEX = -1; diff --git a/src/api/Orleans.Runtime/Orleans.Runtime.cs b/src/api/Orleans.Runtime/Orleans.Runtime.cs index 66d0945dad8..5cb045453b6 100644 --- a/src/api/Orleans.Runtime/Orleans.Runtime.cs +++ b/src/api/Orleans.Runtime/Orleans.Runtime.cs @@ -125,6 +125,8 @@ public partial class DeploymentLoadPublisherOptions { public static readonly System.TimeSpan DEFAULT_DEPLOYMENT_LOAD_PUBLISHER_REFRESH_TIME; public System.TimeSpan DeploymentLoadPublisherRefreshTime { get { throw null; } set { } } + + public DisseminationNamespaceOptions Dissemination { get { throw null; } set { } } } public partial class DevelopmentClusterMembershipOptions diff --git a/test/Dissemination.IntegrationHarness/README.md b/test/Dissemination.IntegrationHarness/README.md new file mode 100644 index 00000000000..0a32851a3bf --- /dev/null +++ b/test/Dissemination.IntegrationHarness/README.md @@ -0,0 +1,46 @@ +# Dissemination compatibility tests + +This harness tests actual old/new runtime binaries in separate OS processes. The +controller references shared control types and loads neither Orleans runtime. +The `Dissemination compatibility` workflow builds each silo executable against +its own isolated runtime checkout and verifies the loaded assembly hashes, +versions, and paths. + +The original runtime is pinned to +`6739589254b746a8790cf53524e6abe372bb53d4`. Only harness sources are copied into +that checkout; Orleans runtime sources remain unchanged. + +## Coverage + +- Rolling upgrades, mixed enablement, legacy fallback, and rollback. +- Exact load-state convergence after three partition/heal cycles, at four and + eight silos, using original, current-disabled, and current-enabled runtimes. +- Full-snapshot and same-version heartbeat repair with tree delivery, direct + gossip, and membership-table reads independently isolated. +- Cancellation of an entered remote RPC and bounded partitioned shutdown. +- Harness guards for connection draining, exact inventory/value comparison, + public readonly statistics fields, and outgoing-activity observation. + +Commands own the load-publication cadence. Partition healing drains middleware +and transport closure at both endpoints, then establishes acknowledged +bidirectional readiness before one publication round. Recovery elapsed time +includes those probes. The disposed publication timer reference is retained for +the original runtime's shutdown lifecycle. + +## Execution and artifacts + +CI runs `.github\scripts\dissemination-compatibility.ps1`. Runtime builds are +CI-only. To reproduce locally, download the matching +`dissemination-compatibility-binaries` artifact into +`Artifacts\DisseminationCompatibility\bin`, then run: + +```powershell +.\.github\scripts\dissemination-compatibility.ps1 -SkipBuild +``` + +The runner checks eight harness cases and ten process cases. Results are written +under `Artifacts\DisseminationCompatibility\results`: TRX files, invocation +metadata, scenario snapshots, process logs, and final shutdown records. + +Scaling experiments and performance reports are maintained separately from this +correctness suite. diff --git a/test/Dissemination.IntegrationHarness/Shared/Protocol.cs b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs new file mode 100644 index 00000000000..24c92a5f314 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Shared/Protocol.cs @@ -0,0 +1,256 @@ +using System.Text.Json; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed record NodeConfiguration( + string Name, + string ClusterId, + string MembershipDirectory, + int SiloPort, + int ParentProcessId, + bool Enabled, + bool FastRecovery = true); + +internal sealed record Command( + int Id, + string Operation, + string? Peer = null, + int Count = 1, + bool Value = false, + long Version = 0); + +internal sealed record Response(int Id, NodeSnapshot? Snapshot, string? Error); + +internal sealed record BinaryIdentity( + int ProcessId, + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string AssemblyPath, + bool HasDissemination) +{ + public Dictionary Assemblies { get; init; } = []; +} + +internal sealed record AssemblyProof( + string Version, + string InformationalVersion, + string ModuleVersionId, + string Sha256, + string Path); + +internal sealed record TreeGateSnapshot(bool Blocked, int InFlight, long Admitted, long Rejected); + +internal sealed record StateComparisonProbe(string Before, string After, string[] PublicFields); + +internal sealed record ControlledCallSnapshot( + string? Peer, + DateTimeOffset? StartedAtUtc, + DateTimeOffset? CancellationRequestedAtUtc, + bool CancellationRequested, + bool RawTaskCompleted, + string? RawTaskStatus, + string? RawTaskError); + +// The token source outlives the start command. Cancellation is a separate, explicit phase and +// completion means the actual RPC task completed, not a wait canceled by that same token. +internal sealed class ControlledCall : IDisposable +{ + private CancellationTokenSource? _cancellation; + private Task? _request; + private string? _peer; + private DateTimeOffset? _startedAt; + private DateTimeOffset? _cancelledAt; + + public ControlledCallSnapshot Snapshot => new( + _peer, + _startedAt, + _cancelledAt, + _cancellation?.IsCancellationRequested == true, + _request?.IsCompleted == true, + _request?.Status.ToString(), + _request?.Exception?.ToString()); + + public void Start(string peer, Func invoke) + { + if (_request is not null) + { + throw new InvalidOperationException("A controlled RPC has already been started."); + } + + _peer = peer; + _startedAt = DateTimeOffset.UtcNow; + _cancellation = new CancellationTokenSource(); + _request = invoke(_cancellation.Token); + _ = _request.ContinueWith( + static completed => _ = completed.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + public async Task Cancel(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var source = _cancellation ?? throw new InvalidOperationException("No controlled RPC has been started."); + var request = _request ?? throw new InvalidOperationException("The controlled RPC was not created."); + if (request.IsCompleted) + { + throw new InvalidOperationException($"The RPC completed before explicit cancellation: {request.Status}.", request.Exception); + } + + _cancelledAt = DateTimeOffset.UtcNow; + await source.CancelAsync().WaitAsync(cancellationToken); + try + { + await request.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (source.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + return; + } + + throw new InvalidOperationException("The RPC completed successfully instead of observing cancellation."); + } + + public void Dispose() => _cancellation?.Dispose(); +} + +internal static class StateComparison +{ + private static readonly JsonSerializerOptions Options = new() { IncludeFields = true }; + + public static string Serialize(T value) => JsonSerializer.Serialize(value, Options); +} + +// Closing the gate is an admission barrier, not a timer or an assumption about queue expiry. +internal sealed class InFlightCallGate +{ + public const string ClosedMessage = "Harness PushBroadcast gate is closed."; + private readonly object _lock = new(); + private bool _blocked; + private int _inFlight; + private long _admitted; + private long _rejected; + private TaskCompletionSource? _drained; + + public TreeGateSnapshot Snapshot + { + get + { + lock (_lock) + { + return new(_blocked, _inFlight, _admitted, _rejected); + } + } + } + + public async Task Invoke(Func action) + { + lock (_lock) + { + if (_blocked) + { + _rejected++; + throw new InvalidOperationException(ClosedMessage); + } + + _admitted++; + _inFlight++; + } + + try + { + await action(); + } + finally + { + lock (_lock) + { + if (--_inFlight == 0) + { + _drained?.TrySetResult(); + } + } + } + } + + public async Task BlockAndDrain(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task pending; + lock (_lock) + { + _blocked = true; + pending = _inFlight == 0 + ? Task.CompletedTask + : (_drained ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task; + } + + await pending.WaitAsync(cancellationToken); + } + + public void Open() + { + lock (_lock) + { + if (_inFlight != 0) + { + throw new InvalidOperationException("Cannot reopen the tree gate before draining admitted calls."); + } + + _blocked = false; + _drained = null; + } + } +} + +internal sealed record ApplyEvidence( + string Namespace, + string Key, + long FromVersion, + long ToVersion, + string Result, + string? Peer); + +internal sealed record NodeSnapshot +{ + public DateTimeOffset CapturedAtUtc { get; init; } + public required BinaryIdentity Identity { get; init; } + public required string Address { get; init; } + public bool Enabled { get; init; } + public bool NamespaceEnabled { get; init; } + public long MembershipVersion { get; init; } + public required SortedDictionary Membership { get; init; } + public required SortedDictionary Load { get; init; } + public required SortedDictionary LoadVersions { get; init; } + public required string[] ActiveMembers { get; init; } + public required string[] UnconfirmedPeers { get; init; } + public long BroadcastsSent { get; init; } + public long OutgoingRepairs { get; init; } + public required ApplyEvidence[] Applies { get; init; } + public int PendingControlCalls { get; init; } + public int StartedControlCalls { get; init; } + public int CancelledControlCalls { get; init; } + public int ControlCancellationSignals { get; init; } + public bool ControlTokenCanBeCanceled { get; init; } + public bool ControlTokenCancelledOnEntry { get; init; } + public DateTimeOffset? ControlStartedAtUtc { get; init; } + public DateTimeOffset? ControlCancellationObservedAtUtc { get; init; } + public ControlledCallSnapshot? OutboundControlCall { get; init; } + public bool Partitioned { get; init; } + public bool LegacyGossipSuppressed { get; init; } + public bool MembershipReadsFrozen { get; init; } + public int? RemoteProcessId { get; init; } + public string? ProbeError { get; init; } + public bool ProbeTimedOut { get; init; } + public long? RepairFromVersion { get; init; } + public long? HeartbeatTicks { get; init; } + public TreeGateSnapshot? TreeGate { get; init; } + public bool? TreeProbeRejected { get; init; } + public StateComparisonProbe? ComparisonProbe { get; init; } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs b/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs new file mode 100644 index 00000000000..984dab6172d --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/ControlTarget.cs @@ -0,0 +1,110 @@ +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Dissemination.IntegrationHarness; + +[Alias("dissemination-evidence-control-v1")] +internal interface IControlTarget : ISystemTarget +{ + [Alias("echo-v1")] + Task Echo(CancellationToken cancellationToken); + + [Alias("hold-v1")] + Task Hold(CancellationToken cancellationToken); +} + +internal sealed class ControlTarget : SystemTarget, IControlTarget +{ + public static readonly GrainType TargetType = SystemTargetGrainId.CreateGrainType("test.dissemination-evidence"); + private int _pending; + private int _started; + private int _cancelled; + private int _cancellationSignals; + private int _canBeCanceled; + private int _cancelledOnEntry; + private long _startedAtTicks; + private long _cancellationObservedAtTicks; + + public ControlTarget(SystemTargetShared shared) : base(TargetType, shared) => + shared.ActivationDirectory.RecordNewTarget(this); + + public int Pending => Volatile.Read(ref _pending); + public int Started => Volatile.Read(ref _started); + public int Cancelled => Volatile.Read(ref _cancelled); + public int CancellationSignals => Volatile.Read(ref _cancellationSignals); + public bool TokenCanBeCanceled => Volatile.Read(ref _canBeCanceled) != 0; + public bool TokenCancelledOnEntry => Volatile.Read(ref _cancelledOnEntry) != 0; + public DateTimeOffset? StartedAtUtc => Timestamp(Interlocked.Read(ref _startedAtTicks)); + public DateTimeOffset? CancellationObservedAtUtc => Timestamp(Interlocked.Read(ref _cancellationObservedAtTicks)); + + public Task Echo(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(Environment.ProcessId); + } + + public async Task Hold(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _pending); + Interlocked.Increment(ref _started); + Volatile.Write(ref _canBeCanceled, cancellationToken.CanBeCanceled ? 1 : 0); + Volatile.Write(ref _cancelledOnEntry, cancellationToken.IsCancellationRequested ? 1 : 0); + Interlocked.Exchange(ref _startedAtTicks, DateTime.UtcNow.Ticks); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + Interlocked.Exchange(ref _cancellationObservedAtTicks, DateTime.UtcNow.Ticks); + Interlocked.Increment(ref _cancellationSignals); + throw; + } + finally + { + if (cancellationToken.IsCancellationRequested) + { + Interlocked.Increment(ref _cancelled); + } + + Interlocked.Decrement(ref _pending); + } + } + + private static DateTimeOffset? Timestamp(long ticks) => + ticks == 0 ? null : new DateTimeOffset(ticks, TimeSpan.Zero); +} + +internal sealed class GatedMembershipGossiper(MembershipGossiper inner) : IMembershipGossiper +{ + private int _suppressed; + + public bool Suppressed + { + get => Volatile.Read(ref _suppressed) != 0; + set => Volatile.Write(ref _suppressed, value ? 1 : 0); + } + + public Task GossipToRemoteSilos( + List gossipPartners, + MembershipTableSnapshot snapshot, + SiloAddress updatedSilo, +#if NEW_RUNTIME + SiloStatus updatedStatus, + CancellationToken cancellationToken) +#else + SiloStatus updatedStatus) +#endif + { +#if NEW_RUNTIME + cancellationToken.ThrowIfCancellationRequested(); + return Suppressed + ? Task.CompletedTask + : inner.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus, cancellationToken); +#else + return Suppressed + ? Task.CompletedTask + : inner.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus); +#endif + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj b/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj new file mode 100644 index 00000000000..9a4dd8239cc --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Dissemination.Silo.csproj @@ -0,0 +1,18 @@ + + + net10.0 + Exe + + Orleans.Runtime.Tests + Orleans.Dissemination.IntegrationHarness + true + New + $(DefineConstants);NEW_RUNTIME + + + + + + + + diff --git a/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs b/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs new file mode 100644 index 00000000000..4e354dd2f77 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/FileMembershipTable.cs @@ -0,0 +1,238 @@ +using System.Text.Json; +using Orleans.Runtime; + +namespace Orleans.Dissemination.IntegrationHarness; + +// The wire schema is deliberately independent of either Orleans serializer version. +// This is a single-machine CI provider, not a production storage implementation. +internal sealed class FileMembershipTable(NodeConfiguration configuration) : IMembershipTable +{ + private readonly string _dataPath = Path.Combine(configuration.MembershipDirectory, "membership.json"); + private readonly string _lockPath = Path.Combine(configuration.MembershipDirectory, "membership.lock"); + private TableState? _frozen; + + public bool ReadsFrozen => Volatile.Read(ref _frozen) is not null; + + public async Task FreezeReads(bool freeze, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = freeze ? await Access(static state => state, write: false, cancellationToken) : null; + Volatile.Write(ref _frozen, state); + } + +#if NEW_RUNTIME + Task IMembershipTable.InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + Task IMembershipTable.DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + Task IMembershipTable.CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + Task IMembershipTable.ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + Task IMembershipTable.ReadAll() => ReadAllAsync(CancellationToken.None); + + Task IMembershipTable.InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + Task IMembershipTable.UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#else + public Task InitializeMembershipTable(bool tryInitTableVersion) => + InitializeMembershipTableAsync(tryInitTableVersion, CancellationToken.None); + + public Task DeleteMembershipTableEntries(string clusterId) => + DeleteMembershipTableEntriesAsync(clusterId, CancellationToken.None); + + public Task CleanupDefunctSiloEntries(DateTimeOffset beforeDate) => + CleanupDefunctSiloEntriesAsync(beforeDate, CancellationToken.None); + + public Task ReadRow(SiloAddress key) => ReadRowAsync(key, CancellationToken.None); + + public Task ReadAll() => ReadAllAsync(CancellationToken.None); + + public Task InsertRow(MembershipEntry entry, TableVersion tableVersion) => + InsertRowAsync(entry, tableVersion, CancellationToken.None); + + public Task UpdateRow(MembershipEntry entry, string etag, TableVersion tableVersion) => + UpdateRowAsync(entry, etag, tableVersion, CancellationToken.None); + + public Task UpdateIAmAlive(MembershipEntry entry) => UpdateIAmAliveAsync(entry, CancellationToken.None); +#endif + + public Task InitializeMembershipTableAsync(bool tryInitTableVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Directory.CreateDirectory(configuration.MembershipDirectory); + return Access(static _ => true, write: true, cancellationToken); + } + + public Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken cancellationToken) => Access(state => + { + if (!StringComparer.Ordinal.Equals(clusterId, configuration.ClusterId)) + { + throw new InvalidOperationException("The harness provider cannot delete another cluster."); + } + + state.Rows.Clear(); + state.Version++; + return true; + }, write: true, cancellationToken); + + public Task CleanupDefunctSiloEntriesAsync(DateTimeOffset beforeDate, CancellationToken cancellationToken) => Access(state => + { + foreach (var key in state.Rows.Where(pair => + pair.Value.Entry.Status == (int)SiloStatus.Dead + && pair.Value.Entry.IAmAliveTime < beforeDate.UtcDateTime).Select(pair => pair.Key).ToArray()) + { + state.Rows.Remove(key); + } + + return true; + }, write: true, cancellationToken); + + public async Task ReadRowAsync(SiloAddress key, CancellationToken cancellationToken) + { + var all = await ReadAllAsync(cancellationToken); + return new MembershipTableData(all.Members.Where(row => row.Item1.SiloAddress.Equals(key)).ToList(), all.Version); + } + + public async Task ReadAllAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = Volatile.Read(ref _frozen) ?? await Access(static state => state, write: false, cancellationToken); + return new MembershipTableData( + state.Rows.Values.Select(row => Tuple.Create(row.Entry.ToEntry(), row.Etag)).ToList(), + new TableVersion(state.Version, state.Etag)); + } + + public Task InsertRowAsync(MembershipEntry entry, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (state.Rows.ContainsKey(key) || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows.Add(key, new Row(EntryData.From(entry), Guid.NewGuid().ToString("N"))); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateRowAsync(MembershipEntry entry, string etag, TableVersion tableVersion, CancellationToken cancellationToken) => Access(state => + { + var key = entry.SiloAddress.ToParsableString(); + if (!state.Rows.TryGetValue(key, out var row) + || !StringComparer.Ordinal.Equals(row.Etag, etag) + || !StringComparer.Ordinal.Equals(tableVersion.VersionEtag, state.Etag)) + { + return false; + } + + state.Rows[key] = new Row(EntryData.From(entry), Guid.NewGuid().ToString("N")); + state.Version = tableVersion.Version; + state.Etag = Guid.NewGuid().ToString("N"); + return true; + }, write: true, cancellationToken); + + public Task UpdateIAmAliveAsync(MembershipEntry entry, CancellationToken cancellationToken) => Access(state => + { + if (state.Rows.TryGetValue(entry.SiloAddress.ToParsableString(), out var row)) + { + state.Rows[entry.SiloAddress.ToParsableString()] = row with + { + Entry = row.Entry with { IAmAliveTime = entry.IAmAliveTime }, + }; + } + + return true; + }, write: true, cancellationToken); + + private async Task Access(Func action, bool write, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(10)); + FileStream lease; + while (true) + { + deadline.Token.ThrowIfCancellationRequested(); + try + { + lease = new FileStream(_lockPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + break; + } + catch (IOException) + { + await Task.Delay(10, deadline.Token); + } + } + + await using (lease) + { + var state = File.Exists(_dataPath) + ? JsonSerializer.Deserialize(await File.ReadAllTextAsync(_dataPath, deadline.Token))! + : new TableState(); + deadline.Token.ThrowIfCancellationRequested(); + var result = action(state); + if (write) + { + // The sibling file is in the owned artifact directory, never the system temporary directory. + var staging = _dataPath + ".next"; + await File.WriteAllTextAsync(staging, JsonSerializer.Serialize(state), deadline.Token); + deadline.Token.ThrowIfCancellationRequested(); + File.Move(staging, _dataPath, overwrite: true); + } + + return result; + } + } + + private sealed class TableState + { + public int Version { get; set; } + public string Etag { get; set; } = "0"; + public Dictionary Rows { get; set; } = []; + } + + private sealed record Row(EntryData Entry, string Etag); + + internal sealed record EntryData( + string Address, + int Status, + int ProxyPort, + string HostName, + string SiloName, + string? RoleName, + int UpdateZone, + int FaultZone, + DateTime StartTime, + DateTime IAmAliveTime, + Dictionary Suspects) + { + public static EntryData From(MembershipEntry entry) => new( + entry.SiloAddress.ToParsableString(), (int)entry.Status, entry.ProxyPort, + entry.HostName, entry.SiloName, entry.RoleName, entry.UpdateZone, entry.FaultZone, + entry.StartTime, entry.IAmAliveTime, + entry.SuspectTimes?.ToDictionary(pair => pair.Item1.ToParsableString(), pair => pair.Item2) ?? []); + + public MembershipEntry ToEntry() => new() + { + SiloAddress = SiloAddress.FromParsableString(Address), + Status = (SiloStatus)Status, + ProxyPort = ProxyPort, + HostName = HostName, + SiloName = SiloName, + RoleName = RoleName, + UpdateZone = UpdateZone, + FaultZone = FaultZone, + StartTime = StartTime, + IAmAliveTime = IAmAliveTime, + SuspectTimes = Suspects.Select(pair => Tuple.Create(SiloAddress.FromParsableString(pair.Key), pair.Value)).ToList(), + }; + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs new file mode 100644 index 00000000000..572cfef58ab --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/NewRuntime.cs @@ -0,0 +1,215 @@ +#if NEW_RUNTIME +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal static class NewRuntime +{ + public static void Configure(IServiceCollection services, NodeConfiguration configuration) + { + services.Configure(options => options.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + services.Configure(options => options.Dissemination.Enabled = configuration.Enabled); + if (!configuration.Enabled) + { + return; + } + + if (configuration.FastRecovery) + { + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + } + + services.Configure(options => + { + if (configuration.FastRecovery) + { + options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(250); + options.Overlay.AntiEntropyPeerCount = 2; + options.Overlay.MinFanOutFactor = 2; + options.Overlay.MaxFanOutFactor = 2; + options.MaxConcurrentSends = 4; + } + }); + services.Configure(options => ConfigureRecovery(options.Dissemination, configuration.FastRecovery)); + services.Configure(options => ConfigureRecovery(options.Dissemination, configuration.FastRecovery)); + } + + private static void ConfigureRecovery(DisseminationNamespaceOptions options, bool fastRecovery) + { + if (fastRecovery) + { + options.ExpectedUpdateCadence = TimeSpan.FromMilliseconds(100); + options.StaleItemTtl = TimeSpan.FromSeconds(1); + } + } + + public static NodeSnapshot Decorate(IServiceProvider services, NodeSnapshot snapshot) + { + var options = services.GetRequiredService>().Value; + if (!options.Enabled) + { + return snapshot with + { + NamespaceEnabled = services.GetRequiredService>().Value.Dissemination.Enabled, + }; + } + + var ns = services.GetRequiredService(); + return snapshot with + { + Enabled = options.Enabled, + NamespaceEnabled = ns.Options.Enabled, + UnconfirmedPeers = services.GetRequiredService() + .GetUnconfirmedPeers(ns).Select(address => address.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + TreeGate = services.GetService()?.Snapshot, + }; + } + + public static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + { + var manager = services.GetRequiredService(); + var dissemination = services.GetRequiredService(); + var ns = services.GetRequiredService(); + var local = services.GetRequiredService().SiloAddress; + switch (command.Operation) + { + case "block-tree": + { + var gate = services.GetRequiredService(); + if (command.Value) + { + await gate.BlockAndDrain(cancellationToken); + } + else + { + gate.Open(); + } + + break; + } + case "probe-tree": + { + var peer = services.GetRequiredService() + .GetSystemTarget( + Constants.DisseminationSystemTargetType, SiloAddress.FromParsableString(command.Peer!)); + try + { + await peer.PushBroadcast(new() { Sender = local }, cancellationToken); + return Program.Snapshot(services) with { TreeProbeRejected = false }; + } + catch (InvalidOperationException exception) when (exception.Message == InFlightCallGate.ClosedMessage) + { + return Program.Snapshot(services) with { TreeProbeRejected = true }; + } + } + case "probe": + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(8)); + try + { + var peer = services.GetRequiredService() + .GetSystemTarget( + Constants.DisseminationSystemTargetType, SiloAddress.FromParsableString(command.Peer!)); + var response = await peer.ExchangeAntiEntropy(new() + { + Sender = local, + SupportedNamespaces = [ns.Name], + }, deadline.Token); + return Program.Snapshot(services) with + { + ProbeError = response.UnsupportedNamespaces.Contains(ns.Name) ? "namespace-unsupported" : null, + }; + } + catch (Exception exception) when (!cancellationToken.IsCancellationRequested) + { + return Program.Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + ProbeTimedOut = deadline.IsCancellationRequested || exception is TimeoutException, + }; + } + } + case "publish-membership": + await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); + break; + case "membership-update": + { + if (!services.GetRequiredService().Suppressed) + { + throw new InvalidOperationException("Membership injection requires legacy gossip isolation."); + } + + var table = services.GetRequiredService(); + await table.FreezeReads(false, cancellationToken); + var current = await table.ReadAllAsync(cancellationToken); + var row = current.Members.Single(row => row.Item1.SiloAddress.Equals(local)); + row.Item1.HostName = $"membership-update-{current.Version.Version + 1}"; + if (!await table.UpdateRowAsync(row.Item1, row.Item2, current.Version.Next(), cancellationToken)) + { + throw new InvalidOperationException("Unexpected concurrent membership writer during isolated repair test."); + } + + await manager.Refresh(null, cancellationToken); + await ns.PublishAsync(dissemination, manager.CurrentSnapshot, cancellationToken); + await table.FreezeReads(true, cancellationToken); + var repair = ns.CreateRepair(new( + DisseminationKey.Default, command.Version, 1024 * 1024, 1024 * 1024)); + if (repair.Status != DisseminationRepairStatus.Produced) + { + throw new InvalidOperationException($"Expected a production membership repair, got {repair.Status}."); + } + + return Program.Snapshot(services) with { RepairFromVersion = repair.Value.FromVersion }; + } + case "membership-heartbeat": + { + if (!services.GetRequiredService().Suppressed + || !services.GetRequiredService().ReadsFrozen) + { + throw new InvalidOperationException("Heartbeat injection requires storage and legacy gossip isolation."); + } + + var current = manager.CurrentSnapshot; + var entries = current.Entries.ToBuilder(); + var entry = FileMembershipTable.EntryData.From(entries[local]).ToEntry(); + entry.IAmAliveTime = entry.IAmAliveTime.AddHours(1); + entries[local] = entry; + await manager.ProcessGossipSnapshot(new(current.Version, entries.ToImmutable()), cancellationToken); + // Deliberately no Publish: only same-version fingerprint anti-entropy can carry this heartbeat. + return Program.Snapshot(services) with { HeartbeatTicks = entry.IAmAliveTime.Ticks }; + } + default: + throw new NotSupportedException(command.Operation); + } + + return Program.Snapshot(services); + } +} + +// Gate cohort ingress and tree application while native anti-entropy uses the real connection. +internal sealed class DisseminationTreeGate : IIncomingGrainCallFilter +{ + private readonly InFlightCallGate _gate = new(); + + public TreeGateSnapshot Snapshot => _gate.Snapshot; + + public Task BlockAndDrain(CancellationToken cancellationToken) => _gate.BlockAndDrain(cancellationToken); + + public void Open() => _gate.Open(); + + public Task Invoke(IIncomingGrainCallContext context) => + context.InterfaceMethod.DeclaringType == typeof(IDisseminationSystemTarget) + && context.InterfaceMethod.Name is nameof(IDisseminationSystemTarget.PushBroadcast) or nameof(IDisseminationSystemTarget.PublishAggregated) + ? _gate.Invoke(context.Invoke) + : context.Invoke(); +} +#endif diff --git a/test/Dissemination.IntegrationHarness/Silo/Observation.cs b/test/Dissemination.IntegrationHarness/Silo/Observation.cs new file mode 100644 index 00000000000..65f91c22d8f --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Observation.cs @@ -0,0 +1,240 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.AspNetCore.Connections; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed class Observation : IDisposable, IObserver, IObserver> +{ + private readonly object _lock = new(); + private readonly MeterListener _meters = new(); + private readonly List _subscriptions = []; + private readonly Queue _applies = []; + private readonly object _connectionLock = new(); + private readonly Dictionary _connections = []; + private readonly HashSet _drainingConnections = []; + private Func? _blockedConnections; + private int _partitioned; + private long _broadcastsSent; + private long _outgoingRepairs; + + public Observation() + { + _meters.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name.StartsWith("Microsoft.Orleans", StringComparison.Ordinal) + && instrument.Name is "orleans-dissemination-broadcast-sent" or "orleans-dissemination-anti-entropy-exchanges") + { + listener.EnableMeasurementEvents(instrument); + } + }; + _meters.SetMeasurementEventCallback((instrument, value, tags, _) => Record(instrument, value, tags)); + _meters.Start(); + _subscriptions.Add(DiagnosticListener.AllListeners.Subscribe(this)); + } + + public bool Partitioned => Volatile.Read(ref _partitioned) != 0; + public long BroadcastsSent => Interlocked.Read(ref _broadcastsSent); + public long OutgoingRepairs => Interlocked.Read(ref _outgoingRepairs); + + public void Partition() + { + ConnectionLifetime[] connections; + lock (_connectionLock) + { + Volatile.Write(ref _partitioned, 1); + connections = CaptureConnectionsUnsafe(static _ => true); + } + + foreach (var connection in connections) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + } + + public async Task DrainConnections( + Func matches, + Func close, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConnectionLifetime[] connections; + lock (_connectionLock) + { + _blockedConnections = matches; + connections = CaptureConnectionsUnsafe(matches); + } + + var closing = new List(connections.Length * 2); + foreach (var connection in connections) + { + if (!connection.Completion.Task.IsCompleted) + { + connection.Context.Abort(new ConnectionAbortedException("Harness partition")); + } + + var closeTask = close(connection.Context); + lock (_connectionLock) + { + connection.CloseTask = closeTask; + } + + closing.Add(closeTask); + closing.Add(connection.Completion.Task); + } + + var drained = Task.WhenAll(closing); + _ = drained.ContinueWith( + static task => _ = task.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + await drained.WaitAsync(cancellationToken); + } + + public void ResumeConnections() + { + lock (_connectionLock) + { + if (_drainingConnections.Any(static connection => + !connection.Completion.Task.IsCompleted || connection.CloseTask is not { IsCompletedSuccessfully: true })) + { + throw new InvalidOperationException("Partitioned connection middleware must drain before reopening."); + } + + _drainingConnections.Clear(); + _blockedConnections = null; + Volatile.Write(ref _partitioned, 0); + } + } + + private ConnectionLifetime[] CaptureConnectionsUnsafe(Func matches) + { + foreach (var connection in _connections.Values) + { + if (matches(connection.Context)) + { + _drainingConnections.Add(connection); + } + } + + return [.. _drainingConnections]; + } + + public async Task Connection(ConnectionContext context, ConnectionDelegate next) + { + ConnectionLifetime? lifetime = null; + lock (_connectionLock) + { + if (!Partitioned && _blockedConnections?.Invoke(context) != true) + { + lifetime = new(context); + _connections.Add(context, lifetime); + } + } + + if (lifetime is null) + { + context.Abort(new ConnectionAbortedException("Harness partition")); + return; + } + + try + { + await next(context); + } + finally + { + lock (_connectionLock) + { + _connections.Remove(context); + lifetime.Completion.SetResult(); + } + } + } + + private sealed class ConnectionLifetime(ConnectionContext context) + { + public ConnectionContext Context { get; } = context; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public Task? CloseTask { get; set; } + } + + public ApplyEvidence[] Applies() + { + lock (_lock) + { + return _applies.ToArray(); + } + } + + private void Record(Instrument instrument, long value, ReadOnlySpan> tags) + { + if (instrument.Name == "orleans-dissemination-broadcast-sent") + { + Interlocked.Add(ref _broadcastsSent, value); + return; + } + + foreach (var tag in tags) + { + if (tag.Key == "direction" && Equals(tag.Value, "out")) + { + Interlocked.Add(ref _outgoingRepairs, value); + return; + } + } + } + + public void OnNext(DiagnosticListener value) + { + if (value.Name == "Microsoft.Orleans.Dissemination") + { + lock (_lock) + { + _subscriptions.Add(value.Subscribe(this)); + } + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Key != "Dissemination.ValueApply" || value.Value is not { } payload) + { + return; + } + + // Reflection keeps this observer identical in both binaries: the old runtime has no such type. + object? Property(string name) => payload.GetType().GetProperty(name)?.GetValue(payload); + var evidence = new ApplyEvidence( + Property("Namespace")?.ToString() ?? "", + Property("Key")?.ToString() ?? "", + Convert.ToInt64(Property("FromVersion"), System.Globalization.CultureInfo.InvariantCulture), + Convert.ToInt64(Property("ToVersion"), System.Globalization.CultureInfo.InvariantCulture), + Property("Result")?.ToString() ?? "", + Property("Peer")?.ToString()); + lock (_lock) + { + _applies.Enqueue(evidence); + while (_applies.Count > 4096) + { + _applies.Dequeue(); + } + } + } + + public void OnCompleted() { } + public void OnError(Exception error) { } + + public void Dispose() + { + _meters.Dispose(); + lock (_lock) + { + foreach (var subscription in _subscriptions) + { + subscription.Dispose(); + } + } + } +} diff --git a/test/Dissemination.IntegrationHarness/Silo/Program.cs b/test/Dissemination.IntegrationHarness/Silo/Program.cs new file mode 100644 index 00000000000..f68de2fd5c5 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Silo/Program.cs @@ -0,0 +1,386 @@ +using System.Diagnostics; +using System.Net; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Messaging; +using Orleans.Runtime.Scheduler; +using Orleans.Statistics; +using Orleans.TestingHost; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal static class Program +{ + private static readonly MethodInfo PublishMethod = typeof(DeploymentLoadPublisher).GetMethod( + "PublishStatistics", BindingFlags.Instance | BindingFlags.NonPublic)!; + + public static async Task Main(string[] args) + { + if (args is not ["--silo", var configurationPath]) + { + Console.Error.WriteLine("Expected --silo ."); + return 2; + } + + var configuration = JsonSerializer.Deserialize(await File.ReadAllTextAsync(configurationPath))!; +#if !NEW_RUNTIME + if (configuration.Enabled) + { + throw new InvalidOperationException("The pinned old runtime cannot be opted into dissemination."); + } +#endif + using var lifetime = new CancellationTokenSource(TimeSpan.FromMinutes(45)); + using var observation = new Observation(); + var membership = new FileMembershipTable(configuration); + using var host = TestClusterHostFactory.CreateSiloHost( + configuration.Name, + new ConfigurationBuilder().Build(), + builder => builder.ConfigureServices(services => + { + services.AddSingleton(configuration); + services.AddSingleton(observation); + services.AddSingleton(membership); + services.AddSingleton(membership); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.Configure(options => + { + options.ClusterId = configuration.ClusterId; + options.ServiceId = "dissemination-compatibility"; + }); + services.Configure(options => options.SiloName = configuration.Name); + services.Configure(options => + { + options.AdvertisedIPAddress = IPAddress.Loopback; + options.SiloPort = configuration.SiloPort; + options.GatewayPort = 0; + }); + services.Configure(options => + { + // Partitions test dissemination repair, not the separately tested failure detector. + options.LivenessEnabled = false; + options.TableRefreshTimeout = TimeSpan.FromHours(1); + options.IAmAliveTablePublishTimeout = TimeSpan.FromHours(1); + options.MaxJoinAttemptTime = TimeSpan.FromSeconds(60); + }); + services.Configure(options => + options.DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(1)); + services.Configure(options => options.ResponseTimeout = TimeSpan.FromSeconds(5)); + services.Configure(options => options.ShutdownTimeout = TimeSpan.FromSeconds(15)); + services.Configure(options => + { + options.ConfigureSiloInboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + options.ConfigureSiloOutboundConnection(connection => connection.Use(next => context => observation.Connection(context, next))); + }); + services.AddLogging(logging => + { + logging.SetMinimumLevel(LogLevel.Warning).AddConsole(options => + options.LogToStandardErrorThreshold = LogLevel.Trace); + if (configuration.ClusterId.StartsWith("bounded-shutdown-", StringComparison.Ordinal)) + { + logging.AddFilter("Orleans.Runtime.GrainCallCancellationManager", LogLevel.Debug); + } + }); +#if NEW_RUNTIME + NewRuntime.Configure(services, configuration); +#endif + })); + + _ = host.Services.GetRequiredService(); + var monitor = MonitorParent(configuration.ParentProcessId, lifetime); + try + { + using var startup = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + startup.CancelAfter(TimeSpan.FromSeconds(90)); + await host.StartAsync(startup.Token); + var publisher = host.Services.GetRequiredService(); + // Commands own publication cadence; lifecycle shutdown retains its disposable timer reference. + await publisher.RunOrQueueTask(() => + { + var field = typeof(DeploymentLoadPublisher).GetField("_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!; + ((IDisposable)field.GetValue(publisher)!).Dispose(); + return Task.CompletedTask; + }); + await Write(new Response(0, Snapshot(host.Services), null)); + + while (await Console.In.ReadLineAsync(lifetime.Token) is { } line) + { + var command = JsonSerializer.Deserialize(line)!; + using var operation = CancellationTokenSource.CreateLinkedTokenSource(lifetime.Token); + operation.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + var result = await Execute(host.Services, command, operation.Token); + await Write(new Response(command.Id, result, null)); + if (command.Operation == "stop") + { + break; + } + } + catch (Exception exception) + { + await Write(new Response(command.Id, null, exception.ToString())); + } + } + + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + await host.StopAsync(shutdown.Token).WaitAsync(shutdown.Token); + await Write(new Response(-1, Snapshot(host.Services), null)); + return 0; + } + finally + { + await lifetime.CancelAsync(); + await monitor; + } + } + + private static async Task Execute(IServiceProvider services, Command command, CancellationToken cancellationToken) + { + var membership = services.GetRequiredService(); + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + switch (command.Operation) + { + case "snapshot": + case "stop": + break; + case "verify-load-comparison": + { + var original = new EnvironmentStatistics(10, 20, 100, 100, 900, 900, 1000); + var changed = new EnvironmentStatistics(10, 35, 100, 100, 900, 900, 1000); + return Snapshot(services) with + { + ComparisonProbe = new( + StateComparison.Serialize(new { EnvironmentStatistics = original }), + StateComparison.Serialize(new { EnvironmentStatistics = changed }), + typeof(EnvironmentStatistics).GetFields(BindingFlags.Instance | BindingFlags.Public) + .Select(field => field.Name).ToArray()), + }; + } + case "refresh": + await manager.Refresh(null, cancellationToken); + break; + case "publish": + for (var index = 0; index < command.Count; index++) + { + await publisher.RunOrQueueTask(() => (Task)PublishMethod.Invoke(publisher, [cancellationToken])!) + .WaitAsync(cancellationToken); + } + + break; + case "partition": + { + var observation = services.GetRequiredService(); + if (command.Value) + { + observation.Partition(); + } + else + { + await DrainConnections(services, peer: null, cancellationToken); + observation.ResumeConnections(); + } + + break; + } + case "drain-connections": + await DrainConnections(services, command.Peer, cancellationToken); + break; + case "resume-connections": + services.GetRequiredService().ResumeConnections(); + break; + case "isolate-membership": + services.GetRequiredService().Suppressed = command.Value; + await membership.FreezeReads(command.Value, cancellationToken); + break; + case "forget-load": + await publisher.RunOrQueueTask(() => + { + cancellationToken.ThrowIfCancellationRequested(); + publisher.PeriodicStatistics.TryRemove(SiloAddress.FromParsableString(command.Peer!), out _); + return Task.CompletedTask; + }).WaitAsync(cancellationToken); + break; + case "echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var processId = await peer.Echo(cancellationToken).WaitAsync(cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + case "probe-echo": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + using var probeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + try + { + var request = peer.Echo(probeCancellation.Token); + request.Ignore(); + var processId = await request.WaitAsync(TimeSpan.FromSeconds(2), cancellationToken); + return Snapshot(services) with { RemoteProcessId = processId }; + } + catch (Exception exception) when (exception is ConnectionFailedException + or SiloUnavailableException or OrleansMessageRejectionException or TimeoutException) + { + return Snapshot(services) with + { + ProbeError = $"{exception.GetType().FullName}: {exception.Message}", + ProbeTimedOut = exception is TimeoutException, + }; + } + finally + { + await probeCancellation.CancelAsync(); + } + } + case "start-cancel-rpc": + { + var peer = services.GetRequiredService().GetSystemTarget( + ControlTarget.TargetType, SiloAddress.FromParsableString(command.Peer!)); + var call = services.GetRequiredService(); + // Match dissemination's silo-system-target calling context, not a hosted-client call. + await services.GetRequiredService().RunOrQueueTask(() => + { + cancellationToken.ThrowIfCancellationRequested(); + call.Start(command.Peer!, peer.Hold); + return Task.CompletedTask; + }).WaitAsync(cancellationToken); + + break; + } + case "cancel-started-rpc": + await services.GetRequiredService().Cancel(cancellationToken); + break; + default: +#if NEW_RUNTIME + return await NewRuntime.Execute(services, command, cancellationToken); +#else + throw new NotSupportedException($"The pinned old runtime does not support '{command.Operation}'."); +#endif + } + + return Snapshot(services); + } + + private static Task DrainConnections(IServiceProvider services, string? peer, CancellationToken cancellationToken) + { + var address = peer is null ? null : SiloAddress.FromParsableString(peer); + return services.GetRequiredService().DrainConnections( + context => + { + var connection = (SiloConnection)context.Features.Get()!; + // Inbound handshakes have no remote silo identity until their preamble completes. + return address is null || connection.RemoteSiloAddress is null || connection.RemoteSiloAddress.Equals(address); + }, + context => context.Features.Get()!.CloseAsync(new ConnectionAbortedException("Harness partition")), + cancellationToken); + } + + internal static NodeSnapshot Snapshot(IServiceProvider services) + { + var manager = services.GetRequiredService(); + var publisher = services.GetRequiredService(); + var observation = services.GetRequiredService(); + var membership = manager.CurrentSnapshot; + var result = new NodeSnapshot + { + CapturedAtUtc = DateTimeOffset.UtcNow, + Identity = Identity(), + Address = services.GetRequiredService().SiloAddress.ToParsableString(), + MembershipVersion = membership.Version.Value, + Membership = new(membership.Entries.ToDictionary( + pair => pair.Key.ToParsableString(), + pair => JsonSerializer.Serialize(FileMembershipTable.EntryData.From(pair.Value))), StringComparer.Ordinal), + ActiveMembers = membership.Entries.Values.Where(entry => entry.Status == SiloStatus.Active) + .Select(entry => entry.SiloAddress.ToParsableString()).Order(StringComparer.Ordinal).ToArray(), + Load = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => StateComparison.Serialize(pair.Value)), StringComparer.Ordinal), + LoadVersions = new(publisher.PeriodicStatistics.ToDictionary( + pair => pair.Key.ToParsableString(), pair => pair.Value.DateTime.Ticks), StringComparer.Ordinal), + UnconfirmedPeers = [], + BroadcastsSent = observation.BroadcastsSent, + OutgoingRepairs = observation.OutgoingRepairs, + Applies = observation.Applies(), + PendingControlCalls = services.GetRequiredService().Pending, + StartedControlCalls = services.GetRequiredService().Started, + CancelledControlCalls = services.GetRequiredService().Cancelled, + ControlCancellationSignals = services.GetRequiredService().CancellationSignals, + ControlTokenCanBeCanceled = services.GetRequiredService().TokenCanBeCanceled, + ControlTokenCancelledOnEntry = services.GetRequiredService().TokenCancelledOnEntry, + ControlStartedAtUtc = services.GetRequiredService().StartedAtUtc, + ControlCancellationObservedAtUtc = services.GetRequiredService().CancellationObservedAtUtc, + OutboundControlCall = services.GetRequiredService().Snapshot, + Partitioned = observation.Partitioned, + LegacyGossipSuppressed = services.GetRequiredService().Suppressed, + MembershipReadsFrozen = services.GetRequiredService().ReadsFrozen, + }; +#if NEW_RUNTIME + result = NewRuntime.Decorate(services, result); +#endif + return result; + } + + private static BinaryIdentity Identity() + { + // Hash once: periodic snapshots must not read the runtime image repeatedly. + return CachedIdentity.Value; + } + + private static readonly Lazy CachedIdentity = new(() => + { + var assembly = typeof(Silo).Assembly; + var metadata = typeof(Program).Assembly.GetCustomAttributes() + .ToDictionary(attribute => attribute.Key, attribute => attribute.Value); + return new( + Environment.ProcessId, metadata["HarnessRuntime"]!, metadata["HarnessSourceRevision"]!, + assembly.GetName().Name!, assembly.GetName().Version!.ToString(), + assembly.GetCustomAttribute()!.InformationalVersion, + assembly.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(assembly.Location))), + assembly.Location, + assembly.GetType("Orleans.Runtime.Dissemination.DisseminationSystemTarget") is not null) + { + Assemblies = AppDomain.CurrentDomain.GetAssemblies() + .Where(candidate => !candidate.IsDynamic && candidate.GetName().Name!.StartsWith("Orleans", StringComparison.Ordinal)) + .ToDictionary(candidate => candidate.GetName().Name!, candidate => new AssemblyProof( + candidate.GetName().Version!.ToString(), + candidate.GetCustomAttribute()?.InformationalVersion ?? "", + candidate.ManifestModule.ModuleVersionId.ToString(), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(candidate.Location))), + candidate.Location)), + }; + }); + + private static async Task MonitorParent(int parentProcessId, CancellationTokenSource lifetime) + { + try + { + using var parent = Process.GetProcessById(parentProcessId); + await parent.WaitForExitAsync(lifetime.Token); + await lifetime.CancelAsync(); + } + catch (ArgumentException) + { + await lifetime.CancelAsync(); + } + catch (OperationCanceledException) when (lifetime.IsCancellationRequested) + { + } + } + + private static Task Write(Response response) => Console.Out.WriteLineAsync("HARNESS " + JsonSerializer.Serialize(response)); +} diff --git a/test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs b/test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs new file mode 100644 index 00000000000..85a21e35a16 --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/CompatibilityHarnessTests.cs @@ -0,0 +1,281 @@ +using System.Diagnostics.Metrics; +using System.IO.Pipelines; +using System.Text.Json; +using Microsoft.AspNetCore.Connections; +using Xunit; + +namespace Orleans.Dissemination.IntegrationHarness; + +public sealed class CompatibilityHarnessTests +{ + [Fact] + public void LoadStateComparisonRequiresExactInventoryAndValues() + { + var expected = new Dictionary + { + ["silo@2"] = "current", + ["peer@1"] = "peer-value", + }; + var actual = new Dictionary(expected); + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + + actual["silo@1"] = "retired"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@2"); + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual.Remove("silo@1"); + actual["silo@2"] = "older-value"; + Assert.False(ProcessCluster.MatchesExpectedLoad(actual, expected)); + actual["silo@2"] = "current"; + Assert.True(ProcessCluster.MatchesExpectedLoad(actual, expected)); + } + + [Fact] + public async Task ConnectionDrain_WaitsForInitializationAndCloseBeforeReopening() + { + using var observation = new Observation(); + await using var context = CreateConnectionContext("initializing"); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var closed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var middleware = observation.Connection(context, async _ => + { + entered.SetResult(); + await release.Task; + }); + await entered.Task.WaitAsync(TestContext.Current.CancellationToken); + observation.Partition(); + Assert.True(context.ConnectionClosed.IsCancellationRequested); + var closeCalls = 0; + var drain = observation.DrainConnections( + static _ => true, + connection => + { + Assert.Same(context, connection); + closeCalls++; + return closed.Task; + }, + TestContext.Current.CancellationToken); + try + { + Assert.Equal(1, closeCalls); + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + await using var rejected = CreateConnectionContext("rejected"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("A partitioned connection entered.")); + release.SetResult(); + await middleware; + Assert.False(drain.IsCompleted); + Assert.Throws(observation.ResumeConnections); + closed.SetResult(); + await drain.WaitAsync(TestContext.Current.CancellationToken); + observation.ResumeConnections(); + Assert.False(observation.Partitioned); + var resumed = false; + await using var reopened = CreateConnectionContext("reopened"); + await observation.Connection(reopened, _ => + { + resumed = true; + return Task.CompletedTask; + }); + Assert.True(resumed); + } + finally + { + release.TrySetResult(); + closed.TrySetResult(); + await middleware; + await drain; + } + } + + [Fact] + public async Task ConnectionDrain_CapturesUnidentifiedPeersAndPreservesHealthyConnections() + { + using var observation = new Observation(); + await using var unknown = CreateConnectionContext("unknown"); + await using var healthy = CreateConnectionContext("healthy"); + healthy.Items["peer"] = "healthy"; + var releaseUnknown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseHealthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unknownMiddleware = observation.Connection(unknown, _ => releaseUnknown.Task); + var healthyMiddleware = observation.Connection(healthy, _ => releaseHealthy.Task); + using var cancellation = new CancellationTokenSource(); + var closed = new List(); + var drain = observation.DrainConnections( + context => !context.Items.TryGetValue("peer", out var peer) || Equals(peer, "partitioned"), + context => + { + closed.Add(context.ConnectionId); + return Task.CompletedTask; + }, + cancellation.Token); + try + { + Assert.Equal(new[] { "unknown" }, closed); + Assert.True(unknown.ConnectionClosed.IsCancellationRequested); + Assert.False(healthy.ConnectionClosed.IsCancellationRequested); + Assert.False(healthyMiddleware.IsCompleted); + await cancellation.CancelAsync(); + await Assert.ThrowsAnyAsync(() => drain); + await using var rejected = CreateConnectionContext("new-unknown"); + await observation.Connection(rejected, _ => throw new InvalidOperationException("An unidentified connection entered.")); + releaseUnknown.SetResult(); + await unknownMiddleware; + observation.ResumeConnections(); + Assert.False(healthyMiddleware.IsCompleted); + } + finally + { + releaseUnknown.TrySetResult(); + releaseHealthy.TrySetResult(); + await Task.WhenAll(unknownMiddleware, healthyMiddleware); + } + } + + [Fact] + public async Task ControlledCall_WaitsForRawCancellationAfterExplicitSignal() + { + var cancellationToken = TestContext.Current.CancellationToken; + using var call = new ControlledCall(); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken actualToken = default; + call.Start("receiver", token => + { + actualToken = token; + return raw.Task; + }); + using var registration = actualToken.UnsafeRegister( + static state => ((TaskCompletionSource)state!).SetResult(), cancellationObserved); + Assert.True(actualToken.CanBeCanceled); + Assert.False(actualToken.IsCancellationRequested); + Assert.False(call.Snapshot.CancellationRequested); + Assert.False(call.Snapshot.RawTaskCompleted); + var cancelling = call.Cancel(cancellationToken); + await cancellationObserved.Task.WaitAsync(cancellationToken); + Assert.True(actualToken.IsCancellationRequested); + Assert.False(cancelling.IsCompleted); + Assert.False(call.Snapshot.RawTaskCompleted); + raw.SetCanceled(actualToken); + await cancelling.WaitAsync(cancellationToken); + Assert.True(call.Snapshot.CancellationRequested); + Assert.True(call.Snapshot.RawTaskCompleted); + Assert.Equal("Canceled", call.Snapshot.RawTaskStatus); + } + + [Fact] + public async Task ControlledCall_DoesNotTreatRawTimeoutAsCancellation() + { + using var call = new ControlledCall(); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + call.Start("receiver", _ => raw.Task); + var cancelling = call.Cancel(TestContext.Current.CancellationToken); + raw.SetException(new TimeoutException("Raw RPC expired without observing cancellation.")); + var exception = await Assert.ThrowsAsync(() => cancelling); + Assert.Equal("Raw RPC expired without observing cancellation.", exception.Message); + Assert.True(call.Snapshot.CancellationRequested); + Assert.True(call.Snapshot.RawTaskCompleted); + Assert.Equal("Faulted", call.Snapshot.RawTaskStatus); + } + + [Fact] + public void StateComparison_ObservesNestedPublicReadonlyFieldChanges() + { + var original = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(20) }; + var changed = new { Version = 7, EnvironmentStatistics = new ReadonlyStatistics(35) }; + // This is the previous comparison's blind spot: properties alone produce identical JSON. + Assert.Equal(JsonSerializer.Serialize(original), JsonSerializer.Serialize(changed)); + var before = StateComparison.Serialize(original); + var after = StateComparison.Serialize(changed); + Assert.NotEqual(before, after); + using var beforeDocument = JsonDocument.Parse(before); + using var afterDocument = JsonDocument.Parse(after); + Assert.Equal(20, beforeDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, afterDocument.RootElement.GetProperty("EnvironmentStatistics").GetProperty("RawCpuUsagePercentage").GetSingle()); + } + + [Fact] + public async Task InFlightCallGate_DrainsPriorCallsAndRejectsLaterCalls() + { + var cancellationToken = TestContext.Current.CancellationToken; + var gate = new InFlightCallGate(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var prior = gate.Invoke(async () => + { + entered.SetResult(); + await release.Task.WaitAsync(cancellationToken); + }); + try + { + await entered.Task.WaitAsync(cancellationToken); + var drained = gate.BlockAndDrain(cancellationToken); + Assert.False(drained.IsCompleted); + var rejected = await Assert.ThrowsAsync(() => gate.Invoke( + () => throw new InvalidOperationException("A blocked call must never be invoked."))); + Assert.Equal(InFlightCallGate.ClosedMessage, rejected.Message); + Assert.Equal(new TreeGateSnapshot(true, 1, 1, 1), gate.Snapshot); + release.SetResult(); + await prior.WaitAsync(cancellationToken); + await drained.WaitAsync(cancellationToken); + Assert.Equal(new TreeGateSnapshot(true, 0, 1, 1), gate.Snapshot); + await Assert.ThrowsAsync(() => gate.Invoke( + () => throw new InvalidOperationException("A call must remain blocked after the drain."))); + Assert.Equal(new TreeGateSnapshot(true, 0, 1, 2), gate.Snapshot); + gate.Open(); + await gate.Invoke(() => Task.CompletedTask); + Assert.Equal(new TreeGateSnapshot(false, 0, 2, 2), gate.Snapshot); + } + finally + { + release.TrySetResult(); + await prior.WaitAsync(cancellationToken); + } + } + + [Fact] + public void DisseminationActivityCountsOnlyOutgoingWork() + { + using var observation = new Observation(); + using var meter = new Meter("Microsoft.Orleans.CompatibilityHarnessTest"); + var broadcasts = meter.CreateCounter("orleans-dissemination-broadcast-sent"); + var repairs = meter.CreateCounter("orleans-dissemination-anti-entropy-exchanges"); + broadcasts.Add(2); + repairs.Add(3, new KeyValuePair("direction", "out")); + repairs.Add(5, new KeyValuePair("direction", "in")); + + Assert.Equal(2, observation.BroadcastsSent); + Assert.Equal(3, observation.OutgoingRepairs); + } + + private sealed record TestPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; + + private static DefaultConnectionContext CreateConnectionContext(string id) + { + var pipe = new Pipe(); + return new TestConnectionContext(id) { Transport = new TestPipe(pipe.Reader, pipe.Writer) }; + } + + // DefaultConnectionContext queues Cancel independently of Dispose. These tests own abort synchronously; + // separate barriers continue to control middleware and transport completion. + private sealed class TestConnectionContext : DefaultConnectionContext + { + private readonly CancellationTokenSource _closed = new(); + + public TestConnectionContext(string id) : base(id) => ConnectionClosed = _closed.Token; + + public override void Abort(ConnectionAbortedException abortReason) => _closed.Cancel(); + + public override ValueTask DisposeAsync() + { + _closed.Dispose(); + return base.DisposeAsync(); + } + } + + private readonly struct ReadonlyStatistics(float rawCpuUsagePercentage) + { + public readonly float RawCpuUsagePercentage = rawCpuUsagePercentage; + } +} diff --git a/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj b/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj new file mode 100644 index 00000000000..6ce835f241a --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/Dissemination.IntegrationHarness.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + + + + + + + + diff --git a/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs new file mode 100644 index 00000000000..aa7a08e4aff --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/ProcessCluster.cs @@ -0,0 +1,425 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using Xunit; + +namespace Orleans.Dissemination.IntegrationHarness; + +internal sealed record AssemblyManifest(string AssemblyVersion, string Sha256); +internal sealed record BuildManifest( + string Runtime, + string SourceRevision, + string AssemblyName, + string AssemblyVersion, + string Sha256, + Dictionary Assemblies); + +internal sealed class ProcessCluster : IAsyncDisposable +{ + public const string Baseline = "6739589254b746a8790cf53524e6abe372bb53d4"; + private readonly List _all = []; + private readonly string _membershipDirectory; + private int _sequence; + + public ProcessCluster(string scenario, bool fastRecovery = true) + { + var artifactRoot = Environment.GetEnvironmentVariable("ORLEANS_DISSEMINATION_RESULTS") + ?? throw new InvalidOperationException("Run .github/scripts/dissemination-compatibility.ps1 first, or set ORLEANS_DISSEMINATION_{OLD,NEW,RESULTS}."); + Directory = Path.Combine(Path.GetFullPath(artifactRoot), $"{scenario}-{Guid.NewGuid():N}"); + System.IO.Directory.CreateDirectory(Directory); + _membershipDirectory = Path.Combine(Directory, "membership"); + System.IO.Directory.CreateDirectory(_membershipDirectory); + FastRecovery = fastRecovery; + } + + public string Directory { get; } + public bool FastRecovery { get; } + public SiloProcess[] Active => _all.Where(node => !node.Stopped).ToArray(); + + public async Task Start(string runtime, bool enabled, int? port = null) + { + var binaryDirectory = Path.GetFullPath(Environment.GetEnvironmentVariable($"ORLEANS_DISSEMINATION_{runtime.ToUpperInvariant()}") + ?? throw new InvalidOperationException($"Missing {runtime} published binary directory.")); + var manifest = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(binaryDirectory, "manifest.json")))!; + Assert.Equal(runtime, manifest.Runtime); + if (runtime == "Old") + { + Assert.Equal(Baseline, manifest.SourceRevision); + } + else + { + Assert.NotEqual(Baseline, manifest.SourceRevision); + } + + var name = $"{runtime}-{_sequence++}-{(enabled ? "enabled" : "default")}"; + var node = new SiloProcess( + binaryDirectory, + new(name, Path.GetFileName(Directory), _membershipDirectory, port ?? AvailablePort(), Environment.ProcessId, + enabled, FastRecovery), + Directory, + manifest); + _all.Add(node); + await node.Start(); + Assert.DoesNotContain(_all.Where(other => !ReferenceEquals(other, node) && !other.Stopped), + other => other.Last.Identity.ProcessId == node.Last.Identity.ProcessId); + return node; + } + + public async Task Stabilize() + { + var expected = Active.Select(node => node.Last.Address).Order(StringComparer.Ordinal).ToArray(); + await Eventually("every process has the exact active membership", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => snapshot.ActiveMembers.SequenceEqual(expected)); + }); + } + + public async Task AssertControlRpcs() + { + var nodes = Active; + foreach (var node in nodes) + { + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) + { + var result = await node.Send("echo", peer: peer.Last.Address); + Assert.Equal(peer.Last.Identity.ProcessId, result.RemoteProcessId); + Assert.NotEqual(node.Last.Identity.ProcessId, result.RemoteProcessId); + } + } + } + + public async Task PublishAndConverge() + { + var expected = new Dictionary(StringComparer.Ordinal); + foreach (var node in Active) + { + var previous = node.Last.LoadVersions.GetValueOrDefault(node.Last.Address); + var snapshot = await node.Send("publish"); + Assert.True(snapshot.LoadVersions[snapshot.Address] > previous, "The production publisher did not create a new sample."); + expected.Add(snapshot.Address, snapshot.Load[snapshot.Address]); + } + + await Eventually("exact production load state at every active process", async () => + { + var snapshots = await Task.WhenAll(Active.Select(node => node.Send("snapshot"))); + return snapshots.All(snapshot => MatchesExpectedLoad(snapshot.Load, expected)); + }, expected: expected); + } + + internal static bool MatchesExpectedLoad( + IReadOnlyDictionary actual, + IReadOnlyDictionary expected) => + actual.Count == expected.Count + && expected.All(pair => actual.TryGetValue(pair.Key, out var value) + && StringComparer.Ordinal.Equals(pair.Value, value)); + + public async Task HealPartition(SiloProcess partitioned) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var cancellationToken = deadline.Token; + var phase = "retiring partitioned connections at both endpoints"; + try + { + var nodes = Active; + // Drain admitted middleware, including unfinished handshakes, before reopening any affected endpoint. + await Task.WhenAll(nodes.Select(node => node.Send( + cancellationToken, + "drain-connections", + peer: ReferenceEquals(node, partitioned) ? null : partitioned.Last.Address))); + await Task.WhenAll(nodes.Select(node => node.Send(cancellationToken, "resume-connections"))); + phase = "bidirectional control RPCs after transport healing"; + await Eventually(phase, async () => + { + foreach (var node in nodes) + { + foreach (var peer in nodes.Where(peer => !ReferenceEquals(peer, node))) + { + var result = await node.Send(cancellationToken, "probe-echo", peer: peer.Last.Address); + if (result.RemoteProcessId is not { } processId) + { + Assert.NotNull(result.ProbeError); + return false; + } + + Assert.Equal(peer.Last.Identity.ProcessId, processId); + Assert.NotEqual(node.Last.Identity.ProcessId, processId); + } + } + + return true; + }); + } + catch (OperationCanceledException) when (deadline.IsCancellationRequested && !TestContext.Current.CancellationToken.IsCancellationRequested) + { + await Save("failure-state.json", new { Phase = phase, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + } + + public async Task Eventually(string phase, Func> assertion, TimeSpan? timeout = null, object? expected = null) + { + var clock = Stopwatch.StartNew(); + while (true) + { + if (await assertion()) + { + return; + } + + if (clock.Elapsed > (timeout ?? TimeSpan.FromSeconds(60))) + { + await Save("failure-state.json", new { Phase = phase, Expected = expected, Nodes = _all.Select(node => node.Last) }); + Assert.Fail($"Timed out during {phase}. Exact snapshots/logs: {Directory}"); + } + + await Task.Delay(100, TestContext.Current.CancellationToken); + } + } + + public Task Save(string filename, object value) => + File.WriteAllTextAsync(Path.Combine(Directory, filename), JsonSerializer.Serialize(value, new JsonSerializerOptions { WriteIndented = true })); + + public async ValueTask DisposeAsync() + { + List failures = []; + foreach (var node in _all.AsEnumerable().Reverse()) + { + try + { + await node.DisposeAsync(); + } + catch (Exception exception) + { + failures.Add(exception); + } + } + + await Save("final-processes.json", _all.Select(node => new { node.Configuration, node.Last, node.ShutdownMilliseconds, node.ForcedKill })); + if (failures.Count > 0) + { + throw new AggregateException("One or more silos exceeded shutdown bounds.", failures); + } + } + + private static int AvailablePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + finally + { + listener.Stop(); + } + } +} + +internal sealed class SiloProcess : IAsyncDisposable +{ + private readonly Process _process; + private readonly string _configurationPath; + private readonly string _logPath; + private readonly string _errorPath; + private readonly BuildManifest _manifest; + private readonly ConcurrentDictionary> _pending = new(); + private readonly SemaphoreSlim _commandLock = new(1); + private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task _output = Task.CompletedTask; + private Task _error = Task.CompletedTask; + private int _commandId; + private bool _started; + private bool _disposed; + + public SiloProcess(string binaryDirectory, NodeConfiguration configuration, string artifacts, BuildManifest manifest) + { + Configuration = configuration; + _manifest = manifest; + _configurationPath = Path.Combine(artifacts, configuration.Name + ".configuration.json"); + _logPath = Path.Combine(artifacts, configuration.Name + ".stdout.jsonl"); + _errorPath = Path.Combine(artifacts, configuration.Name + ".stderr.log"); + _process = new Process + { + StartInfo = new ProcessStartInfo(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet") + { + WorkingDirectory = binaryDirectory, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + ArgumentList = { Path.Combine(binaryDirectory, "Orleans.Runtime.Tests.dll"), "--silo", _configurationPath }, + }, + }; + _pending[0] = _ready; + } + + public NodeConfiguration Configuration { get; } + public NodeSnapshot Last { get; private set; } = null!; + public bool Stopped { get; private set; } + public bool ForcedKill { get; private set; } + public double ShutdownMilliseconds { get; private set; } + + public async Task Start() + { + await File.WriteAllTextAsync(_configurationPath, JsonSerializer.Serialize(Configuration)); + Assert.True(_process.Start(), "Failed to launch silo OS process."); + _started = true; + _output = PumpOutput(); + _error = PumpErrors(); + var ready = await _ready.Task.WaitAsync(TimeSpan.FromSeconds(100), TestContext.Current.CancellationToken); + Assert.Null(ready.Error); + Last = ready.Snapshot!; + Assert.Equal(_process.Id, Last.Identity.ProcessId); + Assert.Equal(_manifest.Runtime, Last.Identity.Runtime); + Assert.Equal(_manifest.SourceRevision, Last.Identity.SourceRevision); + Assert.Equal(_manifest.AssemblyName, Last.Identity.AssemblyName); + Assert.Equal(_manifest.AssemblyVersion, Last.Identity.AssemblyVersion); + Assert.Equal(_manifest.Sha256, Last.Identity.Sha256, ignoreCase: true); + Assert.Contains(_manifest.SourceRevision, Last.Identity.InformationalVersion, StringComparison.OrdinalIgnoreCase); + Assert.Equal(_manifest.Runtime == "New", Last.Identity.HasDissemination); + Assert.Equal(Configuration.Enabled, Last.Enabled); + Assert.Equal(Configuration.Enabled, Last.NamespaceEnabled); + var actualDirectory = Path.GetDirectoryName(Path.GetFullPath(Last.Identity.AssemblyPath)); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), actualDirectory); + foreach (var name in new[] { "Orleans.Runtime", "Orleans.Core", "Orleans.Core.Abstractions", "Orleans.Serialization" }) + { + Assert.Contains(name, Last.Identity.Assemblies.Keys); + } + + foreach (var (name, proof) in Last.Identity.Assemblies) + { + Assert.True(_manifest.Assemblies.TryGetValue(name, out var expected), $"Loaded unexpected assembly {name}."); + Assert.Equal(expected!.AssemblyVersion, proof.Version); + Assert.Equal(expected.Sha256, proof.Sha256, ignoreCase: true); + Assert.Equal(Path.GetFullPath(_process.StartInfo.WorkingDirectory), Path.GetDirectoryName(Path.GetFullPath(proof.Path))); + } + } + + public Task Send(string operation, string? peer = null, int count = 1, bool value = false, long version = 0) => + Send(TestContext.Current.CancellationToken, operation, peer, count, value, version); + + public async Task Send(CancellationToken cancellationToken, string operation, string? peer = null, int count = 1, bool value = false, long version = 0) + { + await _commandLock.WaitAsync(cancellationToken); + try + { + var command = new Command(Interlocked.Increment(ref _commandId), operation, peer, count, value, version); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _pending[command.Id] = completed; + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(command).AsMemory(), cancellationToken); + await _process.StandardInput.FlushAsync(cancellationToken); + var response = await completed.Task.WaitAsync(TimeSpan.FromSeconds(70), cancellationToken); + Assert.True(response.Error is null, $"{Configuration.Name}/{operation}: {response.Error}; {_errorPath}"); + Last = response.Snapshot!; + return Last; + } + finally + { + _commandLock.Release(); + } + } + + public async Task Stop() + { + if (Stopped || !_started) + { + return; + } + + var clock = Stopwatch.StartNew(); + try + { + if (!_process.HasExited) + { + // Cleanup has its own bound, independent of a cancelled xUnit test token. + await _process.StandardInput.WriteLineAsync(JsonSerializer.Serialize(new Command(-2, "stop"))); + await _process.StandardInput.FlushAsync(); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(35)); + } + + await Task.WhenAll(_output, _error).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, _process.ExitCode); + } + catch + { + if (!_process.HasExited) + { + ForcedKill = true; + _process.Kill(entireProcessTree: true); + await _process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10)); + } + + throw; + } + finally + { + ShutdownMilliseconds = clock.Elapsed.TotalMilliseconds; + Stopped = true; + } + } + + private async Task PumpOutput() + { + await using var log = File.CreateText(_logPath); + while (await _process.StandardOutput.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + if (!line.StartsWith("HARNESS ", StringComparison.Ordinal)) + { + continue; + } + + var response = JsonSerializer.Deserialize(line["HARNESS ".Length..])!; + if (response.Snapshot is not null) + { + Last = response.Snapshot; + } + + if (_pending.TryRemove(response.Id, out var completed)) + { + completed.TrySetResult(response); + } + } + + foreach (var completion in _pending.Values) + { + completion.TrySetException(new InvalidOperationException($"Silo stdout closed; inspect {_errorPath}.")); + } + } + + private async Task PumpErrors() + { + await using var log = File.CreateText(_errorPath); + while (await _process.StandardError.ReadLineAsync() is { } line) + { + await log.WriteLineAsync(line); + await log.FlushAsync(); + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + try + { + await Stop(); + } + finally + { + _process.Dispose(); + _commandLock.Dispose(); + } + } +} diff --git a/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs new file mode 100644 index 00000000000..152755d62af --- /dev/null +++ b/test/Dissemination.IntegrationHarness/Tests/VersionSkewTests.cs @@ -0,0 +1,378 @@ +using System.Diagnostics; +using System.Text.Json; +using Xunit; +using Xunit.Sdk; +using Xunit.v3; + +[assembly: Parallelization(Mode = ParallelMode.None)] + +namespace Orleans.Dissemination.IntegrationHarness; + +public sealed class VersionSkewTests +{ + [Theory] + [InlineData("Old", false, 4)] + [InlineData("Old", false, 8)] + [InlineData("New", false, 4)] + [InlineData("New", false, 8)] + [InlineData("New", true, 4)] + [InlineData("New", true, 8)] + [Trait("Category", "DisseminationProcess")] + public async Task PartitionHealing_ConfirmsTransportBeforeSinglePublicationRound(string runtime, bool enabled, int size) + { + await using var cluster = new ProcessCluster($"partition-publication-{runtime}-{enabled}-{size}", fastRecovery: false); + for (var index = 0; index < size; index++) + { + await cluster.Start(runtime, enabled); + } + + var nodes = cluster.Active; + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + + for (var iteration = 0; iteration < 3; iteration++) + { + var receiver = nodes[iteration % size]; + await receiver.Send("partition", value: true); + if (iteration == 0) + { + var probe = await nodes[1].Send("probe-echo", peer: receiver.Last.Address); + Assert.Null(probe.RemoteProcessId); + Assert.NotNull(probe.ProbeError); + } + + foreach (var source in nodes.Where(node => !ReferenceEquals(node, receiver))) + { + await source.Send("publish"); + } + + var recovery = Stopwatch.StartNew(); + await cluster.HealPartition(receiver); + await cluster.PublishAndConverge(); + Assert.All(cluster.Active, node => Assert.False(node.Last.Partitioned)); + await cluster.Save($"healed-publication-{iteration}.json", new + { + RecoveryMilliseconds = recovery.Elapsed.TotalMilliseconds, + Nodes = cluster.Active.Select(node => node.Last), + }); + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task PinnedBinaries_RollForward_MixedEnablement_AndRollback() + { + await using var cluster = new ProcessCluster("rolling-upgrade"); + var old = new[] + { + await cluster.Start("Old", enabled: false), + await cluster.Start("Old", enabled: false), + await cluster.Start("Old", enabled: false), + }; + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + + var replacements = new List(); + for (var index = 0; index < old.Length; index++) + { + var replacement = await cluster.Start("New", enabled: index != 0); + replacements.Add(replacement); + Assert.NotEqual(old[index].Last.Identity.Sha256, replacement.Last.Identity.Sha256); + Assert.NotEqual(old[index].Last.Identity.ModuleVersionId, replacement.Last.Identity.ModuleVersionId); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + if (index != 0) + { + // A working stable RPC on this exact connection brackets the unsupported new RPC. + var probe = await replacement.Send("probe", peer: old[index].Last.Address); + Assert.NotNull(probe.ProbeError); + Assert.False(probe.ProbeTimedOut, $"Unknown-RPC compatibility must not be inferred from a timeout: {probe.ProbeError}"); + Assert.Equal(old[index].Last.Identity.ProcessId, + (await replacement.Send("echo", peer: old[index].Last.Address)).RemoteProcessId); + Assert.Contains(old[index].Last.Address, replacement.Last.UnconfirmedPeers); + var beforeFallback = old[index].Last.LoadVersions[replacement.Last.Address]; + await cluster.PublishAndConverge(); + Assert.True(old[index].Last.LoadVersions[replacement.Last.Address] > beforeFallback, + "The old process must receive a fresh exact load sample after the unsupported dissemination RPC."); + Assert.Contains(old[index].Last.Address, replacement.Last.UnconfirmedPeers); + Assert.Empty(old[index].Last.Applies); + await cluster.Save($"old-rpc-probe-{index}.json", probe); + await cluster.Save($"old-rpc-legacy-fallback-{index}.json", new + { + BeforeVersion = beforeFallback, + Sender = replacement.Last, + LegacyReceiver = old[index].Last, + }); + } + + await cluster.Save($"upgrade-{index}-before-stop.json", cluster.Active.Select(node => node.Last)); + await old[index].Stop(); + await cluster.Stabilize(); + await cluster.PublishAndConverge(); + } + + var disabled = replacements[0]; + var enabled = replacements.Skip(1).ToArray(); + await cluster.Eventually("enabled peers confirm support, explicitly disabled peers remain legacy", async () => + { + foreach (var node in enabled) + { + await node.Send("snapshot"); + } + + return enabled.All(node => + node.Last.UnconfirmedPeers.Contains(disabled.Last.Address) + && enabled.Where(peer => !ReferenceEquals(peer, node)).All(peer => + !node.Last.UnconfirmedPeers.Contains(peer.Last.Address))); + }); + var disabledProbe = await enabled[0].Send("probe", peer: disabled.Last.Address); + Assert.Equal("namespace-unsupported", disabledProbe.ProbeError); + await disabled.Send("snapshot"); + Assert.False(disabled.Last.Enabled); + Assert.False(disabled.Last.NamespaceEnabled); + Assert.Equal(0, disabled.Last.BroadcastsSent); + Assert.Equal(0, disabled.Last.OutgoingRepairs); + Assert.Empty(disabled.Last.Applies); + + // The same original binary is reintroduced, not a flag on the new binary. + for (var index = replacements.Count - 1; index >= 0; index--) + { + var rolledBack = await cluster.Start("Old", enabled: false, port: old[index].Configuration.SiloPort); + Assert.Equal(old[index].Last.Identity.Sha256, rolledBack.Last.Identity.Sha256); + Assert.NotEqual(replacements[index].Last.Identity.ProcessId, rolledBack.Last.Identity.ProcessId); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + await replacements[index].Stop(); + await cluster.Stabilize(); + await cluster.PublishAndConverge(); + await cluster.Save($"rollback-{index}.json", cluster.Active.Select(node => node.Last)); + } + + Assert.All(cluster.Active, node => Assert.Equal("Old", node.Last.Identity.Runtime)); + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task PartitionSnapshotAndHeartbeat_RequireDisseminationProvenance() + { + await using var cluster = new ProcessCluster("partition-snapshot-heartbeat"); + var origin = await cluster.Start("New", enabled: true); + var relay = await cluster.Start("New", enabled: true); + var receiver = await cluster.Start("New", enabled: true); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + await cluster.PublishAndConverge(); + await cluster.Eventually("identical membership before isolation", async () => + { + var snapshots = await Task.WhenAll(cluster.Active.Select(node => node.Send("refresh"))); + return snapshots.All(snapshot => SameMembership(snapshots[0], snapshot)); + }); + try + { + foreach (var node in cluster.Active) + { + var isolated = await node.Send("isolate-membership", value: true); + Assert.True(isolated.LegacyGossipSuppressed); + Assert.True(isolated.MembershipReadsFrozen); + } + + Assert.False((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + foreach (var node in cluster.Active) + { + var quiesced = await node.Send("block-tree", value: true); + Assert.NotNull(quiesced.TreeGate); + Assert.True(quiesced.TreeGate.Blocked); + Assert.Equal(0, quiesced.TreeGate.InFlight); + } + + // Verify the native PushBroadcast RPC is intercepted, not just that a local flag was set. + Assert.True((await origin.Send("probe-tree", peer: receiver.Last.Address)).TreeProbeRejected); + await Task.WhenAll(cluster.Active.Select(node => node.Send("snapshot"))); + var before = cluster.Active.ToDictionary(node => node.Last.Identity.ProcessId, node => node.Last); + var oldVersion = receiver.Last.MembershipVersion; + await receiver.Send("partition", value: true); + await relay.Send("partition", value: true); + var source = await origin.Send("membership-update", version: oldVersion); + Assert.Equal(oldVersion + 1, source.MembershipVersion); + Assert.Equal(0, source.RepairFromVersion); + var blocked = await receiver.Send("snapshot"); + Assert.Equal(oldVersion, blocked.MembershipVersion); + Assert.Equal(oldVersion, (await relay.Send("snapshot")).MembershipVersion); + Assert.False(SameMembership(source, blocked)); + await receiver.Send("partition", value: false); + + await cluster.Eventually("anti-entropy-only full snapshot repair after a real transport partition", async () => + { + await receiver.Send("snapshot"); + return SameMembership(source, receiver.Last) + && receiver.Last.Applies.Any(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == source.MembershipVersion + && evidence.Result == "Applied" && evidence.Peer is not null); + }); + await relay.Send("partition", value: false); + await cluster.Eventually("relay convergence after the receiver's full repair", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(source, receiver.Last) && SameMembership(source, relay.Last); + }); + await AssertNoTreeAdmissions(cluster, before); + Assert.True(receiver.Last.MembershipReadsFrozen); + Assert.True(receiver.Last.LegacyGossipSuppressed); + await cluster.Save("snapshot-recovery.json", new { Before = before, After = cluster.Active.Select(node => node.Last) }); + + var priorHeartbeatApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "membership" && evidence.Result == "Applied" && evidence.ToVersion == source.MembershipVersion); + var heartbeat = await origin.Send("membership-heartbeat"); + Assert.Equal(source.MembershipVersion, heartbeat.MembershipVersion); + Assert.NotNull(heartbeat.HeartbeatTicks); + Assert.False(SameMembership(source, heartbeat)); + await cluster.Eventually("same-version heartbeat fingerprint repair without Publish, gossip, or table reads", async () => + { + await receiver.Send("snapshot"); + await relay.Send("snapshot"); + return SameMembership(heartbeat, receiver.Last) && SameMembership(heartbeat, relay.Last) + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "membership" + && evidence.FromVersion == 0 && evidence.ToVersion == heartbeat.MembershipVersion + && evidence.Result == "Applied") > priorHeartbeatApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("same-version-heartbeat-recovery.json", cluster.Active.Select(node => node.Last)); + + var expectedLoad = receiver.Last.Load[origin.Last.Address]; + var expectedVersion = receiver.Last.LoadVersions[origin.Last.Address]; + var priorLoadApplications = receiver.Last.Applies.Count(evidence => + evidence.Namespace == "load" && evidence.Result == "Applied" && evidence.ToVersion == expectedVersion); + await receiver.Send("forget-load", peer: origin.Last.Address); + Assert.DoesNotContain(origin.Last.Address, receiver.Last.Load.Keys); + await cluster.Eventually("load repair reaches exact state with no new publication or direct refresh", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.Load.TryGetValue(origin.Last.Address, out var actual) + && actual == expectedLoad + && receiver.Last.Applies.Count(evidence => evidence.Namespace == "load" + && evidence.FromVersion == 0 && evidence.ToVersion == expectedVersion + && evidence.Result == "Applied") > priorLoadApplications; + }); + await AssertNoTreeAdmissions(cluster, before); + await cluster.Save("load-full-repair.json", receiver.Last); + } + finally + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + foreach (var node in cluster.Active) + { + await node.Send(cleanup.Token, "isolate-membership", value: false); + await node.Send(cleanup.Token, "block-tree", value: true); + if (node.Last.Partitioned) + { + await node.Send(cleanup.Token, "partition", value: false); + } + + await node.Send(cleanup.Token, "block-tree", value: false); + } + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task RuntimeStatisticsComparison_ObservesPublicReadonlyFieldMutation() + { + foreach (var runtime in new[] { "Old", "New" }) + { + await using var cluster = new ProcessCluster($"statistics-projection-{runtime}"); + var node = await cluster.Start(runtime, enabled: false); + var result = await node.Send("verify-load-comparison"); + var probe = Assert.IsType(result.ComparisonProbe); + Assert.NotEqual(probe.Before, probe.After); + using var original = JsonDocument.Parse(probe.Before); + using var changed = JsonDocument.Parse(probe.After); + var originalEnvironment = original.RootElement.GetProperty("EnvironmentStatistics"); + var changedEnvironment = changed.RootElement.GetProperty("EnvironmentStatistics"); + Assert.Contains("RawCpuUsagePercentage", probe.PublicFields); + Assert.All(probe.PublicFields, field => + { + Assert.True(originalEnvironment.TryGetProperty(field, out _), $"Missing original field {field}"); + Assert.True(changedEnvironment.TryGetProperty(field, out _), $"Missing changed field {field}"); + }); + Assert.Equal(20, originalEnvironment.GetProperty("RawCpuUsagePercentage").GetSingle()); + Assert.Equal(35, changedEnvironment.GetProperty("RawCpuUsagePercentage").GetSingle()); + await cluster.Save("statistics-field-mutation.json", probe); + } + } + + [Fact] + [Trait("Category", "DisseminationProcess")] + public async Task RequiredCancellationAndPartitionedShutdown_AreBounded() + { + await using var cluster = new ProcessCluster("bounded-shutdown"); + var sender = await cluster.Start("New", enabled: true); + var receiver = await cluster.Start("New", enabled: true); + await cluster.Stabilize(); + await cluster.AssertControlRpcs(); + var previousStarted = receiver.Last.StartedControlCalls; + var previousCancelled = receiver.Last.CancelledControlCalls; + var previousSignals = receiver.Last.ControlCancellationSignals; + await sender.Send("start-cancel-rpc", peer: receiver.Last.Address); + await cluster.Eventually("the remote Hold has entered with a live token before cancellation is issued", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.PendingControlCalls == 1 + && receiver.Last.StartedControlCalls == previousStarted + 1 + && receiver.Last.CancelledControlCalls == previousCancelled + && receiver.Last.ControlCancellationSignals == previousSignals + && receiver.Last.ControlTokenCanBeCanceled + && !receiver.Last.ControlTokenCancelledOnEntry + && receiver.Last.ControlStartedAtUtc.HasValue; + }); + await sender.Send("snapshot"); + Assert.NotNull(sender.Last.OutboundControlCall); + Assert.False(sender.Last.OutboundControlCall.CancellationRequested); + Assert.False(sender.Last.OutboundControlCall.RawTaskCompleted); + await cluster.Save("cancellation-started.json", new { Sender = sender.Last, Receiver = receiver.Last }); + await sender.Send("cancel-started-rpc"); + Assert.True(sender.Last.OutboundControlCall!.CancellationRequested); + Assert.True(sender.Last.OutboundControlCall.RawTaskCompleted); + await cluster.Eventually("the remote cancellable test RPC releases its server-side request", async () => + { + await receiver.Send("snapshot"); + return receiver.Last.PendingControlCalls == 0 + && receiver.Last.StartedControlCalls == previousStarted + 1 + && receiver.Last.CancelledControlCalls == previousCancelled + 1 + && receiver.Last.ControlCancellationSignals == previousSignals + 1 + && receiver.Last.ControlCancellationObservedAtUtc.HasValue; + }); + await cluster.Save("cancellation-observed.json", new { Sender = sender.Last, Receiver = receiver.Last }); + await receiver.Send("partition", value: true); + await sender.Send("publish", count: 4); + await sender.Stop(); + Assert.False(sender.ForcedKill); + Assert.InRange(sender.ShutdownMilliseconds, 0, 35_000); + await receiver.Send("partition", value: false); + await receiver.Stop(); + Assert.False(receiver.ForcedKill); + Assert.InRange(receiver.ShutdownMilliseconds, 0, 35_000); + } + + internal static bool SameMembership(NodeSnapshot left, NodeSnapshot right) => + left.MembershipVersion == right.MembershipVersion && left.Membership.SequenceEqual(right.Membership); + + private static async Task AssertNoTreeAdmissions(ProcessCluster cluster, Dictionary before) + { + foreach (var node in cluster.Active) + { + var snapshot = await node.Send("snapshot"); + Assert.NotNull(snapshot.TreeGate); + Assert.True(snapshot.TreeGate.Blocked); + Assert.Equal(0, snapshot.TreeGate.InFlight); + Assert.Equal(before[snapshot.Identity.ProcessId].TreeGate!.Admitted, snapshot.TreeGate.Admitted); + Assert.True(snapshot.LegacyGossipSuppressed); + Assert.True(snapshot.MembershipReadsFrozen); + } + } + +} diff --git a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs index e945bf8d73f..c8733d68380 100644 --- a/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs +++ b/test/Orleans.Core.Tests/Membership/MembershipGossiperTests.cs @@ -8,6 +8,7 @@ using Orleans.Core.Diagnostics; using Orleans.Internal; using Orleans.Runtime; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.MembershipService; using Orleans.Runtime.Scheduler; using TestExtensions; @@ -18,9 +19,252 @@ namespace NonSilo.Tests.Membership; [TestCategory("BVT"), TestCategory("Membership")] [TestSuite("BVT")] [TestProvider("None")] -[TestArea("Runtime")] +[TestArea("Dissemination")] public class MembershipGossiperTests { + [Fact] + public async Task GossipToRemoteSilos_EligibleLocalStatus_StartsDirectAndDisseminationBeforeEitherCompletes() + { + var directStarted = NewBarrier(); + var releaseDirect = NewBarrier(); + var directCompleted = NewBarrier(); + var disseminationStarted = NewBarrier(); + var releaseDissemination = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + DisseminationKey? publishedKey = null; + long? publishedVersion = null; + CancellationToken publishedToken = default; + + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directStarted.TrySetResult(); + await releaseDirect.Task.WaitAsync(cancellationToken); + directCompleted.TrySetResult(); + }, + async (key, version, cancellationToken) => + { + publishedKey = key; + publishedVersion = version; + publishedToken = cancellationToken; + disseminationStarted.TrySetResult(); + return await releaseDissemination.Task.WaitAsync(cancellationToken); + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.False(gossipTask.IsCompleted); + Assert.False(directCompleted.Task.IsCompleted); + Assert.Equal(1, rig.DisseminationServiceResolutionCount); + } + finally + { + releaseDirect.TrySetResult(); + releaseDissemination.TrySetResult(true); + } + + await gossipTask; + await directCompleted.Task.WaitAsync(TestContext.Current.CancellationToken); + + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + Assert.Equal(DisseminationKey.Default, publishedKey); + Assert.Equal(snapshot.Version.Value, publishedVersion); + Assert.Equal(TestContext.Current.CancellationToken, publishedToken); + } + + [Fact] + public async Task GossipToRemoteSilos_IneligibleLocalStatus_SkipsDisseminationAndCompletesDirectGossip() + { + MembershipTableSnapshot? deliveredSnapshot = null; + using var rig = CreateDisseminationTestRig( + (snapshot, _) => + { + deliveredSnapshot = snapshot; + return Task.CompletedTask; + }, + (_, _, _) => throw new Xunit.Sdk.XunitException("Ineligible membership must not publish via dissemination.")); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Dead); + + await rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Dead, TestContext.Current.CancellationToken); + + Assert.Same(snapshot, deliveredSnapshot); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + Assert.Equal(0, rig.DisseminationServiceResolutionCount); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + } + + [Fact] + public async Task GossipToRemoteSilos_CallerCancellation_CancelsDirectGossipAndDisseminationPublish() + { + var directStarted = NewBarrier(); + var disseminationStarted = NewBarrier(); + CancellationToken directToken = default; + CancellationToken publishedToken = default; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directToken = cancellationToken; + directStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + async (_, _, cancellationToken) => + { + publishedToken = cancellationToken; + disseminationStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Stopping, cancellation.Token); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, cancellation.Token); + } + finally + { + cancellation.Cancel(); + } + + await Assert.ThrowsAnyAsync(() => gossipTask); + Assert.True(gossipTask.IsCanceled); + Assert.Equal(cancellation.Token, directToken); + Assert.Equal(cancellation.Token, publishedToken); + Assert.True(directToken.IsCancellationRequested); + Assert.True(publishedToken.IsCancellationRequested); + } + + [Fact] + public async Task GossipToRemoteSilos_PreCanceled_DoesNotSendOrPublish() + { + using var cancellation = new CancellationTokenSource(); + using var rig = CreateDisseminationTestRig( + (_, _) => Task.CompletedTask, + (_, _, _) => ValueTask.FromResult(true)); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Active, cancellation.Token)); + + Assert.Empty(rig.RemoteMembershipService.ReceivedCalls()); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + } + + [Fact] + public async Task MembershipGossiperStartsDirectGossipBeforeDissemination() + { + var directStarted = NewBarrier(); + var releaseDirect = NewBarrier(); + var disseminationStarted = NewBarrier(); + var releaseDissemination = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resolutionOrder = new List(); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directStarted.TrySetResult(); + await releaseDirect.Task.WaitAsync(cancellationToken); + }, + async (_, _, cancellationToken) => + { + disseminationStarted.TrySetResult(); + return await releaseDissemination.Task.WaitAsync(cancellationToken); + }, + resolutionOrder); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + try + { + await Task.WhenAll(directStarted.Task, disseminationStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.Equal(["MembershipSystemTargetResolved", "DisseminationServiceResolved"], resolutionOrder); + } + finally + { + releaseDirect.TrySetResult(); + releaseDissemination.TrySetResult(true); + } + + await gossipTask; + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipGossiperDirectDeliveryRemainsAuthoritativeWhenDisseminationFails() + { + var disseminationFailure = new InvalidOperationException("Simulated dissemination failure."); + DisseminationKey? publishedKey = null; + long? publishedVersion = null; + using var rig = CreateDisseminationTestRig( + (_, _) => Task.CompletedTask, + (key, version, _) => + { + publishedKey = key; + publishedVersion = version; + throw disseminationFailure; + }); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Active); + + await rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.ShuttingDown, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationKey.Default, publishedKey); + Assert.Equal(snapshot.Version.Value, publishedVersion); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, TestContext.Current.CancellationToken); + await rig.DisseminationService.Received(1).Publish( + Arg.Any(), DisseminationKey.Default, snapshot.Version.Value, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipGossiperPreservesEligibilityAndCallerCancellation() + { + var directStarted = NewBarrier(); + CancellationToken directToken = default; + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var rig = CreateDisseminationTestRig( + async (_, cancellationToken) => + { + directToken = cancellationToken; + directStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, + (_, _, _) => throw new Xunit.Sdk.XunitException("Ineligible membership must not publish via dissemination.")); + var snapshot = CreateSnapshot(rig.LocalSilo, rig.RemoteSilo, SiloStatus.Dead); + var gossipTask = rig.Gossiper.GossipToRemoteSilos( + [rig.RemoteSilo], snapshot, rig.LocalSilo, SiloStatus.Dead, cancellation.Token); + + try + { + await directStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + } + finally + { + cancellation.Cancel(); + } + + var exception = await Assert.ThrowsAnyAsync(() => gossipTask); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(cancellation.Token, directToken); + Assert.Equal(0, rig.DisseminationServiceResolutionCount); + Assert.Empty(rig.DisseminationService.ReceivedCalls()); + await rig.RemoteMembershipService.Received(1).MembershipChangeNotification(snapshot, cancellation.Token); + } + [Theory] [InlineData("Gossip")] [InlineData("Probe")] @@ -158,10 +402,124 @@ private static TestRig CreateTestRig() }); var serviceProvider = services.BuildServiceProvider(); return new(serviceProvider, - new MembershipGossiper(serviceProvider, NullLogger.Instance), + new MembershipGossiper(serviceProvider, localDetails, NullLogger.Instance), new RemoteSiloProber(serviceProvider), remote, localSilo, remoteSilo); } + private static DisseminationTestRig CreateDisseminationTestRig( + Func directGossip, + Func> disseminationPublish, + List? resolutionOrder = null) + { + var localSilo = SiloAddress.FromParsableString("127.0.0.1:100@100"); + var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); + var localDetails = Substitute.For(); + localDetails.SiloAddress.Returns(localSilo); + var remote = Substitute.For(); + remote.MembershipChangeNotification(Arg.Any(), Arg.Any()) + .Returns(call => directGossip(call.ArgAt(0), call.ArgAt(1))); + var grainFactory = Substitute.For(); + grainFactory.GetSystemTarget(Constants.MembershipServiceType, Arg.Any()).Returns(remote); + var membershipManager = Substitute.For(); + membershipManager.CurrentSnapshot.Returns(CreateSnapshot(localSilo, remoteSilo, SiloStatus.Active)); + var options = Substitute.For>(); + options.CurrentValue.Returns(new ClusterMembershipOptions + { + Dissemination = new DisseminationNamespaceOptions { Enabled = true }, + }); + var disseminationNamespace = new MembershipDisseminationNamespace(membershipManager, options, serializer: null!); + var disseminationService = Substitute.For(); + disseminationService.Publish( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => disseminationPublish( + call.ArgAt(1), call.ArgAt(2), call.ArgAt(3))); + var disseminationServiceResolutionCount = 0; + var services = new ServiceCollection(); + services.AddMetrics(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(localDetails); + services.AddSingleton(disseminationNamespace); + services.AddSingleton(_ => + { + disseminationServiceResolutionCount++; + resolutionOrder?.Add("DisseminationServiceResolved"); + return disseminationService; + }); + services.AddSingleton(provider => + { + resolutionOrder?.Add("MembershipSystemTargetResolved"); + var shared = new SystemTargetShared( + runtimeClient: null!, + localDetails, + NullLoggerFactory.Instance, + Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + timerRegistry: null!, + new ActivationDirectory(provider.GetRequiredService()), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService()); + return new MembershipSystemTarget( + membershipManager, + NullLogger.Instance, + grainFactory, + provider.GetRequiredService(), + shared, + TimeProvider.System); + }); + var serviceProvider = services.BuildServiceProvider(); + return new( + serviceProvider, + new MembershipGossiper(serviceProvider, localDetails, NullLogger.Instance), + remote, + disseminationService, + localSilo, + remoteSilo, + () => disseminationServiceResolutionCount); + } + + private static MembershipTableSnapshot CreateSnapshot(SiloAddress localSilo, SiloAddress remoteSilo, SiloStatus localStatus) + { + var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + return new( + new MembershipVersion(7), + ImmutableDictionary.Empty + .Add(localSilo, CreateEntry(localSilo, localStatus, startTime)) + .Add(remoteSilo, CreateEntry(remoteSilo, SiloStatus.Active, startTime))); + } + + private static MembershipEntry CreateEntry(SiloAddress silo, SiloStatus status, DateTime startTime) => new() + { + SiloAddress = silo, + Status = status, + HostName = "localhost", + SiloName = silo.ToParsableString(), + StartTime = startTime, + IAmAliveTime = startTime, + }; + + private static TaskCompletionSource NewBarrier() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private sealed record DisseminationTestRig( + ServiceProvider ServiceProvider, + MembershipGossiper Gossiper, + IMembershipService RemoteMembershipService, + IDisseminationService DisseminationService, + SiloAddress LocalSilo, + SiloAddress RemoteSilo, + Func DisseminationServiceResolutions) : IDisposable + { + public int DisseminationServiceResolutionCount => DisseminationServiceResolutions(); + + public void Dispose() => ServiceProvider.Dispose(); + } + private sealed record TestRig( ServiceProvider ServiceProvider, MembershipGossiper Gossiper, diff --git a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs index 00dd3d6ce4d..8419c0c9ea5 100644 --- a/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs +++ b/test/Orleans.Core.Tests/Runtime/DeploymentLoadPublisherTests.cs @@ -2,13 +2,16 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; using NSubstitute; using Orleans; using Orleans.Configuration; using Orleans.Core.Diagnostics; using Orleans.Internal; using Orleans.Runtime; +using Orleans.Runtime.Dissemination; using Orleans.Runtime.Scheduler; +using Orleans.Serialization; using Orleans.Statistics; using Orleans.Timers; using TestExtensions; @@ -22,6 +25,202 @@ namespace NonSilo.Tests.Runtime; [TestArea("Runtime")] public class DeploymentLoadPublisherTests { + [Theory] + [InlineData(null)] + [InlineData(false)] + public async Task PublishStatistics_DisabledByDefaultOrExplicitly_PreservesDirectPublication(bool? configured) + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: configured); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + Assert.Empty(rig.Dissemination.ReceivedCalls()); + Assert.False(rig.ServiceProvider.GetRequiredService>().Value.Dissemination.Enabled); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PublishStatistics_ExplicitlyEnabled_ConfirmedPeersUseDissemination(bool confirmed) + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); + rig.Dissemination.GetUnconfirmedPeers(Arg.Any()) + .Returns(confirmed ? [] : new[] { remoteSilo }); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.Dissemination.Received(1).PublishAggregated( + Arg.Any(), rig.LocalSilo, + rig.Publisher.LocalRuntimeStatistics.DateTime.Ticks, Arg.Any()); + await rig.DirectTarget.Received(confirmed ? 0 : 1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PublishStatistics_DisseminationCancelsIndependently_FallsBackToDirectPublication() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + rig.Dissemination.PublishAggregated( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => ValueTask.FromCanceled(cancellation.Token)); + + await rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PublishStatistics_CallerCancels_DoesNotFallBackToDirectPublication() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + using var cancellation = new CancellationTokenSource(); + rig.Dissemination.PublishAggregated( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + cancellation.Cancel(); + return ValueTask.FromCanceled(call.ArgAt(3)); + }); + + await Assert.ThrowsAnyAsync(() => rig.Publisher.PublishStatistics(cancellation.Token)); + + Assert.Empty(rig.DirectTarget.ReceivedCalls()); + } + + [Fact] + public async Task PublishStatistics_DeadlineCancelsNativePublicationBeforeDirectDelivery() + { + using var rig = CreateTestRig(TimeSpan.FromSeconds(5), enableDissemination: true); + var timeProvider = (FakeTimeProvider)rig.ServiceProvider.GetRequiredService(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = false; + rig.Dissemination.PublishAggregated( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => new ValueTask(PublishAsync(call.ArgAt(3)))); + var publication = rig.Publisher.PublishStatistics(TestContext.Current.CancellationToken); + var token = await started.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + timeProvider.Advance(TimeSpan.FromSeconds(5)); + Assert.False(token.IsCancellationRequested); + Assert.False(publication.IsCompleted); + timeProvider.Advance(TimeSpan.FromSeconds(5)); + await publication.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.True(token.IsCancellationRequested); + Assert.True(completed); + await rig.DirectTarget.Received(1).UpdateRuntimeStatistics( + rig.LocalSilo, rig.Publisher.LocalRuntimeStatistics, TestContext.Current.CancellationToken); + + async Task PublishAsync(CancellationToken cancellationToken) + { + started.SetResult(cancellationToken); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new(true, TimeSpan.FromSeconds(5)); + } + finally + { + completed = true; + } + } + } + + [Fact] + public async Task ApplyLoadStatistics_UsesPublisherSchedulerAndSuppressesDisseminationDuplicates() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var listener = Substitute.For(); + listener.When(value => value.SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any())) + .Do(_ => + { + Assert.Same(rig.Publisher, RuntimeContext.Current); + rig.Publisher.UnsubscribeStatisticsChangeEvents(listener); + }); + var remaining = Substitute.For(); + rig.Publisher.SubscribeToStatisticsChangeEvents(listener); + rig.Publisher.SubscribeToStatisticsChangeEvents(remaining); + var ns = rig.ServiceProvider.GetRequiredService(); + var value = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + + var applied = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + var duplicate = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, applied); + Assert.Equal(DisseminationApplyResult.Duplicate, duplicate); + listener.Received(1).SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any()); + remaining.Received(1).SiloStatisticsChangeNotification(rig.LocalSilo, Arg.Any()); + Assert.Equal(rig.InitialStatistics.DateTime, rig.Publisher.PeriodicStatistics[rig.LocalSilo].DateTime); + } + + [Fact] + public async Task UpdateRuntimeStatistics_ExplicitlyDisabled_PreservesDuplicateNotifications() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var listener = Substitute.For(); + rig.Publisher.SubscribeToStatisticsChangeEvents(listener); + + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + + listener.Received(2).SiloStatisticsChangeNotification(rig.LocalSilo, rig.InitialStatistics); + Assert.Same(rig.InitialStatistics, rig.Publisher.PeriodicStatistics[rig.LocalSilo]); + } + + [Theory] + [InlineData(SiloStatus.Dead)] + [InlineData(SiloStatus.None)] + public async Task ApplyLoadStatistics_DepartedGenerationRemainsRemoved(SiloStatus status) + { + using var rig = CreateTestRig(TimeSpan.Zero); + await rig.Publisher.UpdateRuntimeStatistics(rig.LocalSilo, rig.InitialStatistics, TestContext.Current.CancellationToken); + var oracle = rig.ServiceProvider.GetRequiredService(); + oracle.GetApproximateSiloStatus(rig.LocalSilo).Returns(status); + rig.Publisher.SiloStatusChangeNotification(rig.LocalSilo, SiloStatus.Dead); + var ns = rig.ServiceProvider.GetRequiredService(); + + var result = await ns.ApplyValueAsync( + ns.CreateValue(rig.LocalSilo, rig.InitialStatistics), TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Rejected, result); + Assert.False(rig.Publisher.PeriodicStatistics.ContainsKey(rig.LocalSilo)); + Assert.Equal(0, ns.GetVersion(rig.LocalSilo)); + } + + [Fact] + public async Task ApplyLoadStatistics_PreCanceledToken_PreservesLocalState() + { + using var rig = CreateTestRig(TimeSpan.Zero); + using var cancellation = new CancellationTokenSource(); + var ns = rig.ServiceProvider.GetRequiredService(); + var value = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await ns.ApplyValueAsync(value, cancellation.Token)); + + Assert.Empty(rig.Publisher.PeriodicStatistics); + } + + [Fact] + public async Task ApplyLoadStatistics_RequiresFullValue() + { + using var rig = CreateTestRig(TimeSpan.Zero); + var ns = rig.ServiceProvider.GetRequiredService(); + var fullValue = ns.CreateValue(rig.LocalSilo, rig.InitialStatistics); + var delta = new DisseminationValue(fullValue.Key, 1, fullValue.ToVersion, fullValue.Payload); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Empty(rig.Publisher.PeriodicStatistics); + } + [Theory] [InlineData(-5000)] [InlineData(-1)] @@ -135,9 +334,10 @@ public async Task PublishStatistics_CancellationPreservesNativeCompletion() } [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RefreshClusterStatistics_Cancellation_CancelsNativeRequests(bool useLinkedReceiverToken) + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + public async Task RefreshClusterStatistics_Cancellation_PreservesNativeOutcome(bool completeSuccessfully, bool useLinkedReceiverToken) { using var rig = CreateTestRig(TimeSpan.Zero); using var cancellation = new CancellationTokenSource(); @@ -148,11 +348,21 @@ public async Task RefreshClusterStatistics_Cancellation_CancelsNativeRequests(bo var requestToken = await requested.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); cancellation.Cancel(); - await Assert.ThrowsAnyAsync( - () => refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); Assert.Equal(cancellation.Token, requestToken); Assert.True(requestToken.IsCancellationRequested); - Assert.Empty(rig.Publisher.PeriodicStatistics); + if (completeSuccessfully) + { + await refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(refresh.IsCompletedSuccessfully); + Assert.Equal(2, rig.Publisher.PeriodicStatistics.Count); + Assert.All(rig.Publisher.PeriodicStatistics.Values, statistics => Assert.Same(rig.InitialStatistics, statistics)); + } + else + { + await Assert.ThrowsAnyAsync( + () => refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + Assert.Empty(rig.Publisher.PeriodicStatistics); + } async Task ReadStatistics(CancellationToken token) { @@ -162,37 +372,17 @@ async Task ReadStatistics(CancellationToken token) requested.TrySetResult(token); } - await Task.Delay(Timeout.InfiniteTimeSpan, useLinkedReceiverToken ? receiverCancellation.Token : token); - return rig.InitialStatistics; - } - } - - [Fact] - public async Task RefreshClusterStatistics_CompletedRequests_PreserveNativeResults() - { - using var rig = CreateTestRig(TimeSpan.Zero); - using var cancellation = new CancellationTokenSource(); - var requested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var startedRequests = 0; - rig.Control.GetRuntimeStatistics(Arg.Any()).Returns(async call => - { - if (Interlocked.Increment(ref startedRequests) == 2) + if (completeSuccessfully) + { + await token.WhenCancelled(); + } + else { - requested.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, useLinkedReceiverToken ? receiverCancellation.Token : token); } - await call.ArgAt(0).WhenCancelled(); return rig.InitialStatistics; - }); - var refresh = rig.Publisher.RefreshClusterStatistics(cancellation.Token); - await requested.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - cancellation.Cancel(); - await refresh.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.True(refresh.IsCompletedSuccessfully); - Assert.Equal(2, rig.Publisher.PeriodicStatistics.Count); - Assert.All(rig.Publisher.PeriodicStatistics.Values, statistics => Assert.Same(rig.InitialStatistics, statistics)); + } } [Fact] @@ -234,7 +424,10 @@ public async Task StatisticsSubscribers_DeliverToRemainingSubscribersBeforeSurfa Assert.Same(rig.InitialStatistics, rig.Publisher.PeriodicStatistics[rig.LocalSilo]); } - private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timerRegistry = null) + private static TestRig CreateTestRig( + TimeSpan refreshTime, + ITimerRegistry? timerRegistry = null, + bool? enableDissemination = false) { var localSilo = SiloAddress.FromParsableString("127.0.0.1:100@100"); var remoteSilo = SiloAddress.FromParsableString("127.0.0.1:200@100"); @@ -258,9 +451,15 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer var control = Substitute.For(); control.GetRuntimeStatistics(Arg.Any()).Returns(Task.FromResult(initialStatistics)); grainFactory.GetSystemTarget(Constants.SiloControlType, Arg.Any()).Returns(control); + var dissemination = Substitute.For(); + dissemination.PublishAggregated( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ValueTask.FromResult(new DisseminationPublicationReceipt(true, refreshTime))); var services = new ServiceCollection(); + services.AddSerializer(); services.AddMetrics(); + services.AddSingleton(new FakeTimeProvider()); services.AddSingleton(NullLoggerFactory.Instance); services.AddSingleton(); services.AddSingleton(); @@ -271,11 +470,18 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer services.AddSingleton(localDetails); services.AddSingleton(statusOracle); services.AddSingleton(grainFactory); + services.AddSingleton(dissemination); services.AddSingleton(Substitute.For()); services.AddSingleton(environmentStatistics); services.AddSingleton>(loadSheddingOptions); - services.AddOptions().Configure( - options => options.DeploymentLoadPublisherRefreshTime = refreshTime); + services.AddOptions().Configure(options => + { + options.DeploymentLoadPublisherRefreshTime = refreshTime; + if (enableDissemination is { } enabled) + { + options.Dissemination.Enabled = enabled; + } + }); services.AddSingleton(); services.AddSingleton(serviceProvider => new SystemTargetShared( runtimeClient: null!, @@ -290,8 +496,9 @@ private static TestRig CreateTestRig(TimeSpan refreshTime, ITimerRegistry? timer serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService())); services.AddSingleton(); + services.AddSingleton(); var serviceProvider = services.BuildServiceProvider(); - return new(serviceProvider, serviceProvider.GetRequiredService(), directTarget, control, initialStatistics, localSilo); + return new(serviceProvider, serviceProvider.GetRequiredService(), directTarget, control, initialStatistics, localSilo, dissemination); } private sealed record TestRig( @@ -300,7 +507,8 @@ private sealed record TestRig( IDeploymentLoadPublisher DirectTarget, ISiloControl Control, SiloRuntimeStatistics InitialStatistics, - SiloAddress LocalSilo) : IDisposable + SiloAddress LocalSilo, + IDisseminationService Dissemination) : IDisposable { public void Dispose() => ServiceProvider.Dispose(); } diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs new file mode 100644 index 00000000000..905226e19d4 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationAggregationOptionsTests.cs @@ -0,0 +1,260 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using Microsoft.Extensions.Options; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationAggregationOptionsTests +{ + [Fact] + public void DefaultsAreBoundedAndOptIn() + { + var options = new DisseminationOptions(); + var load = new DeploymentLoadPublisherOptions(); + var membership = new ClusterMembershipOptions(); + + Assert.Equal(8, options.Overlay.AggregationFanOutFactor); + Assert.Equal(8192, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(options.MaxBatchItems, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(options.MaxBatchBytes, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(8192, load.Dissemination.MaxPendingItemCount); + Assert.Equal(TimeSpan.FromSeconds(5), load.Dissemination.ExpectedUpdateCadence); + Assert.Equal(TimeSpan.FromSeconds(1), load.DeploymentLoadPublisherRefreshTime); + Assert.False(options.Enabled); + Assert.False(new DisseminationNamespaceOptions().Enabled); + Assert.False(load.Dissemination.Enabled); + Assert.False(membership.Dissemination.Enabled); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + Assert.Equal(ValidateOptionsResult.Success, new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, load)); + } + + [Theory] + [InlineData(1, 1, 1)] + [InlineData(int.MaxValue, int.MaxValue, int.MaxValue)] + public void ValidatorAcceptsPositiveBounds(int fanout, int items, int bytes) + { + var options = new DisseminationOptions(); + options.Overlay.AggregationFanOutFactor = fanout; + options.Overlay.MaxAntiEntropyBatchItems = items; + options.Overlay.MaxAntiEntropyBatchBytes = bytes; + + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAggregationFanOut(int value) + { + var options = new DisseminationOptions(); + options.Overlay.AggregationFanOutFactor = value; + + AssertValidationFailure(options, "AggregationFanOutFactor must be greater than 0."); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAntiEntropyItemLimit(int value) + { + var options = new DisseminationOptions(); + options.Overlay.MaxAntiEntropyBatchItems = value; + + AssertValidationFailure(options, "MaxAntiEntropyBatchItems must be greater than 0."); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(int.MinValue)] + public void ValidatorRejectsNonPositiveAntiEntropyByteLimit(int value) + { + var options = new DisseminationOptions(); + options.Overlay.MaxAntiEntropyBatchBytes = value; + + AssertValidationFailure(options, "MaxAntiEntropyBatchBytes must be greater than 0."); + } + + [Fact] + public void AntiEntropyLimitsAreIndependentOfBroadcastLimits() + { + var options = new DisseminationOptions { MaxBatchItems = 1, MaxBatchBytes = 1 }; + + Assert.Equal(8192, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(1048576, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + + options.MaxBatchItems = 16384; + options.MaxBatchBytes = 2 * 1024 * 1024; + options.Overlay.MaxAntiEntropyBatchItems = 2; + options.Overlay.MaxAntiEntropyBatchBytes = 3; + + Assert.Equal(16384, options.MaxBatchItems); + Assert.Equal(2 * 1024 * 1024, options.MaxBatchBytes); + Assert.Equal(2, options.Overlay.MaxAntiEntropyBatchItems); + Assert.Equal(3, options.Overlay.MaxAntiEntropyBatchBytes); + Assert.Equal(ValidateOptionsResult.Success, Validate(options)); + } + + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public void AggregationTopologyHasEightWayBranchesAndDepthFourAtScale(int memberCount) + { + var members = CreateSilos(memberCount); + var overlay = new DisseminationOverlayOptions(); + var children = new Dictionary>(memberCount); + var roots = new List(); + + Assert.Equal(32, overlay.GetFanOutFactor(memberCount)); + for (var index = 0; index < members.Length; index++) + { + var silo = members[index]; + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, overlay); + children.Add(silo, snapshot.AggregationChildren); + + Assert.InRange(snapshot.AggregationChildren.Length, 0, 8); + Assert.DoesNotContain(silo, snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + Assert.Equal(members.Skip(32 * (index + 1)).Take(32), snapshot.ForwardingTreeTargets); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); + if (snapshot.IsAggregationRoot) + { + roots.Add(silo); + Assert.Equal(8, snapshot.AggregationChildren.Length); + Assert.Equal(members.Skip(1).Take(8), snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + } + else + { + Assert.Equal(members[0], Assert.Single(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree))); + } + } + + Assert.Equal(members[0], Assert.Single(roots)); + Assert.Equal(memberCount - 1, children.Values.Sum(targets => targets.Length)); + var reached = new HashSet { members[0] }; + var pending = new Queue<(SiloAddress Silo, int Depth)>(); + pending.Enqueue((members[0], 0)); + var maxDepth = 0; + while (pending.TryDequeue(out var current)) + { + maxDepth = Math.Max(maxDepth, current.Depth); + foreach (var child in children[current.Silo]) + { + Assert.True(reached.Add(child), $"Member {child} was reached more than once."); + pending.Enqueue((child, current.Depth + 1)); + } + } + + Assert.Equal(members, reached.Order()); + Assert.Equal(4, maxDepth); + } + + [Theory] + [InlineData(int.MinValue)] + [InlineData(1)] + [InlineData(2)] + [InlineData(32)] + [InlineData(int.MaxValue)] + public void MembershipFanOutSelectorDoesNotChangeAggregationTopology(int selectedFanout) + { + var members = CreateSilos(65); + var calls = 0; + var overlay = new DisseminationOverlayOptions + { + FanOutFactor = count => + { + Assert.Equal(members.Length, count); + calls++; + return selectedFanout; + }, + }; + var membershipFanout = Math.Clamp(selectedFanout, 1, members.Length); + for (var index = 0; index < members.Length; index++) + { + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), members[index], members, overlay); + + Assert.Equal(members.Skip(index * 8 + 1).Take(8), snapshot.AggregationChildren); + Assert.Equal(members.Skip(membershipFanout * (index + 1)).Take(membershipFanout), snapshot.ForwardingTreeTargets); + var originatorTargets = members.Take(membershipFanout).Where(silo => !silo.Equals(members[index])) + .Concat(snapshot.ForwardingTreeTargets); + Assert.Equal(originatorTargets, snapshot.OriginatorTreeTargets); + Assert.Equal(snapshot.OriginatorTreeTargets.Length, snapshot.OriginatorTreeTargets.Distinct().Count()); + } + + Assert.Equal(members.Length, calls); + } + + [Theory] + [InlineData(1, int.MaxValue)] + [InlineData(2, 8)] + [InlineData(7, int.MaxValue)] + [InlineData(17, 1)] + [InlineData(17, 2)] + public void AggregationTopologyClampsConfiguredFanOutToMembership(int memberCount, int fanout) + { + var members = CreateSilos(memberCount); + var overlay = new DisseminationOverlayOptions { AggregationFanOutFactor = fanout }; + var effectiveFanout = Math.Min(memberCount, fanout); + for (var index = 0; index < members.Length; index++) + { + var snapshot = new DisseminationMembershipSnapshot(new MembershipVersion(1), members[index], members, overlay); + + Assert.Equal(members.Skip(index * effectiveFanout + 1).Take(effectiveFanout), snapshot.AggregationChildren); + Assert.DoesNotContain(members[index], snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + } + } + + [Theory] + [InlineData(0)] + [InlineData(8)] + public void NonMembersHaveNoAggregationNeighbors(int memberCount) + { + var local = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 30000), 1); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + CreateSilos(memberCount), + new DisseminationOverlayOptions { AggregationFanOutFactor = int.MaxValue }); + + Assert.False(snapshot.IsAggregationRoot); + Assert.Empty(snapshot.AggregationChildren); + Assert.Empty(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + Assert.Empty(snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); + } + + private static ValidateOptionsResult Validate(DisseminationOptions options) => + new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + private static void AssertValidationFailure(DisseminationOptions options, string message) + { + var result = Validate(options); + Assert.True(result.Failed); + Assert.NotNull(result.Failures); + Assert.Equal(message, Assert.Single(result.Failures)); + } + + private static ImmutableArray CreateSilos(int count) => + Enumerable.Range(11111, count) + .Select(port => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port)) + .Order() + .ToImmutableArray(); +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs new file mode 100644 index 00000000000..1a1bb763bbd --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationCancellationTests.cs @@ -0,0 +1,70 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public sealed class DisseminationCancellationTests +{ + [Theory] + [InlineData(typeof(IDisseminationSystemTarget))] + public void DisseminationRpcMethodsExposeRequiredCancellationToken(Type interfaceType) + { + foreach (var method in interfaceType.GetMethods()) + { + Assert.Equal(typeof(CancellationToken), method.GetParameters().LastOrDefault()?.ParameterType); + Assert.False(method.GetParameters()[^1].IsOptional); + } + } + + [Theory] + [InlineData(typeof(DisseminationProtocol))] + [InlineData(typeof(DisseminationBroadcastQueue))] + [InlineData(typeof(DisseminationSendGate))] + [InlineData(typeof(DisseminationSystemTarget))] + [InlineData(typeof(DisseminationMembership))] + [InlineData(typeof(MembershipDisseminationNamespace))] + [InlineData(typeof(DeploymentLoadStatisticsDisseminationNamespace))] + [InlineData(typeof(WakeTimer))] + public void DisseminationAsyncMethodsExposeRequiredCancellationToken(Type componentType) + { + var types = componentType.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic).Prepend(componentType); + foreach (var type in types) + { + foreach (var method in type.GetMethods( + BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly)) + { + var returnType = method.ReturnType; + var isAsync = typeof(Task).IsAssignableFrom(returnType) + || returnType == typeof(ValueTask) + || returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(ValueTask<>); + if (!isAsync || method.IsSpecialName || method.Name.StartsWith('<')) + { + continue; + } + + // IAsyncDisposable defines the parameterless cleanup boundary. + if (method.Name == nameof(IAsyncDisposable.DisposeAsync) && typeof(IAsyncDisposable).IsAssignableFrom(type)) + { + continue; + } + + Assert.True( + method.GetParameters().Any(static parameter => parameter.ParameterType == typeof(CancellationToken)), + $"{type.FullName}.{method.Name} must expose a CancellationToken parameter."); + Assert.All( + method.GetParameters().Where(static parameter => parameter.ParameterType == typeof(CancellationToken)), + static parameter => Assert.False(parameter.IsOptional)); + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs new file mode 100644 index 00000000000..968ae86ab1f --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationClusterTests.cs @@ -0,0 +1,133 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.TestingHost; +using Xunit; + +namespace UnitTests.Dissemination; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DisseminationDiagnosticCollection +{ + public const string Name = "Dissemination diagnostics"; +} + +/// +/// Exercises dissemination across a real in-process cluster so that membership updates flow through the +/// full serialization pipeline (the in-memory transport is a byte pipe, not object pass-through). This +/// guards against wire types that lack serialization codecs, which unit tests using a fake transport +/// cannot catch. +/// +[TestCategory("Functional"), TestCategory("Dissemination")] +[TestSuite("Functional")] +[TestProvider("None")] +[TestArea("Dissemination")] +[Collection(DisseminationDiagnosticCollection.Name)] +public sealed class DisseminationClusterTests +{ + [Fact] + public async Task MembershipUpdatesAreDisseminatedAcrossRealClusterWithExplicitOptIn() + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromSeconds(120)); + var cancellationToken = cancellation.Token; + + var builder = new InProcessTestClusterBuilder(3); + builder.ConfigureSilo((_, silo) => new EnabledDisseminationConfigurator().Configure(silo)); + await using var cluster = builder.Build(); + await cluster.DeployAsync(cancellationToken); + await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); + + var existingSilos = cluster.GetActiveSilos().Select(static silo => silo.SiloAddress).ToHashSet(); + var baselineVersion = cluster.Silos[0].ServiceProvider + .GetRequiredService() + .CurrentSnapshot.Version.Value; + + // Arm the observer after initial stabilization and accept only a later membership version applied by + // pre-existing silos, so cluster startup or unrelated parallel tests cannot satisfy the assertion. + var observer = new ValueApplyObserver(targetDistinctSilos: 2, existingSilos, baselineVersion); + using var subscription = DisseminationEvents.Listener.Subscribe( + observer, + static name => name == "Dissemination.ValueApply"); + + // Adding a silo produces new membership updates that must propagate to the existing silos. + await cluster.StartAdditionalSiloAsync().WaitAsync(cancellationToken); + await cluster.WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken); + + await observer.Reached.WaitAsync(cancellationToken); + + Assert.True( + observer.AppliedSilos.Count >= 2, + "Expected membership updates to be applied on at least 2 distinct silos, but saw: " + + string.Join(", ", observer.AppliedSilos.Select(static silo => silo.ToString()))); + } + + public sealed class EnabledDisseminationConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder builder) => builder.ConfigureServices(services => + { + services.Configure(options => options.Enabled = true); + services.Configure(options => options.Dissemination.Enabled = true); + services.Configure(options => options.Dissemination.Enabled = true); + }); + } + + private sealed class ValueApplyObserver( + int targetDistinctSilos, + IReadOnlySet expectedSilos, + long baselineVersion) : IObserver> + { + private readonly object _lock = new(); + private readonly HashSet _appliedSilos = []; + private readonly TaskCompletionSource _reached = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task Reached => _reached.Task; + + public IReadOnlyCollection AppliedSilos + { + get + { + lock (_lock) + { + return _appliedSilos.ToArray(); + } + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is not DisseminationValueEvent evt + || evt.Namespace != DisseminationNamespaceNames.Membership + || evt.Key != DisseminationKey.Default + || evt.ToVersion <= baselineVersion + || !expectedSilos.Contains(evt.LocalSilo) + || evt.Result is not (nameof(DisseminationApplyResult.Applied) or nameof(DisseminationApplyResult.Duplicate)) + || evt.PayloadBytes <= 0) + { + return; + } + + lock (_lock) + { + _appliedSilos.Add(evt.LocalSilo); + if (_appliedSilos.Count >= targetDistinctSilos) + { + _reached.TrySetResult(); + } + } + } + + public void OnCompleted() { } + + public void OnError(Exception error) { } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs new file mode 100644 index 00000000000..223b5818c18 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationMembershipSnapshotTests.cs @@ -0,0 +1,762 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using CsCheck; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationMembershipSnapshotTests +{ + [Fact] + public void CollectionsSatisfyMembershipAndRoutingInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 127], Gen.Int[0, 1], Gen.Int[0, 144], static (count, localSeed, localMode, fanoutSeed) => + { + return new ValidSnapshotTestCase( + count, + localSeed, + LocalIsMember: localMode == 0, + RawFanout: fanoutSeed - 16); + }).Sample(testCase => + { + var members = CreateSilos(testCase.Count).ToImmutableArray(); + var local = testCase.LocalIsMember + ? members[testCase.LocalSeed % members.Length] + : CreateSilo(20000 + testCase.LocalSeed); + var snapshot = CreateSnapshot(local, members, testCase.RawFanout); + + Assert.Equal(members, snapshot.Members); + AssertNoDuplicates(nameof(snapshot.Members), snapshot.Members); + + foreach (var member in members) + { + Assert.True(snapshot.ContainsMember(member)); + } + + var outsideMember = CreateSilo(25000 + testCase.LocalSeed); + Assert.False(snapshot.ContainsMember(outsideMember)); + + var originatorTargets = snapshot.OriginatorTreeTargets; + var forwardingTargets = snapshot.ForwardingTreeTargets; + + AssertNoDuplicates(nameof(originatorTargets), originatorTargets); + AssertNoDuplicates(nameof(forwardingTargets), forwardingTargets); + AssertSubset(nameof(originatorTargets), originatorTargets, members); + AssertSubset(nameof(forwardingTargets), forwardingTargets, members); + Assert.DoesNotContain(local, originatorTargets); + Assert.DoesNotContain(local, forwardingTargets); + Assert.Equal(originatorTargets, snapshot.OriginatorTreeTargets); + Assert.Equal(forwardingTargets, snapshot.ForwardingTreeTargets); + + if (!members.Contains(local)) + { + Assert.Empty(originatorTargets); + Assert.Empty(forwardingTargets); + return; + } + + var effectiveFanout = GetEffectiveFanout(members.Length, testCase.RawFanout); + var maxTargets = Math.Max(0, members.Length - 1); + Assert.True( + originatorTargets.Length <= Math.Min(effectiveFanout * 2, maxTargets), + "Originator target count exceeded the expected bound."); + Assert.True( + forwardingTargets.Length <= Math.Min(effectiveFanout, maxTargets), + "Forwarding target count exceeded the expected bound."); + }); + } + + [Fact] + public void AntiEntropyPeerSelectionSatisfiesMembershipInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 127], Gen.Int[0, 1], Gen.Int[0, 64], static (count, localSeed, localMode, requestedCount) => + { + return new AntiEntropyTestCase( + count, + localSeed, + LocalIsMember: localMode == 0, + RequestedCount: requestedCount); + }).Sample(testCase => + { + var members = CreateSilos(testCase.Count).ToImmutableArray(); + var local = testCase.LocalIsMember + ? members[testCase.LocalSeed % members.Length] + : CreateSilo(20000 + testCase.LocalSeed); + var snapshot = CreateSnapshot(local, members, fanout: 4); + + var selectedPeers = snapshot.SelectAntiEntropyPeers(testCase.RequestedCount); + + AssertNoDuplicates("Anti-entropy peers", selectedPeers); + AssertSubset("Anti-entropy peers", selectedPeers, members); + Assert.DoesNotContain(local, selectedPeers); + + var expectedCount = members.Contains(local) + ? Math.Min(testCase.RequestedCount, Math.Max(0, members.Length - 1)) + : 0; + Assert.Equal(expectedCount, selectedPeers.Length); + }); + } + + [Fact] + public void ConstructorRejectsDuplicateMembers() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[0, 63], static (count, duplicateSeed) => (Count: count, DuplicateSeed: duplicateSeed)) + .Sample(testCase => + { + var members = CreateSilos(testCase.Count); + var duplicate = members[testCase.DuplicateSeed % members.Length]; + var invalidMembers = members.Append(duplicate).ToImmutableArray(); + + var exception = Assert.Throws(() => CreateSnapshot( + members[0], + invalidMembers, + fanout: 4)); + Assert.Equal("members", exception.ParamName); + }); + } + + private static DisseminationMembershipSnapshot CreateSnapshot( + SiloAddress localSilo, + ImmutableArray members, + int fanout) => new( + new MembershipVersion(1), + localSilo, + members, + CreateOverlayOptions(fanout)); + + private static DisseminationOverlayOptions CreateOverlayOptions(int fanout) => new() + { + FanOutFactor = _ => fanout, + }; + + private static int GetEffectiveFanout(int memberCount, int rawFanout) => + memberCount <= 1 ? 1 : Math.Clamp(rawFanout, 1, memberCount); + + private static void AssertNoDuplicates(string name, IEnumerable values) + { + var array = values.ToArray(); + Assert.True(array.Length == array.Distinct().Count(), $"{name} contains duplicates."); + } + + private static void AssertSubset(string name, IEnumerable values, IEnumerable expectedValues) + { + var expected = expectedValues.ToHashSet(); + foreach (var value in values) + { + Assert.True(expected.Contains(value), $"{name} contains {value} outside the expected membership set."); + } + } + + private static SiloAddress CreateSilo(int port) => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port); + + private static SiloAddress[] CreateSilos(int count) => + Enumerable.Range(11111, count).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + + private readonly record struct ValidSnapshotTestCase( + int Count, + int LocalSeed, + bool LocalIsMember, + int RawFanout); + + private readonly record struct AntiEntropyTestCase( + int Count, + int LocalSeed, + bool LocalIsMember, + int RequestedCount); + + [Fact] + public void ActiveScopeProjectionIncludesOnlyActiveSilos() + { + var (snapshots, manager, silos) = CreateScopeProjections(); + + Assert.Equal(new[] { silos.Local, silos.Active, silos.ActiveTwo }, snapshots.ActiveMembers.Members); + Assert.Equal(new MembershipVersion(42), snapshots.ActiveMembers.MembershipVersion); + Assert.DoesNotContain(silos.Joining, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.ShuttingDown, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.Stopping, snapshots.ActiveMembers.Members); + Assert.DoesNotContain(silos.Dead, snapshots.ActiveMembers.Members); + Assert.Equal(1, manager.SnapshotReadCount); + } + + [Fact] + public void AllEligibleScopeProjectionIncludesJoiningActiveShuttingDownAndStoppingSilos() + { + var (snapshots, manager, silos) = CreateScopeProjections(); + + Assert.Equal( + new[] { silos.Local, silos.Active, silos.ActiveTwo, silos.Joining, silos.ShuttingDown, silos.Stopping }, + snapshots.AllMembers.Members); + Assert.Equal(new MembershipVersion(42), snapshots.AllMembers.MembershipVersion); + Assert.DoesNotContain(silos.Dead, snapshots.AllMembers.Members); + Assert.Equal(1, manager.SnapshotReadCount); + } + + [Fact] + public void ScopeProjectionsShareTheSameSourceMembershipVersion() + { + var (snapshots, manager, _) = CreateScopeProjections(); + + Assert.Equal(new MembershipVersion(42), snapshots.MembershipVersion); + Assert.Equal(snapshots.MembershipVersion, snapshots.ActiveMembers.MembershipVersion); + Assert.Equal(snapshots.MembershipVersion, snapshots.AllMembers.MembershipVersion); + Assert.Equal(1, manager.SnapshotReadCount); + Assert.NotSame(snapshots.ActiveMembers, snapshots.AllMembers); + } + + [Fact] + public void ScopeProjectionTreeIsDeterministicForSelectedMemberArray() + { + var first = CreateScopeProjections(reverseSourceEntries: false).Snapshots; + var second = CreateScopeProjections(reverseSourceEntries: true).Snapshots; + var active = first.ActiveMembers.Members; + var all = first.AllMembers.Members; + + Assert.Equal(active, second.ActiveMembers.Members); + Assert.Equal(all, second.AllMembers.Members); + Assert.Equal(new[] { active[1], active[2] }, first.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(new[] { active[2] }, first.ActiveMembers.ForwardingTreeTargets); + Assert.Equal( + new[] { all[1], all[2], all[3] }, + first.AllMembers.OriginatorTreeTargets); + Assert.Equal(new[] { all[2], all[3] }, first.AllMembers.ForwardingTreeTargets); + Assert.Equal(first.ActiveMembers.OriginatorTreeTargets, second.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(first.AllMembers.ForwardingTreeTargets, second.AllMembers.ForwardingTreeTargets); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ScopeProjectionOrderingDoesNotDependOnStartTimePrecision(bool activeMembers) + { + var members = CreateSilos(6).Select(static silo => SiloAddress.New(silo.Endpoint, 42)).ToArray(); + var entries = members.Select((member, index) => + { + var entry = CreateScopeMembershipEntry(member, activeMembers ? SiloStatus.Active : SiloStatus.Joining, 0); + // A joining silo can publish its own entry before reading back the storage-rounded timestamp. + entry.StartTime = DateTime.UnixEpoch.AddTicks(index == 0 ? 1 : 0); + return entry; + }).ToArray(); + var persistedEntries = entries.Select(entry => + { + var persisted = entry.Copy(); + persisted.StartTime = LogFormatter.ParseDate(LogFormatter.PrintDate(entry.StartTime)); + return persisted; + }).ToArray(); + + foreach (var local in members) + { + var original = CreateProjections(entries, local); + var persisted = CreateProjections(persistedEntries, local); + + Assert.Equal(activeMembers ? members : [], original.ActiveMembers.Members); + Assert.Equal(activeMembers ? members : [], persisted.ActiveMembers.Members); + Assert.Equal(members, original.AllMembers.Members); + Assert.Equal(members, persisted.AllMembers.Members); + Assert.Equal(original.ActiveMembers.OriginatorTreeTargets, persisted.ActiveMembers.OriginatorTreeTargets); + Assert.Equal(original.ActiveMembers.ForwardingTreeTargets, persisted.ActiveMembers.ForwardingTreeTargets); + Assert.Equal(original.AllMembers.OriginatorTreeTargets, persisted.AllMembers.OriginatorTreeTargets); + Assert.Equal(original.AllMembers.ForwardingTreeTargets, persisted.AllMembers.ForwardingTreeTargets); + } + + static DisseminationMembershipSnapshots CreateProjections(MembershipEntry[] entries, SiloAddress local) => + new DisseminationMembership( + new MutableMembershipManager(new( + new MembershipVersion(42), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)), TestContext.Current.CancellationToken), + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions + { + Overlay = CreateOverlayOptions(2), + })).CurrentSnapshots; + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MembershipVersionChangesPreserveAntiEntropyPeerRotation(bool activeScope) + { + var members = CreateSilos(5); + var local = members[0]; + var peers = members[1..]; + var manager = new MutableMembershipManager( + CreateSourceSnapshot(1, local, peers), + TestContext.Current.CancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var scope = activeScope ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; + + for (var round = 0; round < peers.Length * 2; round++) + { + manager.SetSnapshot(CreateSourceSnapshot(round + 1, local, peers)); + var snapshot = membership.GetSnapshot(scope); + + Assert.Equal(new MembershipVersion(round + 1), snapshot.MembershipVersion); + Assert.Equal(new[] { peers[round % peers.Length] }, snapshot.SelectAntiEntropyPeers(1)); + } + } + + [Fact] + public void AntiEntropyPeerRotationWrapsWhenMembershipShrinks() + { + var members = CreateSilos(5); + var local = members[0]; + var peers = members[1..]; + var manager = new MutableMembershipManager( + CreateSourceSnapshot(1, local, peers), + TestContext.Current.CancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var previous = membership.CurrentSnapshot; + Assert.Equal(peers[..3], previous.SelectAntiEntropyPeers(3)); + + manager.SetSnapshot(CreateSourceSnapshot(2, local, peers[..2])); + var current = membership.CurrentSnapshot; + + Assert.Equal(new[] { peers[1], peers[0] }, current.SelectAntiEntropyPeers(2)); + Assert.Equal(new[] { peers[1] }, current.SelectAntiEntropyPeers(1)); + } + + [Fact] + public void ScopeProjectionAntiEntropySelectionIsDeterministic() + { + var snapshots = CreateScopeProjections().Snapshots; + var activePeers = snapshots.ActiveMembers.Members[1..]; + var allPeers = snapshots.AllMembers.Members[1..]; + + Assert.Equal(new[] { activePeers[0] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new[] { activePeers[1] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new[] { activePeers[0] }, snapshots.ActiveMembers.SelectAntiEntropyPeers(1)); + + for (var i = 0; i < allPeers.Length; i++) + { + Assert.Equal(new[] { allPeers[i] }, snapshots.AllMembers.SelectAntiEntropyPeers(1)); + } + + Assert.Equal(new[] { allPeers[0] }, snapshots.AllMembers.SelectAntiEntropyPeers(1)); + Assert.Equal(new MembershipVersion(42), snapshots.ActiveMembers.MembershipVersion); + Assert.Equal(new MembershipVersion(42), snapshots.AllMembers.MembershipVersion); + } + + [Fact] + public async Task ConcurrentRefreshDoesNotReturnOlderProjection() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(32001); + var firstPeer = CreateSilo(32002); + var secondPeer = CreateSilo(32003); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, local), cancellationToken); + var options = new BlockingDisseminationOptions(cancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + options); + Assert.Equal(new MembershipVersion(40), membership.CurrentSnapshots.MembershipVersion); + + manager.SetSnapshot(CreateSourceSnapshot(41, local, firstPeer)); + var firstRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await options.RefreshBlocked.Task.WaitAsync(cancellationToken); + + manager.SetSnapshot(CreateSourceSnapshot(42, local, firstPeer, secondPeer)); + var secondRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await manager.ThirdRead.Task.WaitAsync(cancellationToken); + options.ReleaseRefresh(); + + Assert.Equal(new MembershipVersion(41), (await firstRefresh).MembershipVersion); + var latest = await secondRefresh; + Assert.Equal(new MembershipVersion(42), latest.MembershipVersion); + Assert.Equal(new[] { local, firstPeer, secondPeer }, latest.ActiveMembers.Members); + Assert.Equal(5, manager.SnapshotReadCount); + } + + [Fact] + public async Task StaleConcurrentRefreshDoesNotRegressNewerProjection() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(32101); + var firstPeer = CreateSilo(32102); + var secondPeer = CreateSilo(32103); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, local), cancellationToken) + { + BlockSecondSnapshotRead = true, + }; + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Equal(new MembershipVersion(40), membership.CurrentSnapshots.MembershipVersion); + + manager.SetSnapshot(CreateSourceSnapshot(41, local, firstPeer)); + var staleRefresh = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + await manager.SecondReadCaptured.Task.WaitAsync(cancellationToken); + + manager.SetSnapshot(CreateSourceSnapshot(42, local, firstPeer, secondPeer)); + var newer = await Task.Run(() => membership.CurrentSnapshots, cancellationToken); + manager.ReleaseSecondRead(); + var stale = await staleRefresh; + + Assert.Equal(new MembershipVersion(42), newer.MembershipVersion); + Assert.Equal(new MembershipVersion(42), stale.MembershipVersion); + Assert.Equal(new[] { local, firstPeer, secondPeer }, membership.CurrentSnapshots.ActiveMembers.Members); + Assert.Equal(6, manager.SnapshotReadCount); + } + + [Theory] + [InlineData(1)] + [InlineData(40)] + public async Task AuthoritativeTopologyReplacementWinsOverStaleConcurrentRead(long replacementVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(3); + var manager = new MutableMembershipManager(CreateSourceSnapshot(40, members[0], members[1]), cancellationToken) + { + BlockSecondSnapshotRead = true, + }; + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Contains(members[1], membership.CurrentSnapshot.Members); + var stale = Task.Run(() => membership.CurrentSnapshots, cancellationToken); + DisseminationMembershipSnapshots replacement; + try + { + await manager.SecondReadCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + manager.SetSnapshot(CreateSourceSnapshot(replacementVersion, members[0], members[2])); + replacement = membership.CurrentSnapshots; + Assert.Equal(new MembershipVersion(replacementVersion), replacement.MembershipVersion); + Assert.Equal(new[] { members[0], members[2] }, replacement.ActiveMembers.Members); + } + finally + { + manager.ReleaseSecondRead(); + } + + Assert.Same(replacement, await stale.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Same(replacement, membership.CurrentSnapshots); + Assert.DoesNotContain(members[1], replacement.AllMembers.Members); + } + + [Theory] + [InlineData(SiloStatus.Joining)] + [InlineData(SiloStatus.ShuttingDown)] + [InlineData(SiloStatus.Stopping)] + public void SameVersionCleanupRefreshesEligibleTopologyAndPreservesRotation(SiloStatus removedStatus) + { + var members = CreateSilos(4); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1], members[2]).Entries.Add( + members[3], CreateScopeMembershipEntry(members[3], removedStatus, 3))); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + Assert.Equal(new[] { members[1] }, before.AllMembers.SelectAntiEntropyPeers(1)); + + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[3]))); + var after = membership.CurrentSnapshots; + + Assert.NotSame(before, after); + Assert.Equal(source.Version, after.MembershipVersion); + Assert.Equal(members[..3], after.AllMembers.Members); + Assert.Equal(members[..3], after.ActiveMembers.Members); + Assert.DoesNotContain(members[3], after.AllMembers.OriginatorTreeTargets); + Assert.DoesNotContain(members[3], after.AllMembers.ForwardingTreeTargets); + Assert.Equal(new[] { members[2] }, after.AllMembers.SelectAntiEntropyPeers(1)); + Assert.Same(after, membership.CurrentSnapshots); + } + + [Theory] + [InlineData(SiloStatus.Created)] + [InlineData(SiloStatus.Dead)] + public void SameVersionCleanupOfNonParticipantsReusesTopology(SiloStatus removedStatus) + { + var members = CreateSilos(3); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1]).Entries.Add( + members[2], CreateScopeMembershipEntry(members[2], removedStatus, 2))); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[2]))); + + Assert.Same(before, membership.CurrentSnapshots); + Assert.Equal(members[..2], membership.CurrentSnapshot.Members); + } + + [Fact] + public void SameVersionHeartbeatReusesTopologyAndRotation() + { + var members = CreateSilos(3); + var source = CreateSourceSnapshot(42, members[0], members[1], members[2]); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var before = membership.CurrentSnapshots; + Assert.Equal(new[] { members[1] }, before.AllMembers.SelectAntiEntropyPeers(1)); + manager.SetSnapshot(new(source.Version, source.Entries.SetItem(members[1], + source.Entries[members[1]].WithIAmAliveTime(DateTime.UnixEpoch.AddMinutes(1))))); + + var after = membership.CurrentSnapshots; + + Assert.Same(before, after); + Assert.Equal(new[] { members[2] }, after.AllMembers.SelectAntiEntropyPeers(1)); + } + + [Fact] + public async Task StaleConcurrentReadCannotRestoreSameVersionRemovedParticipant() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(3); + var source = new MembershipTableSnapshot(new MembershipVersion(42), + CreateSourceSnapshot(42, members[0], members[1]).Entries.Add( + members[2], CreateScopeMembershipEntry(members[2], SiloStatus.Joining, 2))); + var manager = new MutableMembershipManager(source, cancellationToken) { BlockSecondSnapshotRead = true }; + var membership = new DisseminationMembership(manager, new ScopeLocalSiloDetails(members[0]), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + Assert.Contains(members[2], membership.CurrentSnapshot.Members); + var stale = Task.Run(() => membership.CurrentSnapshots); + DisseminationMembershipSnapshots updated; + try + { + await manager.SecondReadCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + manager.SetSnapshot(new(source.Version, source.Entries.Remove(members[2]))); + updated = membership.CurrentSnapshots; + Assert.DoesNotContain(members[2], updated.AllMembers.Members); + } + finally + { + manager.ReleaseSecondRead(); + } + + Assert.Same(updated, await stale.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Same(updated, membership.CurrentSnapshots); + Assert.Equal(source.Version, updated.MembershipVersion); + } + + private static ( + DisseminationMembershipSnapshots Snapshots, + MutableMembershipManager Manager, + ScopeSilos Silos) CreateScopeProjections(bool reverseSourceEntries = false) + { + var silos = new ScopeSilos( + CreateSilo(31001), + CreateSilo(31002), + CreateSilo(31003), + CreateSilo(31004), + CreateSilo(31005), + CreateSilo(31006), + CreateSilo(31007)); + var entries = new[] + { + CreateScopeMembershipEntry(silos.Dead, SiloStatus.Dead, 5), + CreateScopeMembershipEntry(silos.Stopping, SiloStatus.Stopping, 4), + CreateScopeMembershipEntry(silos.ShuttingDown, SiloStatus.ShuttingDown, 3), + CreateScopeMembershipEntry(silos.Joining, SiloStatus.Joining, 2), + CreateScopeMembershipEntry(silos.ActiveTwo, SiloStatus.Active, 2), + CreateScopeMembershipEntry(silos.Active, SiloStatus.Active, 1), + CreateScopeMembershipEntry(silos.Local, SiloStatus.Active, 0), + }; + if (reverseSourceEntries) + { + Array.Reverse(entries); + } + + var source = new MembershipTableSnapshot( + new MembershipVersion(42), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + var manager = new MutableMembershipManager(source, TestContext.Current.CancellationToken); + var membership = new DisseminationMembership( + manager, + new ScopeLocalSiloDetails(silos.Local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions + { + Overlay = new DisseminationOverlayOptions + { + FanOutFactor = static _ => 2, + }, + })); + + return (membership.CurrentSnapshots, manager, silos); + } + + private static MembershipEntry CreateScopeMembershipEntry( + SiloAddress silo, + SiloStatus status, + int startSeconds) => new() + { + SiloAddress = silo, + Status = status, + ProxyPort = silo.Endpoint.Port, + HostName = "localhost", + SiloName = silo.ToParsableString(), + RoleName = "test", + StartTime = DateTime.UnixEpoch.AddSeconds(startSeconds), + IAmAliveTime = DateTime.UnixEpoch.AddSeconds(startSeconds), + }; + + private static MembershipTableSnapshot CreateSourceSnapshot( + long version, + SiloAddress local, + params SiloAddress[] peers) + { + var entries = peers + .Prepend(local) + .Select((silo, index) => CreateScopeMembershipEntry(silo, SiloStatus.Active, index)) + .ToImmutableDictionary(static entry => entry.SiloAddress); + return new(new MembershipVersion(version), entries); + } + + private readonly record struct ScopeSilos( + SiloAddress Local, + SiloAddress Active, + SiloAddress ActiveTwo, + SiloAddress Joining, + SiloAddress ShuttingDown, + SiloAddress Stopping, + SiloAddress Dead); + + private sealed class ScopeLocalSiloDetails(SiloAddress siloAddress) : ILocalSiloDetails + { + public string Name => "test"; + + public string ClusterId => "test"; + + public string DnsHostName => "localhost"; + + public SiloAddress SiloAddress => siloAddress; + + public SiloAddress GatewayAddress => siloAddress; + } + + private sealed class MutableMembershipManager( + MembershipTableSnapshot snapshot, + CancellationToken cancellationToken) + : Orleans.Runtime.MembershipService.IMembershipManager + { + private readonly ManualResetEventSlim _releaseSecondRead = new(); + private MembershipTableSnapshot _snapshot = snapshot; + private int _snapshotReadCount; + + public int SnapshotReadCount => Volatile.Read(ref _snapshotReadCount); + + public bool BlockSecondSnapshotRead { get; init; } + + public TaskCompletionSource SecondReadCaptured { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ThirdRead { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public MembershipTableSnapshot CurrentSnapshot + { + get + { + var result = Volatile.Read(ref _snapshot); + var readCount = Interlocked.Increment(ref _snapshotReadCount); + if (readCount == 2 && BlockSecondSnapshotRead) + { + SecondReadCaptured.TrySetResult(); + _releaseSecondRead.Wait(cancellationToken); + } + + if (readCount == 3) + { + ThirdRead.TrySetResult(); + } + + return result; + } + } + + public IAsyncEnumerable MembershipUpdates => EmptyUpdates(TestContext.Current.CancellationToken); + + public SiloStatus LocalSiloStatus => SiloStatus.Active; + + public void ReleaseSecondRead() => _releaseSecondRead.Set(); + + public void SetSnapshot(MembershipTableSnapshot value) => Volatile.Write(ref _snapshot, value); + + public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task TrySuspectSilo( + SiloAddress silo, + SiloAddress? indirectProbingSilo, + CancellationToken cancellationToken) => Task.FromResult(false); + + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) => Task.CompletedTask; + + public Task ProcessGossipSnapshot( + MembershipTableSnapshot value, + CancellationToken cancellationToken) => Task.CompletedTask; + + public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; + + public bool CheckHealth(DateTime lastCheckTime, out string reason) + { + reason = string.Empty; + return true; + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + + private static async IAsyncEnumerable EmptyUpdates( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.CompletedTask; + yield break; + } + } + + private sealed class BlockingDisseminationOptions(CancellationToken cancellationToken) + : Microsoft.Extensions.Options.IOptions + { + private readonly ManualResetEventSlim _releaseRefresh = new(); + private int _readCount; + + public TaskCompletionSource RefreshBlocked { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public DisseminationOptions Value + { + get + { + if (Interlocked.Increment(ref _readCount) == 2) + { + RefreshBlocked.TrySetResult(); + _releaseRefresh.Wait(cancellationToken); + } + + return new() + { + Overlay = new() + { + FanOutFactor = static _ => 2, + }, + }; + } + } + + public void ReleaseRefresh() => _releaseRefresh.Set(); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs new file mode 100644 index 00000000000..65a322bb9aa --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.AdmissionReview.cs @@ -0,0 +1,259 @@ +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ShutdownDrainsAdmittedPublicationBeforeBroadcastQueue() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40501); + var peer = CreateSilo(40502); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("value", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var release = new ManualResetEventSlim(); + var repairs = 0; + ns.RepairHandler = request => + { + if (Interlocked.Increment(ref repairs) == 1) + { + entered.SetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken)); + } + + return ns.Inner.CreateRepair(request); + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: new FakeTimeProvider()); + var publication = Task.Run(async () => await protocol.Publish(ns, "value", 1, cancellationToken), cancellationToken); + Task stop = Task.CompletedTask; + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + Assert.Equal(1, Volatile.Read(ref repairs)); + release.Set(); + + Assert.True(await publication.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion); + } + finally + { + release.Set(); + await publication.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownDrainsAdmittedBroadcastApplicationAndForwarding() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40511); + var sender = CreateSilo(40512); + var child = CreateSilo(40513); + var transport = new FakeTransport(local, sender, child); + var ns = new ProtocolReviewNamespace(local); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + await release.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2); + var batch = new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(sender, "value", 1)), + }; + var receive = protocol.ReceiveBroadcast(batch, cancellationToken); + Task stop = Task.CompletedTask; + try + { + Assert.Single(ns.Attempts); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + Assert.Single(ns.Attempts); + release.SetResult(); + + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + } + finally + { + release.TrySetResult(); + await receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownDrainsAdmittedAntiEntropyResponse() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40521); + var peer = CreateSilo(40522); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("value", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var release = new ManualResetEventSlim(); + ns.RepairHandler = request => + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), cancellationToken)); + return ns.Inner.CreateRepair(request); + }; + var protocol = CreateProtocol(transport, [ns]); + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("value", 0)), + }; + var response = Task.Run(async () => await protocol.ReceiveAntiEntropy(request, cancellationToken), cancellationToken); + Task stop = Task.CompletedTask; + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + stop = protocol.StopAsync(cancellationToken); + Assert.False(stop.IsCompleted); + await Assert.ThrowsAsync(async () => await protocol.ReceiveAntiEntropy(request, cancellationToken)); + release.Set(); + + var result = await response.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, Assert.Single(result.Values[ns.Name]).Value.ToVersion); + } + finally + { + release.Set(); + await response.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownCancelsAndDrainsLocalAntiEntropyRound() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40531); + var peer = CreateSilo(40532); + var transport = new FakeTransport(local, peer); + var raw = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = (_, _, token) => + { + started.SetResult(token); + return new(raw.Task); + }; + var protocol = CreateProtocol(transport, new FakeNamespace(local)); + var round = protocol.RunAntiEntropyRound(cancellationToken); + try + { + var transportToken = await started.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.True(transportToken.IsCancellationRequested); + await Assert.ThrowsAnyAsync( + () => round.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.False(raw.Task.IsCompleted); + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Single(transport.AntiEntropyRequests); + } + finally + { + raw.TrySetResult(new DisseminationAntiEntropyResponse { Sender = peer }); + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task ShutdownCancellationKeepsAdmissionClosed() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40541); + var peer = CreateSilo(40542); + var transport = new FakeTransport(local, peer); + var ns = new ProtocolReviewNamespace(local); + using var requestCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ns.ApplyHandler = async (_, token) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return DisseminationApplyResult.Applied; + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: new FakeTimeProvider()); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }; + var receive = protocol.ReceiveBroadcast(batch, requestCancellation.Token); + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var stop = protocol.StopAsync(shutdown.Token); + try + { + Assert.Single(ns.Attempts); + Assert.False(stop.IsCompleted); + shutdown.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => stop.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + Assert.Equal(shutdown.Token, exception.CancellationToken); + Assert.False(receive.IsCompleted); + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + } + finally + { + requestCancellation.Cancel(); + await Assert.ThrowsAnyAsync( + () => receive.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + await protocol.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Assert.Equal(0, ns.GetVersion("value")); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ShutdownRejectsPublicationWithNoPeersAndPreservesCallerCancellation() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40551); + var transport = new FakeTransport(local); + var ns = new FakeNamespace(local); + ns.SetValue("value", 1); + var protocol = CreateProtocol(transport, ns); + await protocol.StopAsync(cancellationToken); + + Assert.False(await protocol.Publish(ns, "value", 1, cancellationToken)); + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + async () => await protocol.Publish(ns, "value", 1, canceled.Token)); + Assert.Equal(canceled.Token, exception.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs new file mode 100644 index 00000000000..71c63c97625 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Aggregation.cs @@ -0,0 +1,492 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(1, 4)] + [InlineData(8, 4)] + [InlineData(32, 6)] + [InlineData(100, 10)] + [InlineData(256, 4)] + public void AggregationTopologyHasOneRootAndLinearDistributionEdges(int count, int fanout) + { + var members = CreateSilos(count).ToImmutableArray(); + var overlay = new DisseminationOverlayOptions { AggregationFanOutFactor = fanout }; + var snapshots = members.ToDictionary( + silo => silo, + silo => new DisseminationMembershipSnapshot(new MembershipVersion(1), silo, members, overlay)); + Assert.Equal(count - 1, snapshots.Values.Sum(snapshot => snapshot.AggregationChildren.Length)); + Assert.Equal(members.Skip(1), snapshots.Values.SelectMany(snapshot => snapshot.AggregationChildren).Order()); + foreach (var (silo, snapshot) in snapshots) + { + Assert.DoesNotContain(silo, snapshot.AggregationChildren); + Assert.Equal(snapshot.AggregationChildren.Length, snapshot.AggregationChildren.Distinct().Count()); + Assert.InRange(snapshot.AggregationChildren.Length, 0, fanout); + Assert.All(snapshot.AggregationChildren, child => Assert.True(child.CompareTo(silo) > 0)); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetForwardingTargets(DisseminationRoutingMode.AggregationTree)); + if (snapshot.IsAggregationRoot) + { + Assert.Equal(members[0], silo); + Assert.Equal(snapshot.AggregationChildren, snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree)); + } + else + { + Assert.Equal(members[0], Assert.Single(snapshot.GetOriginatorTargets(DisseminationRoutingMode.AggregationTree))); + } + } + + var reached = new HashSet { members[0] }; + var pending = new Queue(reached); + while (pending.TryDequeue(out var silo)) + { + foreach (var peer in snapshots[silo].AggregationChildren) + { + if (reached.Add(peer)) + { + pending.Enqueue(peer); + } + } + } + + Assert.Equal(members.Order(), reached.Order()); + } + + [Theory] + [InlineData(8, 4)] + [InlineData(32, 6)] + [InlineData(100, 10)] + [InlineData(256, 4)] + public async Task AggregationTreeDeliversWholeRoundsWithLinearBatchCount(int count, int fanout) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(count); + var clock = new FakeTimeProvider(); + var nodes = new Dictionary(); + foreach (var member in members) + { + var ns = new FakeNamespace(member) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.AggregationPeriod = TimeSpan.FromHours(1); + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AggregationFanOutFactor = fanout; + options.MaxConcurrentSends = 1; + }, clock); + nodes.Add(member, (ns, transport, protocol)); + transport.SendBroadcastResponseHandler = (peer, batch, token) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + return nodes[peer].Protocol.ReceiveBroadcast(batch, token); + }; + } + + try + { + for (var round = 1; round <= 3; round++) + { + var before = nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count); + foreach (var member in members) + { + var node = nodes[member]; + node.Namespace.SetValue(member, round); + Assert.True(await node.Protocol.Publish(node.Namespace, member, round, cancellationToken)); + } + + // Finish producer ingresses before flushing the root, then drain distribution in tree order. + for (var index = members.Length - 1; index >= 0; index--) + { + await nodes[members[index]].Protocol.FlushPendingBroadcast(cancellationToken); + } + foreach (var member in members) + { + await nodes[member].Protocol.FlushPendingBroadcast(cancellationToken); + } + + foreach (var node in nodes.Values) + { + foreach (var origin in members) + { + Assert.Equal(round, node.Namespace.GetVersion(origin)); + } + } + + var batches = nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count) - before; + Assert.Equal(2 * (count - 1), batches); + Assert.Empty(nodes.Values.SelectMany(node => node.Transport.AntiEntropyRequests)); + } + } + finally + { + foreach (var node in nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + foreach (var node in nodes.Values) + { + await node.Protocol.StopAsync(cancellationToken); + } + } + } + + [Theory] + [InlineData(1)] + [InlineData(14)] + public async Task RootAggregationBatchesOnceAndCrossesMultipleLevelsWithoutAnotherDelay(int originIndex) + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(15); + var root = members[0]; + var origin = members[originIndex]; + var clock = new FakeTimeProvider(); + var ingested = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var nodes = new Dictionary(); + foreach (var member in members) + { + var ns = new FakeNamespace(member) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + ApplyObserved = new(TaskCreationOptions.RunContinuationsAsynchronously), + }; + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AggregationFanOutFactor = 2; + options.MaxConcurrentSends = 1; + }, clock); + nodes.Add(member, (ns, transport, protocol)); + transport.SendBroadcastResponseHandler = async (peer, batch, token) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + var response = await nodes[peer].Protocol.ReceiveBroadcast(batch, token); + if (member.Equals(origin) && peer.Equals(root)) + { + ingested.TrySetResult(); + } + return response; + }; + } + + try + { + var start = clock.GetTimestamp(); + var source = nodes[origin]; + source.Namespace.SetValue("value", 1); + Assert.True(await source.Protocol.Publish(source.Namespace, "value", 1, cancellationToken)); + await ingested.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(1, nodes[root].Namespace.GetVersion("value")); + Assert.Equal(1, nodes.Values.Sum(node => node.Transport.BroadcastBatches.Count)); + + clock.Advance(TimeSpan.FromMilliseconds(499)); + Assert.All(members.Where(silo => !silo.Equals(root) && !silo.Equals(origin)), + silo => Assert.Equal(0, nodes[silo].Namespace.GetVersion("value"))); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await Task.WhenAll(members.Where(silo => !silo.Equals(origin)) + .Select(silo => nodes[silo].Namespace.ApplyObserved!.Task)) + .WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.All(nodes.Values, node => Assert.Equal(1, node.Namespace.GetVersion("value"))); + Assert.Equal(TimeSpan.FromMilliseconds(500), clock.GetElapsedTime(start)); + } + finally + { + foreach (var node in nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + foreach (var node in nodes.Values) + { + await node.Protocol.StopAsync(cancellationToken); + } + } + } + + [Fact] + public async Task MembershipBroadcastsDrainAheadOfAggregatedValues() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40821); + var peer = CreateSilo(40822); + var transport = new FakeTransport(local, peer); + var membership = new FakeNamespace(local, "membership"); + var load = new FakeNamespace(local, "load", DisseminationMembershipScope.ActiveMembers) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + }; + membership.SetValue("value", 1); + load.SetValue("value", 2); + var queue = CreateBroadcastQueue(transport, [load, membership], options => options.MaxBatchItems = 1); + try + { + var accepted = BeforeBroadcastPumpsRun(() => ( + Load: queue.Notify(peer, load, "value"), + Membership: queue.Notify(peer, membership, "value"))); + Assert.True(accepted.Load); + Assert.True(accepted.Membership); + await queue.FlushPendingBroadcast(cancellationToken); + + Assert.Equal( + new[] { (membership.Name, 1L), (load.Name, 2L) }, + transport.BroadcastBatches.Select(batch => ( + Assert.Single(batch.Batch.Values.Keys), + Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion))); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(peer, batch.Peer)); + } + finally + { + await queue.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AggregationDistributionRequiresBroadcastAcknowledgmentBeforeSuppressingKnownValue() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(2); + var ns = new FakeNamespace(members[0]) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(members[0], members[1]); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + ns.SetValue("value", 1); + Assert.True(BeforeBroadcastPumpsRun(() => + { + var accepted = queue.Notify(members[1], ns, "value", force: false); + queue.ObservePeerVersion(members[1], ns.Name, "value", 1); + Assert.Empty(transport.BroadcastBatches); + return accepted; + })); + await queue.FlushPendingBroadcast(cancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches).Batch; + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + + Assert.True(queue.Notify(members[1], ns, "value", force: false)); + await queue.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task FormerAggregationRootRedirectsIngressToCurrentRootImmediately() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(7); + var local = members[1]; + var sender = members[^1]; + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); + var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (peer, batch, _) => + { + transport.BroadcastBatches.Add((peer, batch)); + forwarded.TrySetResult(); + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AggregationFanOutFactor = 2, clock); + var start = clock.GetTimestamp(); + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = sender, + Values = CreateValueGroups([ns.CreateItem(sender, sender, 1)]), + }, cancellationToken); + await forwarded.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + + var sent = Assert.Single(transport.BroadcastBatches); + Assert.Equal(members[0], sent.Peer); + Assert.Equal(sender, Assert.Single(GetBroadcastValues(sent.Batch)).Value.Key.Value); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AggregationRelayForwardsWholeBatchAtomicallyWithoutCoalescing() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(7); + var local = members[1]; + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.AggregationPeriod = TimeSpan.FromMilliseconds(500); + var transport = new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray()); + var forwarded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var count = 0; + transport.SendBroadcastResponseHandler = (peer, batch, _) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + count++; + if (count == 2) + { + forwarded.TrySetResult(); + } + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AggregationFanOutFactor = 2, clock); + var values = Enumerable.Range(0, 64).Select(index => ns.CreateItem(members[0], $"key-{index}", 1)).ToArray(); + var start = clock.GetTimestamp(); + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = members[0], + Values = CreateValueGroups(values), + }, cancellationToken); + await forwarded.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + Assert.Equal(new[] { members[3], members[4] }, transport.BroadcastBatches.Select(batch => batch.Peer).Order()); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(64, GetBroadcastValues(batch.Batch).Count()); + Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); + }); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public void RootAggregationRequiresAddressOrderedActiveMembership() + { + var local = CreateSilo(40811); + var ns = new FakeNamespace(local) { RoutingMode = DisseminationRoutingMode.AggregationTree }; + + var exception = Assert.Throws(() => CreateProtocol(new FakeTransport(local), ns)); + + Assert.Equal("disseminationNamespaces", exception.ParamName); + } + + [Fact] + public async Task QueueBatchesAcceptedUpdatesAcrossInFlightSend() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40801); + var peer = CreateSilo(40802); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + var transport = new FakeTransport(local, peer); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timestamps = new List(); + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + timestamps.Add(clock.GetTimestamp()); + if (timestamps.Count == 1) + { + firstStarted.SetResult(); + await releaseFirst.Task; + } + else + { + secondStarted.TrySetResult(); + } + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + var start = clock.GetTimestamp(); + for (var key = 0; key < 100; key++) + { + ns.SetValue($"key-{key}", 1); + } + Assert.True(queue.NotifyBatch(peer, ns, + [.. Enumerable.Range(0, 100).Select(key => new DisseminationBroadcastQueue.KeyNotification($"key-{key}", 1, true))])); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.Equal(100, GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch).Count()); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start, timestamps[0])); + Assert.All(GetBroadcastValues(transport.BroadcastBatches[0].Batch), value => Assert.Equal(1, value.Value.ToVersion)); + + for (var key = 0; key < 100; key++) + { + ns.SetValue($"key-{key}", 2); + } + Assert.True(queue.NotifyBatch(peer, ns, + [.. Enumerable.Range(0, 100).Select(key => new DisseminationBroadcastQueue.KeyNotification($"key-{key}", 2, true))])); + Assert.Single(transport.BroadcastBatches); + releaseFirst.SetResult(); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await secondStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start, timestamps[1])); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal(100, GetBroadcastValues(transport.BroadcastBatches[1].Batch).Count()); + Assert.All(GetBroadcastValues(transport.BroadcastBatches[1].Batch), value => Assert.Equal(2, value.Value.ToVersion)); + } + finally + { + releaseFirst.TrySetResult(); + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs new file mode 100644 index 00000000000..0ec69d226fb --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CohortPublisher.cs @@ -0,0 +1,610 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Orleans.Serialization.Invocation; +using Orleans.Timers; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(25)] + [InlineData(1000)] + public async Task CohortPublisherReceiptReschedulesActualTimer(int sealMilliseconds) + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var first = await harness.Dissemination.NextPublicationAsync(); + var sealDelay = TimeSpan.FromMilliseconds(sealMilliseconds); + harness.Clock.Advance(sealDelay); + first.Receipt.SetResult(new(true, harness.Period - sealDelay)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(harness.Period - sealDelay, harness.TimerClock.DueTime); + if (sealDelay < harness.Period) + { + harness.Clock.Advance(harness.Period - sealDelay - TimeSpan.FromMilliseconds(1)); + Assert.Single(harness.Dissemination.Publications); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + } + + var second = await harness.Dissemination.NextPublicationAsync(); + Assert.Equal(harness.Period, harness.Clock.GetElapsedTime(harness.StartTimestamp)); + Assert.True(second.Version > first.Version); + Assert.Equal(second.Version, harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks); + Assert.False(second.Token.IsCancellationRequested); + Assert.False(second.Receipt.Task.IsCompleted); + Assert.Equal(2, harness.Dissemination.Publications.Count); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherReceiptSubtractsDirectDeliveryTime() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + harness.Dissemination.UnconfirmedPeers = [harness.Peer]; + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDirect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = token => + { + directStarted.TrySetResult(); + return releaseDirect.Task.WaitAsync(token); + }; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + harness.Clock.Advance(TimeSpan.FromMilliseconds(25)); + publication.Receipt.SetResult(new(true, TimeSpan.FromMilliseconds(975))); + await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + harness.Clock.Advance(TimeSpan.FromMilliseconds(200)); + releaseDirect.SetResult(); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(TimeSpan.FromMilliseconds(775), harness.TimerClock.DueTime); + Assert.Equal(publication.Version, Assert.Single(harness.DirectUpdates).DateTime.Ticks); + harness.Clock.Advance(TimeSpan.FromMilliseconds(775)); + var next = await harness.Dissemination.NextPublicationAsync(); + Assert.Equal(harness.Period, harness.Clock.GetElapsedTime(harness.StartTimestamp)); + Assert.True(next.Version > publication.Version); + } + + [Theory] + [InlineData("disabled")] + [InlineData("rejected")] + [InlineData("failed")] + public async Task CohortPublisherDirectFallbackRetainsNormalTimerCadence(string outcome) + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Options.Dissemination.Enabled = outcome != "disabled"; + harness.Dissemination.AutomaticReceipt = new(false, TimeSpan.Zero); + if (outcome == "failed") + { + harness.Dissemination.Failure = new InvalidOperationException("Receipt transport failed."); + } + + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDirect = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = token => + { + directStarted.TrySetResult(); + return releaseDirect.Task.WaitAsync(token); + }; + var tick = await harness.StartTickAsync(); + await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + harness.Clock.Advance(TimeSpan.FromMilliseconds(200)); + releaseDirect.SetResult(); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(harness.Period, harness.TimerClock.DueTime); + Assert.Single(harness.DirectUpdates); + Assert.Equal(outcome == "disabled" ? 0 : 1, harness.Dissemination.Publications.Count); + Assert.Equal(0, harness.Dissemination.QueryCount); + harness.Clock.Advance(harness.Period - TimeSpan.FromMilliseconds(1)); + Assert.Single(harness.DirectUpdates); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + var nextTick = await harness.Timers.NextTickAsync(); + await nextTick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, harness.DirectUpdates.Count); + Assert.Equal(TimeSpan.FromMilliseconds(1200), harness.Clock.GetElapsedTime(harness.StartTimestamp)); + } + + [Fact] + public async Task CohortPublisherReceiptBudgetAllowsOnePeriodAndExpiresAfterTwo() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + + harness.Clock.Advance(harness.Period); + await harness.OwnerBarrierAsync(); + Assert.False(publication.Token.IsCancellationRequested); + Assert.False(tick.IsCompleted); + Assert.Empty(harness.DirectUpdates); + + harness.Clock.Advance(harness.Period - TimeSpan.FromMilliseconds(1)); + Assert.False(publication.Token.IsCancellationRequested); + harness.Clock.Advance(TimeSpan.FromMilliseconds(1)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(publication.Token.IsCancellationRequested); + Assert.Single(harness.DirectUpdates); + Assert.Equal(harness.Period, harness.TimerClock.DueTime); + Assert.Equal(0, harness.Dissemination.QueryCount); + } + + [Fact] + public async Task CohortPublisherCallerCancellationPreservesIdentityWithoutFallback() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var publish = harness.PublishAsync(caller.Token); + var publication = await harness.Dissemination.NextPublicationAsync(); + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + await harness.OwnerBarrierAsync(); + + Assert.Equal(caller.Token, exception.CancellationToken); + Assert.True(publication.Token.IsCancellationRequested); + Assert.Empty(harness.DirectUpdates); + Assert.Equal(0, harness.Dissemination.QueryCount); + } + + [Fact] + public async Task CohortPublisherOwningCancellationPreservesNativeDirectCompletion() + { + await using var harness = new CohortPublisherHarness(refreshTime: TimeSpan.Zero); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var directStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.DirectHandler = async token => + { + using var registration = token.Register(() => cancellationObserved.TrySetResult()); + directStarted.TrySetResult(token); + await cancellationObserved.Task; + }; + var owning = harness.PublishAsync(caller.Token); + try + { + var nativeToken = await directStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var joining = harness.PublishAsync(TestContext.Current.CancellationToken); + await harness.OwnerBarrierAsync(); + var nativeSample = harness.Publisher.LocalRuntimeStatistics; + Assert.False(owning.IsCompleted); + Assert.False(joining.IsCompleted); + Assert.Single(harness.DirectUpdates); + Assert.Equal(caller.Token, nativeToken); + + caller.Cancel(); + await Task.WhenAll(owning, joining).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(owning.IsCompletedSuccessfully); + Assert.True(joining.IsCompletedSuccessfully); + Assert.Same(nativeSample, harness.Publisher.LocalRuntimeStatistics); + Assert.Same(nativeSample, harness.Publisher.PeriodicStatistics[harness.Publisher.Silo]); + Assert.Single(harness.DirectUpdates); + Assert.Empty(harness.Dissemination.Publications); + } + finally + { + caller.Cancel(); + } + } + + [Fact] + public async Task CohortPublisherOwningCancellationPreservesNativeReceiptCompletion() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + harness.Dissemination.PreserveNativeCompletion = true; + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var owning = harness.PublishAsync(caller.Token); + var publication = await harness.Dissemination.NextPublicationAsync(); + caller.Cancel(); + publication.Receipt.SetResult(new(true, harness.Period)); + await owning.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(owning.IsCompletedSuccessfully); + Assert.True(publication.Token.IsCancellationRequested); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(1, harness.Dissemination.QueryCount); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherStartupReceiptWaitKeepsInboundOwnerApplicationLive() + { + await using var harness = new CohortPublisherHarness(); + harness.Dissemination.AutomaticReceipt = null; + var startup = harness.StartAsync(clearStartup: false); + var publication = await harness.Dissemination.NextPublicationAsync(); + var tick = await harness.StartTickAsync(); + await harness.OwnerBarrierAsync(); + + var incoming = CreatePhase5Statistics(42); + var applied = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Peer, incoming, TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(DisseminationApplyResult.Applied, applied); + Assert.Same(incoming, harness.Publisher.PeriodicStatistics[harness.Peer]); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(publication.Version, harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks); + Assert.False(startup.IsCompleted); + Assert.False(tick.IsCompleted); + + harness.Clock.Advance(TimeSpan.FromMilliseconds(25)); + publication.Receipt.SetResult(new(true, TimeSpan.FromMilliseconds(975))); + await Task.WhenAll(startup, tick).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(TimeSpan.FromMilliseconds(975), harness.TimerClock.DueTime); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherOverlappingCallerCancellationDoesNotCancelPendingSample() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var overlapping = harness.PublishAsync(caller.Token); + await harness.OwnerBarrierAsync(); + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => overlapping.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + + Assert.Equal(caller.Token, exception.CancellationToken); + Assert.False(publication.Token.IsCancellationRequested); + Assert.Single(harness.Dissemination.Publications); + publication.Receipt.SetResult(new(true, harness.Period)); + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherStopRetainsDisposedTimerAndCancelsReceipt() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + harness.Dissemination.AutomaticReceipt = null; + var tick = await harness.StartTickAsync(); + var publication = await harness.Dissemination.NextPublicationAsync(); + var timer = harness.Timers.Timer; + await harness.Lifecycle.OnStop(TestContext.Current.CancellationToken); + await Assert.ThrowsAnyAsync( + () => tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + await harness.OwnerBarrierAsync(); + harness.Clock.Advance(harness.Period * 3); + + Assert.True(publication.Token.IsCancellationRequested); + Assert.True(harness.TimerClock.Disposed); + Assert.Same(timer, typeof(DeploymentLoadPublisher).GetField( + "_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(harness.Publisher)); + Assert.Single(harness.Dissemination.Publications); + Assert.Empty(harness.DirectUpdates); + } + + [Fact] + public async Task CohortPublisherExplicitPublicationPreservesDisposedTimerPause() + { + await using var harness = new CohortPublisherHarness(); + await harness.StartAsync(); + // Only timer registration changes the schedule: the explicit startup publication does not. + Assert.Equal(1, harness.TimerClock.ChangeCount); + var timer = harness.Timers.Timer; + var firstVersion = harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks; + await harness.Lifecycle.OnStop(TestContext.Current.CancellationToken); + + await harness.PublishAsync(TestContext.Current.CancellationToken); + harness.Clock.Advance(harness.Period * 3); + + Assert.True(harness.Publisher.LocalRuntimeStatistics.DateTime.Ticks > firstVersion); + Assert.Single(harness.Dissemination.Publications); + Assert.Equal(1, harness.TimerClock.ChangeCount); + Assert.True(harness.TimerClock.Disposed); + Assert.Same(timer, typeof(DeploymentLoadPublisher).GetField( + "_publishTimer", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(harness.Publisher)); + Assert.Empty(harness.Timers.AllTicks); + Assert.Empty(harness.DirectUpdates); + } + + private sealed class CohortPublisherHarness : IAsyncDisposable + { + private readonly ServiceProvider _serializerServices; + + public CohortPublisherHarness(TimeSpan? refreshTime = null) + { + var local = CreateSilo(40501); + Peer = CreateSilo(40502); + var statusOracle = new Phase4FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(Peer, SiloStatus.Active); + var details = new FakeLocalSiloDetails(local); + Options = new() { DeploymentLoadPublisherRefreshTime = refreshTime ?? Period }; + Options.Dissemination.Enabled = true; + _serializerServices = new ServiceCollection().AddSerializer().BuildServiceProvider(); + TimerClock = new(Clock); + Timers = new(new TimerRegistry( + NullLoggerFactory.Instance, + TimerClock, + new MessageFactory(_serializerServices.GetRequiredService(), NullLogger.Instance, null!), + details)); + var shared = new SystemTargetShared( + runtimeClient: null!, + details, + NullLoggerFactory.Instance, + Microsoft.Extensions.Options.Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + Timers, + new ActivationDirectory(CreatePhase4Instruments()), + CreatePhase4Instruments(), + CreatePhase4Instruments(), + CreatePhase4Instruments(), + CreatePhase4Instruments()); + var grainFactory = Substitute.For(); + var siloControl = Substitute.For(); + siloControl.GetRuntimeStatistics(Arg.Any()).Returns(CreatePhase5Statistics(0)); + grainFactory.GetSystemTarget( + Constants.SiloControlType, Arg.Any()).Returns(siloControl); + var directTarget = Substitute.For(); + directTarget.UpdateRuntimeStatistics( + Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + DirectUpdates.Enqueue(call.ArgAt(1)); + return DirectHandler(call.ArgAt(2)); + }); + grainFactory.GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, Peer).Returns(directTarget); + var services = new Phase4MutableServiceProvider(); + services.Add(Clock); + Publisher = new( + details, + statusOracle, + Microsoft.Extensions.Options.Options.Create(Options), + grainFactory, + NullLoggerFactory.Instance, + shared.ActivationDirectory, + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Microsoft.Extensions.Options.Options.Create(new LoadSheddingOptions()), + services, + shared); + services.Add(Dissemination); + services.Add(new DeploymentLoadStatisticsDisseminationNamespace( + Publisher, + new TestOptionsMonitor(Options), + _serializerServices.GetRequiredService())); + var lifecycle = Substitute.For(); + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + Lifecycle = call.ArgAt(2); + return Substitute.For(); + }); + ((ILifecycleParticipant)Publisher).Participate(lifecycle); + StartTimestamp = Clock.GetTimestamp(); + } + + public TimeSpan Period => TimeSpan.FromSeconds(1); + public long StartTimestamp { get; } + public SiloAddress Peer { get; } + public FakeTimeProvider Clock { get; } = new(); + public CohortPublisherTimerClock TimerClock { get; } + public CohortPublisherTimerRegistry Timers { get; } + public DeploymentLoadPublisherOptions Options { get; } + public DeploymentLoadPublisher Publisher { get; } + public CohortPublisherDisseminationService Dissemination { get; } = new(); + public ILifecycleObserver Lifecycle { get; private set; } = null!; + public ConcurrentQueue DirectUpdates { get; } = new(); + public Func DirectHandler { get; set; } = static _ => Task.CompletedTask; + + public async Task StartAsync(bool clearStartup = true) + { + await Lifecycle.OnStart(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (clearStartup) + { + Dissemination.Publications.Clear(); + _ = await Dissemination.NextPublicationAsync(); + Dissemination.QueryCount = 0; + DirectUpdates.Clear(); + } + } + + public async Task StartTickAsync() + { + await Publisher.RunOrQueueTask(() => + { + Timers.Timer.Change(TimeSpan.Zero, Period); + return Task.CompletedTask; + }); + return await Timers.NextTickAsync(); + } + + public Task OwnerBarrierAsync() => Publisher.QueueTask(static () => Task.CompletedTask); + + public Task PublishAsync(CancellationToken cancellationToken) => Publisher.RunOrQueueTask( + async token => + { + await Publisher.PublishStatistics(token); + return true; + }, + cancellationToken); + + public async ValueTask DisposeAsync() + { + await Lifecycle.OnStop(CancellationToken.None); + foreach (var tick in Timers.AllTicks) + { + try + { + await tick.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + catch (OperationCanceledException) when (TimerClock.Disposed) + { + } + } + + await OwnerBarrierAsync(); + _serializerServices.Dispose(); + } + } + + private sealed class CohortPublisherDisseminationService : IDisseminationService + { + private readonly Channel _publications = Channel.CreateUnbounded(); + + public ConcurrentQueue Publications { get; } = new(); + public DisseminationPublicationReceipt? AutomaticReceipt { get; set; } = new(true, TimeSpan.FromSeconds(1)); + public bool PreserveNativeCompletion { get; set; } + public Exception? Failure { get; set; } + public IReadOnlyList UnconfirmedPeers { get; set; } = []; + public int QueryCount { get; set; } + + public ValueTask Publish( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) => + throw new InvalidOperationException("Load publication must use the receipt-returning path."); + + public ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) + { + var publication = new CohortPublisherPublication(version, cancellationToken); + Publications.Enqueue(publication); + if (Failure is { } failure) + { + publication.Receipt.SetException(failure); + } + else if (AutomaticReceipt is { } receipt) + { + publication.Receipt.SetResult(receipt); + } + + var result = new ValueTask( + PreserveNativeCompletion ? publication.Receipt.Task : publication.Receipt.Task.WaitAsync(cancellationToken)); + _publications.Writer.TryWrite(publication); + return result; + } + + public IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace) + { + QueryCount++; + return UnconfirmedPeers; + } + + public Task NextPublicationAsync() => + _publications.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + private sealed record CohortPublisherPublication(long Version, CancellationToken Token) + { + public TaskCompletionSource Receipt { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + // Only message delivery is replaced: timer firing, Change, cancellation and callback completion + // use the real GrainTimer, and its callback runs on the publisher's actual SystemTarget scheduler. + private sealed class CohortPublisherTimerRegistry(TimerRegistry inner) : ITimerRegistry + { + private readonly Channel _ticks = Channel.CreateUnbounded(); + + public IGrainTimer Timer { get; private set; } = null!; + public ConcurrentQueue AllTicks { get; } = new(); + + public IGrainTimer RegisterGrainTimer( + IGrainContext grainContext, Func callback, TState state, GrainTimerCreationOptions options) + { + Assert.True(options.Interleave); + var context = Substitute.For(); + context.GrainId.Returns(grainContext.GrainId); + context.When(value => value.ReceiveMessage(Arg.Any())).Do(call => + { + var message = Assert.IsType(call.Arg()); + var invocation = Assert.IsAssignableFrom(message.BodyObject); + var tick = grainContext.QueueTask(async () => + { + using var response = await invocation.Invoke(); + if (response.Exception is { } exception) + { + ExceptionDispatchInfo.Capture(exception).Throw(); + } + }); + AllTicks.Enqueue(tick); + _ticks.Writer.TryWrite(tick); + }); + // Replace startup jitter with a deterministic offset; keep the runtime timer itself. + return Timer = inner.RegisterGrainTimer(context, callback, state, options with { DueTime = options.Period }); + } + + public IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period) => + throw new NotSupportedException(); + + public Task NextTickAsync() => + _ticks.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask() + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + private sealed class CohortPublisherTimerClock(FakeTimeProvider inner) : TimeProvider + { + public TimeSpan DueTime { get; private set; } + public int ChangeCount { get; private set; } + public bool Disposed { get; private set; } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + new RecordingTimer(this, inner.CreateTimer(callback, state, dueTime, period)); + + private sealed class RecordingTimer(CohortPublisherTimerClock owner, ITimer innerTimer) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + owner.DueTime = dueTime; + owner.ChangeCount++; + return innerTimer.Change(dueTime, period); + } + + public void Dispose() + { + owner.Disposed = true; + innerTimer.Dispose(); + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs new file mode 100644 index 00000000000..a3ec52426bb --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Cohorts.cs @@ -0,0 +1,342 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task HeldCohortReceiptKeepsMembershipPeerAdmissionAvailable() + { + var cancellationToken = TestContext.Current.CancellationToken; + var peer = CreateSilo(41301); + var local = CreateSilo(41302); + var load = CreateCohortNamespace(local); + var membership = new FakeNamespace(local, new DisseminationNamespace("urgent-membership")); + var transport = new FakeTransport(local, peer); + var ingressStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var heldReceipt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var membershipSent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.PublishAggregatedHandler = (_, _, _) => + { + ingressStarted.TrySetResult(); + return heldReceipt.Task; + }; + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + Assert.Equal(peer, target); + Assert.Equal(membership.Name, Assert.Single(batch.Values.Keys)); + membershipSent.TrySetResult(); + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, [load, membership], options => options.MaxConcurrentSends = 1); + try + { + load.SetValue(local, 1); + var publication = protocol.PublishAggregated(load, local, 1, cancellationToken).AsTask(); + await ingressStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + membership.SetValue("membership", 1); + Assert.True(await protocol.Publish(membership, "membership", 1, cancellationToken)); + await membershipSent.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.False(publication.IsCompleted); + heldReceipt.SetResult(new(true, TimeSpan.FromMilliseconds(750))); + Assert.Equal(new DisseminationPublicationReceipt(true, TimeSpan.FromMilliseconds(750)), await publication); + } + finally + { + heldReceipt.TrySetResult(default); + load.Options.Enabled = false; + membership.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Theory] + [InlineData(4)] + [InlineData(10)] + public async Task CompleteCohortIncludesRootAndUsesOneIngressAndDistributionTree(int count) + { + var token = TestContext.Current.CancellationToken; + var members = CreateSilos(count); + var clock = new FakeTimeProvider(); + var startedAt = clock.GetTimestamp(); + var namespaces = members.Select(CreateCohortNamespace).ToArray(); + var transports = members.Select(local => new FakeTransport(local, members.Where(peer => !peer.Equals(local)).ToArray())).ToArray(); + var protocols = transports.Select((transport, index) => CreateProtocol( + transport, namespaces[index], options => options.Overlay.AggregationFanOutFactor = 2, clock)).ToArray(); + for (var index = 0; index < count; index++) + { + transports[index].PublishAggregatedHandler = (peer, request, cancellationToken) => + { + Assert.Equal(members[0], peer); + return protocols[0].ReceivePublication(request, cancellationToken); + }; + var source = transports[index]; + source.SendBroadcastResponseHandler = async (peer, batch, cancellationToken) => + { + lock (source.BroadcastBatches) + { + source.BroadcastBatches.Add((peer, batch)); + } + return await protocols[Array.IndexOf(members, peer)].ReceiveBroadcast(batch, cancellationToken); + }; + namespaces[index].SetValue(members[index], 1); + } + + try + { + var remoteReceipts = Enumerable.Range(1, count - 1) + .Select(index => protocols[index].PublishAggregated(namespaces[index], members[index], 1, token).AsTask()) + .ToArray(); + Assert.All(remoteReceipts, receipt => Assert.False(receipt.IsCompleted)); + var localReceipt = protocols[0].PublishAggregated(namespaces[0], members[0], 1, token).AsTask(); + var receipts = await Task.WhenAll(remoteReceipts.Append(localReceipt)).WaitAsync(TimeSpan.FromSeconds(5), token); + Assert.All(receipts, receipt => + { + Assert.True(receipt.Accepted); + Assert.Equal(TimeSpan.FromSeconds(1), receipt.NextPublicationDelay); + }); + for (var index = 0; index < count; index++) + { + await protocols[index].FlushPendingBroadcast(token); + } + + Assert.Equal(count - 1, transports.Sum(transport => transport.PublicationRequests.Count)); + Assert.Equal(count - 1, transports.Sum(transport => transport.BroadcastBatches.Count)); + Assert.All(namespaces, ns => Assert.All(members, member => Assert.Equal(1, ns.GetVersion(member)))); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(startedAt)); + } + finally + { + foreach (var ns in namespaces) + { + ns.Options.Enabled = false; + } + await Task.WhenAll(protocols.Select(protocol => protocol.StopAsync(token))); + } + } + + [Fact] + public async Task StoppingRootSealsHeldIngressBeforeDrainingProtocolAdmissions() + { + var token = TestContext.Current.CancellationToken; + var local = CreateSilo(41401); + var peer = CreateSilo(41402); + var ns = CreateCohortNamespace(local); + var clock = new FakeTimeProvider(); + var startedAt = clock.GetTimestamp(); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns, timeProvider: clock); + try + { + var request = new DisseminationPublicationRequest { Sender = peer, Namespace = ns.Name, Value = ns.CreateItem(peer, peer, 1) }; + var receipt = protocol.ReceivePublication(request, token); + Assert.False(receipt.IsCompleted); + Assert.Equal(1, ns.GetVersion(peer)); + await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); + Assert.True((await receipt).Accepted); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(startedAt)); + } + finally + { + await protocol.StopAsync(token); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CanceledPublicationReleasesLocalAttemptBeforeNonCooperativeRpcReturns(bool stopProtocol) + { + var token = TestContext.Current.CancellationToken; + var root = CreateSilo(41411); + var local = CreateSilo(41412); + var ns = CreateCohortNamespace(local); + var transport = new FakeTransport(local, root); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var late = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.PublishAggregatedHandler = (_, request, _) => + { + started.TrySetResult(); + return request.Value.Value.ToVersion == 1 ? late.Task : Task.FromResult(new DisseminationPublicationReceipt(true, TimeSpan.FromSeconds(1))); + }; + var protocol = CreateProtocol(transport, ns); + try + { + using var cancellation = new CancellationTokenSource(); + ns.SetValue(local, 1); + var first = protocol.PublishAggregated(ns, local, 1, cancellation.Token).AsTask(); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); + if (stopProtocol) + { + await protocol.StopAsync(token).WaitAsync(TimeSpan.FromSeconds(5), token); + await Assert.ThrowsAnyAsync(() => first); + Assert.False((await protocol.PublishAggregated(ns, local, 1, token)).Accepted); + } + else + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => first); + Assert.Equal(cancellation.Token, exception.CancellationToken); + ns.SetValue(local, 2); + Assert.True((await protocol.PublishAggregated(ns, local, 2, token)).Accepted); + Assert.Equal(2, transport.PublicationRequests.Count); + } + Assert.False(late.Task.IsCompleted); + } + finally + { + late.TrySetResult(default); + await protocol.StopAsync(token); + } + } + + [Fact] + public async Task PublicationToFormerRootRejectsBeforeOwnerMutation() + { + var token = TestContext.Current.CancellationToken; + var currentRoot = CreateSilo(41431); + var local = CreateSilo(41432); + var producer = CreateSilo(41433); + var ns = CreateCohortNamespace(local); + var transport = new FakeTransport(local, currentRoot, producer); + var protocol = CreateProtocol(transport, ns); + try + { + var receipt = await protocol.ReceivePublication(new() + { + Sender = producer, + Namespace = ns.Name, + Value = ns.CreateItem(producer, producer, 1), + }, token); + Assert.False(receipt.Accepted); + Assert.Equal(0, ns.GetVersion(producer)); + Assert.Empty(ns.ApplyCounts); + Assert.Empty(transport.BroadcastBatches); + } + finally + { + await protocol.StopAsync(token); + } + } + + [Fact] + public void CohortPublicationWireContractPreservesPayloadAndSchedulingReceipt() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var producer = CreateSilo(41421); + var ns = CreateCohortNamespace(producer); + var request = new DisseminationPublicationRequest { Sender = producer, Namespace = ns.Name, Value = ns.CreateItem(producer, producer, 7) }; + var decoded = Assert.IsType(serializer.Deserialize(serializer.SerializeToArray(request))); + Assert.Equal(request.Sender, decoded.Sender); + Assert.Equal(request.Namespace, decoded.Namespace); + Assert.Equal(request.Value.Value.Key, decoded.Value.Value.Key); + Assert.Equal(7, decoded.Value.Value.ToVersion); + Assert.Equal(request.Value.Value.Payload.ToArray(), decoded.Value.Value.Payload.ToArray()); + Assert.Equal(request.Value.TimeToLive, decoded.Value.TimeToLive); + var receipt = new DisseminationPublicationReceipt(true, TimeSpan.FromMilliseconds(975)); + Assert.Equal(receipt, serializer.Deserialize(serializer.SerializeToArray(receipt))); + } + + private static FakeNamespace CreateCohortNamespace(SiloAddress local) + { + var result = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + return result; + } + + [Fact] + public async Task RootPendingStateHandsOffImmediatelyWhenTheRootChanges() + { + var cancellationToken = TestContext.Current.CancellationToken; + var newRoot = CreateSilo(41311); + var local = CreateSilo(41312); + var peer = CreateSilo(41313); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + ns.AggregationPeriod = TimeSpan.FromMilliseconds(25); + var transport = new FakeTransport(local, peer); + var handedOff = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + Assert.Equal(newRoot, target); + if (GetBroadcastValues(batch).Any(value => value.Value.Key == new DisseminationKey("pending"))) + { + handedOff.TrySetResult(); + } + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var protocol = CreateProtocol(transport, ns, timeProvider: clock); + var start = clock.GetTimestamp(); + try + { + ns.SetValue("pending", 1); + Assert.True(await protocol.Publish(ns, "pending", 1, cancellationToken)); + transport.Peers.Add(newRoot); + ns.SetValue("new", 1); + Assert.True(await protocol.Publish(ns, "new", 1, cancellationToken)); + clock.Advance(TimeSpan.Zero); + await handedOff.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(TimeSpan.Zero, clock.GetElapsedTime(start)); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task RootAcceptsAndDistributesACompleteTwoThousandSiloInventory() + { + var cancellationToken = TestContext.Current.CancellationToken; + var members = CreateSilos(2000); + var local = members[0]; + var ns = new FakeNamespace(local) + { + RoutingMode = DisseminationRoutingMode.AggregationTree, + MembershipScope = DisseminationMembershipScope.ActiveMembers, + }; + var defaults = new DeploymentLoadPublisherOptions().Dissemination; + ns.Options.MaxPendingItemCount = defaults.MaxPendingItemCount; + ns.AggregationPeriod = new DeploymentLoadPublisherOptions().DeploymentLoadPublisherRefreshTime; + var transport = new FakeTransport(local, members[1..]); + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + try + { + var values = members.Select(member => ns.CreateItem(member, member, 1)).ToArray(); + var response = await protocol.ReceiveBroadcast(new() + { + Sender = members[1], + Values = CreateValueGroups(values), + }, cancellationToken); + Assert.Equal(members.Length, response.Acknowledgments[ns.Name].Count); + await protocol.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(8, transport.BroadcastBatches.Count); + Assert.Equal(members[1..9], transport.BroadcastBatches.Select(batch => batch.Peer).Order()); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(members.Length, GetBroadcastValues(batch.Batch).Count()); + Assert.All(GetBroadcastValues(batch.Batch), value => Assert.Equal(1, value.Value.ToVersion)); + }); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs new file mode 100644 index 00000000000..d6feb4b1728 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.CompactAcknowledgments.cs @@ -0,0 +1,322 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CompactAcknowledgmentsRequireCallerSupport(bool supportsCompact) + { + var local = CreateSilo(40701); + var peer = CreateSilo(40702); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + SupportsCompactAcknowledgments = supportsCompact, + Values = CreateValueGroups(ns.CreateItem(peer, "first", 1), ns.CreateItem(peer, "second", 2)), + }; + try + { + var response = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + + Assert.Equal(supportsCompact, response.AllVersionsAcknowledged); + Assert.Empty(response.UnsupportedNamespaces); + Assert.Equal(ns.Name, Assert.Single(response.Acknowledgments.Keys)); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(2, ns.GetVersion("second")); + if (supportsCompact) + { + Assert.Empty(response.Acknowledgments[ns.Name]); + } + else + { + Assert.Equal(new long[] { 1, 2 }, response.Acknowledgments[ns.Name].Select(static value => value.Version)); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task CompactAcknowledgmentsPreserveExactAheadVersions() + { + var local = CreateSilo(40711); + var peer = CreateSilo(40712); + var ns = new FakeNamespace(local); + ns.SetValue("value", 3); + var protocol = CreateProtocol(new FakeTransport(local, peer), ns); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = CreateValueGroups(ns.CreateItem(peer, "value", 2)), + }, TestContext.Current.CancellationToken); + + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(3, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Equal(3, ns.GetVersion("value")); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData("rejected")] + [InlineData("disabled")] + [InlineData("item-limit")] + public async Task CompactAcknowledgmentsRetainExplicitPartialOutcomes(string scenario) + { + var local = CreateSilo(40721); + var peer = CreateSilo(40722); + var ns = new ProtocolReviewNamespace(local); + var other = new FakeNamespace(local, "other"); + other.Options.Enabled = scenario != "disabled"; + ns.ApplyHandler = (value, token) => value.Key == new DisseminationKey("second") && scenario == "rejected" + ? new(DisseminationApplyResult.Rejected) + : ns.Inner.ApplyValueAsync(value, token); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns, other], options => + { + options.MaxBatchItems = scenario == "item-limit" ? 1 : 10; + }); + var values = CreateValueGroups(ns.Inner.CreateItem(peer, "first", 1), ns.Inner.CreateItem(peer, "second", 2)); + if (scenario == "disabled") + { + values[other.Name] = [other.CreateItem(peer, "other", 1)]; + } + + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = values, + }, TestContext.Current.CancellationToken); + + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(1, response.Acknowledgments[ns.Name].Single(value => value.Key == new DisseminationKey("first")).Version); + if (scenario == "rejected") + { + Assert.Equal(0, response.Acknowledgments[ns.Name].Single(value => value.Key == new DisseminationKey("second")).Version); + } + else if (scenario == "item-limit") + { + Assert.Single(response.Acknowledgments[ns.Name]); + Assert.Equal(0, ns.GetVersion("second")); + } + else + { + Assert.Equal(other.Name, Assert.Single(response.UnsupportedNamespaces)); + Assert.False(response.Acknowledgments.ContainsKey(other.Name)); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public void CompactAcknowledgmentFieldsPreserveLegacyWireSchemas() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var peer = CreateSilo(40731); + var current = new DisseminationBroadcastBatch + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = CreateValueGroups(new FakeNamespace(peer).CreateItem(peer, "value", 1)), + }; + var legacyBatch = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(current))); + Assert.Equal(peer, legacyBatch.Sender); + Assert.Equal(1, Assert.Single(legacyBatch.Values[FakeNamespace.DefaultName]).Value.ToVersion); + var request = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(legacyBatch))); + Assert.False(request.SupportsCompactAcknowledgments); + + var legacyResponse = new CompactReviewLegacyResponse + { + Acknowledgments = new() { [FakeNamespace.DefaultName] = [new("value", 3)] }, + }; + var response = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(legacyResponse))); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(3, Assert.Single(response.Acknowledgments[FakeNamespace.DefaultName]).Version); + var oldReader = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(response))); + Assert.Equal(3, Assert.Single(oldReader.Acknowledgments[FakeNamespace.DefaultName]).Version); + var compact = new DisseminationBroadcastResponse + { + AllVersionsAcknowledged = true, + Acknowledgments = new() { [FakeNamespace.DefaultName] = [] }, + }; + var ignoresNewField = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(compact))); + Assert.Equal(FakeNamespace.DefaultName, Assert.Single(ignoresNewField.Acknowledgments.Keys)); + Assert.Empty(ignoresNewField.Acknowledgments[FakeNamespace.DefaultName]); + } + + [Fact] + public void CompactAcknowledgmentBodyIsSmallerAndRoundTrips() + { + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var explicitResponse = new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = CreateSilos(128).Select(silo => new DigestEntry(silo, 639000000000000000)).ToList(), + }, + }; + var compactResponse = new DisseminationBroadcastResponse + { + AllVersionsAcknowledged = true, + Acknowledgments = new() { [FakeNamespace.DefaultName] = [] }, + }; + + var explicitBytes = serializer.SerializeToArray(explicitResponse); + var compactBytes = serializer.SerializeToArray(compactResponse); + var roundTrip = Assert.IsType( + serializer.Deserialize(compactBytes)); + + Assert.True(roundTrip.AllVersionsAcknowledged); + Assert.Empty(roundTrip.Acknowledgments[FakeNamespace.DefaultName]); + Assert.True(compactBytes.Length * 4 < explicitBytes.Length, + $"Compact acknowledgment body={compactBytes.Length} bytes; explicit body={explicitBytes.Length} bytes."); + } + + [Fact] + public async Task PruningScratchReuseBoundsSteadyStateAllocations() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40751); + var ns = new CompactReviewInventoryNamespace(local); + var keys = Enumerable.Range(0, 1024).Select(index => new DisseminationKey($"key-{index}")).ToArray(); + ns.ReadKeys = () => keys; + var transport = new FakeTransport(local); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + var snapshots = membership.CurrentSnapshots; + try + { + await queue.Prune(snapshots, cancellationToken); + await queue.Prune(snapshots, cancellationToken); + const int iterations = 128; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var iteration = 0; iteration < iterations; iteration++) + { + var operation = queue.Prune(snapshots, cancellationToken); + if (!operation.IsCompletedSuccessfully) + { + throw new InvalidOperationException("An empty peer set must prune synchronously."); + } + } + + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(allocated < iterations * 512, + $"Pruning a stable 1024-key inventory allocated {allocated} bytes over {iterations} operations."); + } + finally + { + await queue.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task PruningScratchInventoryIsClearedAfterEnumerationFailure() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40741); + var peer = CreateSilo(40742); + var ns = new CompactReviewInventoryNamespace(local); + ns.Inner.SetValue("old", 1); + var transport = new FakeTransport(local, peer); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + try + { + Assert.True(queue.Notify(peer, ns, "old")); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Single(transport.BroadcastBatches); + + ns.ReadKeys = FailingKeys; + await Assert.ThrowsAsync(() => queue.Prune(membership.CurrentSnapshots, cancellationToken)); + ns.ReadKeys = static () => new DisseminationKey[] { "new" }; + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + ns.ReadKeys = static () => new DisseminationKey[] { "old", "new" }; + + Assert.True(queue.Notify(peer, ns, "old", force: false)); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + } + finally + { + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + static IEnumerable FailingKeys() + { + yield return "old"; + throw new InvalidOperationException("Inventory enumeration failed after producing a key."); + } + } + + private sealed class CompactReviewInventoryNamespace(SiloAddress local) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local); + public Func> ReadKeys { get; set; } = static () => new DisseminationKey[] { "old" }; + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Keys => ReadKeys(); + public IEnumerable Digests => Inner.Digests; + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => Inner.CreateRepair(request); + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + Inner.ApplyValueAsync(value, cancellationToken); + } + + [GenerateSerializer] + internal sealed class CompactReviewLegacyBatch + { + [Id(0)] public SiloAddress Sender { get; init; } = null!; + [Id(1)] public Dictionary> Values { get; init; } = []; + } + + [GenerateSerializer] + internal sealed class CompactReviewLegacyResponse + { + [Id(0)] public Dictionary> Acknowledgments { get; init; } = []; + [Id(1)] public List UnsupportedNamespaces { get; init; } = []; + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs new file mode 100644 index 00000000000..01a34835e6e --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FinalComments.cs @@ -0,0 +1,172 @@ +#nullable enable + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipInventoryOnlyRepairConvergesWithoutHeartbeat(bool advanceVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40301); + var sourceAddress = CreateSilo(40302); + var live = CreateMembershipEntry( + CreateSilo(40303), SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20)); + var retired = CreateMembershipEntry(CreateSilo(40304), SiloStatus.Dead, DateTime.UnixEpoch); + var previous = CreateMembershipSnapshot(2, live, retired); + var version = previous.Version.Value + (advanceVersion ? 1 : 0); + var incoming = CreateMembershipSnapshot(version, live.WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))); + var expected = CreateMembershipSnapshot(version, live); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var receiverManager = CreateMembershipReviewManager(local, out _); + using var sourceManager = CreateMembershipReviewManager(sourceAddress, out _); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(previous, cancellationToken); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(expected, cancellationToken); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var before = Assert.Single(receiver.Digests); + var update = new MembershipTableSnapshotUpdate { Snapshot = incoming }; + var value = new DisseminationValue(DisseminationKey.Default, 0, version, serializer.SerializeToArray(update)); + + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); + Assert.NotEqual(before.Fingerprint, Assert.Single(receiver.Digests).Fingerprint); + Assert.Equal(version, Assert.Single(receiver.Digests).Version); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(value, cancellationToken)); + var staleInventory = new DisseminationValue( + DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = previous })); + Assert.Equal(advanceVersion ? DisseminationApplyResult.Obsolete : DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(staleInventory, cancellationToken)); + AssertMembershipState(expected, receiverManager.MembershipTableSnapshot); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), incoming.Entries[live.SiloAddress].IAmAliveTime); + + var protocol = CreateProtocol(new FakeTransport(sourceAddress, local), [source]); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = local, + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(response)); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task MembershipInventoryCleanupDoesNotAcknowledgeRetainedLocalEntry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40311); + var peer = CreateSilo(40312); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(local, out _); + var localEntry = manager.MembershipTableSnapshot.Entries[local].WithStatus(SiloStatus.Dead); + var previous = CreateMembershipSnapshot( + 2, localEntry, CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var current = manager.MembershipTableSnapshot; + var incoming = CreateMembershipSnapshot(2, previous.Entries[peer]); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, 0, 2, serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); + + Assert.True(incoming.IsSuccessorTo(current)); + Assert.Equal(DisseminationApplyResult.Rejected, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.Same(current, manager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, manager.MembershipTableSnapshot.Entries[local].Status); + Assert.Equal(new[] { local, peer }, manager.MembershipTableSnapshot.Entries.Keys.Order()); + + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = new() { [ns.Name] = [new() { Value = value, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }, cancellationToken); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(current.Version.Value, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.Same(current, manager.MembershipTableSnapshot); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisseminationStopReportsCallerCancellationAfterCleanup(bool cancelCaller) + { + var local = CreateSilo(40321); + var transport = new FakeTransport(local, CreateSilo(40322)); + var options = new ReviewOptionsMonitor(new DisseminationOptions { Enabled = false }); + var clock = new ReviewTimeProvider(); + var details = new FakeLocalSiloDetails(local); + var target = new DisseminationSystemTarget( + details, + transport.GrainFactory, + new DisseminationMembership(transport.MembershipManager, details, Options.Create(options.CurrentValue)), + options, + [new FakeNamespace(local)], + clock, + NullLogger.Instance, + NullLogger.Instance, + CreatePhase4SystemTargetShared(local)); + var lifecycle = Substitute.For(); + ILifecycleObserver? observer = null; + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + observer = call.ArgAt(2); + return new ReviewSubscription(static () => { }); + }); + ((ILifecycleParticipant)target).Participate(lifecycle); + Assert.NotNull(observer); + await observer.OnStart(TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + if (cancelCaller) + { + caller.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => observer.OnStop(caller.Token).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(caller.Token, exception.CancellationToken); + } + else + { + await observer.OnStop(caller.Token).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await options.Unsubscribed.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, options.SubscriptionCount); + Assert.Empty(transport.AntiEntropyRequests); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs new file mode 100644 index 00000000000..01a83bfae6e --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.FollowupReview.cs @@ -0,0 +1,244 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task CompletedAsyncApplicationWinsAConcurrentLocalWaitTimeout() + { + var local = CreateSilo(40401); + var sender = CreateSilo(40402); + var child = CreateSilo(40403); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local, "async-terminal-result-review"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var value = ns.Inner.CreateValue("value", 1); + var application = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = (_, _) => new(application.Task); + var transport = new FakeTransport(local, sender, child); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2, clock); + var context = new MembershipReviewContinuationContext(); + var previousContext = SynchronizationContext.Current; + Task receive; + SynchronizationContext.SetSynchronizationContext(context); + try + { + receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(sender, value)), + }, TestContext.Current.CancellationToken); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + try + { + clock.Advance(ns.Options.StaleItemTtl); + var timedOutWait = await context.TakeContinuation(TestContext.Current.CancellationToken); + Assert.False(receive.IsCompleted); + ns.Inner.PublishValue(value); + application.SetResult(DisseminationApplyResult.Applied); + timedOutWait.Callback(timedOutWait.State); + for (var index = 0; index < 4 && !receive.IsCompleted; index++) + { + var continuation = await context.TakeContinuation(TestContext.Current.CancellationToken); + continuation.Callback(continuation.State); + } + + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + Assert.Equal(1, ns.GetVersion("value")); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.NotEmpty(transport.AntiEntropyRequests); + Assert.All(transport.AntiEntropyRequests, request => Assert.False(request.Request.Digests.ContainsKey(ns.Name))); + } + finally + { + application.TrySetResult(DisseminationApplyResult.Rejected); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task CompletedApplicationRemainsAppliedWhenItsLocalDeadlineRacesCompletion() + { + var local = CreateSilo(39651); + var sender = CreateSilo(39652); + var child = CreateSilo(39653); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local, "terminal-result-review"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + CancellationToken applicationToken = default; + ns.ApplyHandler = (value, token) => + { + applicationToken = token; + var result = ns.Inner.ApplyValueAsync(value, token); + Assert.True(result.IsCompletedSuccessfully); + clock.Advance(ns.Options.StaleItemTtl); + return result; + }; + var transport = new FakeTransport(local, sender, child); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.FanOutFactor = static _ => 2, clock); + try + { + var response = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = sender, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(sender, "value", 1)), + }, TestContext.Current.CancellationToken); + + Assert.True(applicationToken.IsCancellationRequested); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + Assert.Equal(1, Assert.Single(GetBroadcastValues(forwarded.Batch)).Value.ToVersion); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.NotEmpty(transport.AntiEntropyRequests); + Assert.All(transport.AntiEntropyRequests, request => Assert.False(request.Request.Digests.ContainsKey(ns.Name))); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ChangedOversizedBatchRejectsNonFullValuesBeforeOwnerApplication() + { + var local = CreateSilo(39661); + var peer = CreateSilo(39662); + var first = new FakeNamespace(local, "cursor-first"); + var second = new FakeNamespace(local, "cursor-second"); + second.ExpectedKeys.Add("stream"); + var protocol = CreateProtocol( + new FakeTransport(local, peer), + [first, second], + options => options.MaxBatchItems = 1); + try + { + var firstResponse = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = new Dictionary> + { + [first.Name] = [first.CreateItem(peer, "other", 1)], + [second.Name] = [second.CreateItem(peer, "stream", 1)], + }, + }, TestContext.Current.CancellationToken); + Assert.False(firstResponse.Acknowledgments.ContainsKey(second.Name)); + + var changedBatch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, + second.CreateItem(peer, "stream", 1), + second.CreateItem(peer, "stream", 2, fromVersion: 1)), + }; + var skippedPrefix = await protocol.ReceiveBroadcast(changedBatch, TestContext.Current.CancellationToken); + Assert.Equal(0, Assert.Single(skippedPrefix.Acknowledgments[second.Name]).Version); + Assert.Equal(0, second.GetVersion("stream")); + + var prefix = await protocol.ReceiveBroadcast(changedBatch, TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(prefix.Acknowledgments[second.Name]).Version); + var nonFull = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, second.CreateItem(peer, "stream", 2, fromVersion: 1)), + }, TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(nonFull.Acknowledgments[second.Name]).Version); + Assert.Equal(1, second.GetVersion("stream")); + Assert.Equal(1, second.ApplyCounts["stream"]); + + var suffix = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(second.Name, second.CreateItem(peer, "stream", 2)), + }, TestContext.Current.CancellationToken); + Assert.Equal(2, Assert.Single(suffix.Acknowledgments[second.Name]).Version); + Assert.Equal(1, first.GetVersion("other")); + Assert.Equal(2, second.GetVersion("stream")); + Assert.Equal(2, second.ApplyCounts["stream"]); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublicationRejectsSingleValueBeyondGlobalBatchBudget() + { + var local = CreateSilo(39671); + var peer = CreateSilo(39672); + var ns = new FakeNamespace(local); + ns.Options.MaxPayloadBytes = 16; + var payload = new byte[9]; + BitConverter.GetBytes(1L).CopyTo(payload, 0); + ns.PublishValue(new DisseminationValue("value", 0, 1, payload)); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => options.MaxBatchBytes = 8); + try + { + Assert.False(await protocol.Publish(ns, "value", 1, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + Assert.Equal(0, transport.GetTargetResolutionCount(peer)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublicationUsesCurrentFullRepairWithinActualSendBudgets() + { + var local = CreateSilo(39681); + var peer = CreateSilo(39682); + var ns = new FakeNamespace(local); + ns.PublishValue(ns.CreateValue("chain", 1)); + ns.PublishValue(ns.CreateValue("chain", 2)); + ns.PublishValue(ns.CreateValue("chain", 3)); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }); + try + { + Assert.True(await protocol.Publish(ns, "chain", 3, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 3 }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + Assert.All(transport.BroadcastBatches, batch => + { + Assert.Equal(8, Assert.Single(GetBroadcastValues(batch.Batch)).Value.Payload.Length); + Assert.Equal(0, Assert.Single(GetBroadcastValues(batch.Batch)).Value.FromVersion); + }); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs new file mode 100644 index 00000000000..062479c1005 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.Maintenance.cs @@ -0,0 +1,178 @@ +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ReceiveTrafficSharesInventoryMaintenanceUntilTimeOrMembershipChanges() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41201); + var peer = CreateSilo(41202); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new CompactReviewInventoryNamespace(local); + var inventories = 0; + DisseminationKey[] keys = ["value"]; + ns.ReadKeys = () => + { + inventories++; + return keys; + }; + var protocol = CreateProtocol(transport, [ns], timeProvider: clock); + try + { + for (var version = 1; version <= 2000; version++) + { + await Receive(version); + } + Assert.Equal(1, inventories); + Assert.Equal(2000, ns.GetVersion("value")); + + clock.Advance(TimeSpan.FromMilliseconds(999)); + await Receive(2001); + Assert.Equal(1, inventories); + clock.Advance(TimeSpan.FromMilliseconds(1)); + await Receive(2002); + Assert.Equal(2, inventories); + + transport.Peers.Add(CreateSilo(41203)); + await Receive(2003); + Assert.Equal(3, inventories); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + + Task Receive(long version) => protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", version)), + }, cancellationToken); + } + + [Fact] + public async Task FailedInventoryMaintenanceIsRetriedWithoutWaitingForTheInterval() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41211); + var peer = CreateSilo(41212); + var ns = new CompactReviewInventoryNamespace(local); + var inventories = 0; + ns.ReadKeys = () => ++inventories == 1 + ? throw new InvalidOperationException("Inventory failed.") + : new DisseminationKey[] { "value" }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: new FakeTimeProvider()); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }; + try + { + await Assert.ThrowsAsync(() => protocol.ReceiveBroadcast(batch, cancellationToken)); + var response = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal(2, inventories); + Assert.Equal(1, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task InventoryMaintenanceAllowsSynchronousPublicationReentry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41221); + var peer = CreateSilo(41222); + var ns = new CompactReviewInventoryNamespace(local); + ns.Inner.SetValue("nested", 1); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: new FakeTimeProvider()); + Task? nested = null; + ns.ReadKeys = () => + { + nested = protocol.Publish(ns, "nested", 1, cancellationToken).AsTask(); + Assert.True(nested.IsCompletedSuccessfully); + return new DisseminationKey[] { "nested", "value" }; + }; + try + { + await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "value", 1)), + }, cancellationToken); + + Assert.NotNull(nested); + Assert.True(await nested); + Assert.Equal(1, ns.GetVersion("value")); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Theory] + [InlineData(1, 64, 1)] + [InlineData(8, 16, 2)] + public async Task AntiEntropyUsesIndependentResponseAndReceiveBudgets(int repairItems, int repairBytes, int expectedItems) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41231); + var peer = CreateSilo(41232); + var ns = new FakeNamespace(local); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxBatchItems = 8; + options.MaxBatchBytes = 64; + options.Overlay.MaxAntiEntropyBatchItems = repairItems; + options.Overlay.MaxAntiEntropyBatchBytes = repairBytes; + }); + var keys = new DisseminationKey[] { "first", "second", "third", "fourth" }; + ns.ExpectedKeys.UnionWith(keys); + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + Assert.Equal(repairItems, request.MaxResponseItems); + Assert.Equal(repairBytes, request.MaxResponseBytes); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.Name, keys.Select(key => ns.CreateItem(peer, key, 1)).ToArray()), + }); + }; + try + { + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(expectedItems, keys.Count(key => ns.GetVersion(key) == 1)); + + var broadcast = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, keys.Select(key => ns.CreateItem(peer, key, 2)).ToArray()), + }, cancellationToken); + Assert.Equal(keys.Length, broadcast.Acknowledgments[ns.Name].Count); + Assert.All(keys, key => Assert.Equal(2, ns.GetVersion(key))); + + var repair = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = keys.Select(key => new DigestEntry(key, 0)).ToList() }, + }, cancellationToken); + Assert.Equal(expectedItems, GetAntiEntropyResponseValues(repair).Count()); + Assert.True(repair.Truncated); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs new file mode 100644 index 00000000000..7a9aa2843ed --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipDeltas.cs @@ -0,0 +1,662 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public void MembershipBroadcastDeltaIsSparseWhileRepairRemainsFull() + { + var members = Enumerable.Range(43000, 2000).Select(CreateSilo).ToArray(); + var initial = CreateMembershipSnapshot(1, members.Select(silo => + CreateMembershipEntry(silo, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var ns = CreateMembershipNamespace(manager, serializer); + var request = new DisseminationRepairRequest(DisseminationKey.Default, 1, 1024 * 1024, 1024 * 1024); + var probe = ns.CreateBroadcast(request, baseline: null); + Assert.Equal(DisseminationRepairStatus.Produced, probe.Status); + Assert.Empty(ReadMembershipDelta(serializer, probe.Value).UpdatedEntries); + var baseline = Assert.IsAssignableFrom(probe.BroadcastState); + manager.CurrentSnapshot = new(new MembershipVersion(2), + initial.Entries.SetItem(members[999], initial.Entries[members[999]].WithStatus(SiloStatus.ShuttingDown))); + + var broadcast = ns.CreateBroadcast(request, baseline); + var full = ns.CreateRepair(request); + + Assert.Equal(DisseminationRepairStatus.Produced, broadcast.Status); + Assert.Equal(DisseminationRepairStatus.Produced, full.Status); + var delta = ReadMembershipDelta(serializer, broadcast.Value); + Assert.Equal((1L, 2L), (broadcast.Value.FromVersion, broadcast.Value.ToVersion)); + var changed = Assert.Single(delta.UpdatedEntries); + Assert.Equal(members[999], changed.SiloAddress); + Assert.Equal(SiloStatus.ShuttingDown, changed.Status); + Assert.Empty(delta.RemovedSilos); + Assert.Equal(0, full.Value.FromVersion); + var repair = Assert.IsType(serializer.Deserialize(full.Value.Payload)); + Assert.Null(repair.Delta); + Assert.Equal(2000, Assert.IsType(repair.Snapshot).Entries.Count); + Assert.True(broadcast.Value.Payload.Length * 10 < full.Value.Payload.Length, + $"Sparse broadcast {broadcast.Value.Payload.Length} bytes, full repair {full.Value.Payload.Length} bytes."); + Assert.Equal(broadcast.Value.Payload, ns.CreateBroadcast(request, baseline).Value.Payload); + } + + [Fact] + public async Task MembershipBroadcastCoalescesFromAcknowledgedSnapshot() + { + var local = CreateSilo(45001); + var peer = CreateSilo(45002); + var removed = CreateSilo(45003); + var added = CreateSilo(45004); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(removed, SiloStatus.Dead, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + await link.Send(); + // Metadata can return to its prior value across skipped observations; compare with the actual baseline. + var temporaryEntry = CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch); + temporaryEntry.HostName = "temporary"; + var temporary = new MembershipTableSnapshot(new MembershipVersion(2), + initial.Entries.SetItem(local, temporaryEntry)); + var final = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.Remove(removed) + .SetItem(peer, initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))) + .Add(added, CreateMembershipEntry(added, SiloStatus.Joining, DateTime.UnixEpoch))); + var flush = BeforeBroadcastPumpsRun(() => + { + link.SourceManager.CurrentSnapshot = temporary; + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + link.SourceManager.CurrentSnapshot = final; + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + return link.Queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + }); + + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, link.Batches.Count); + var value = Assert.Single(GetBroadcastValues(link.Batches[1])).Value; + var delta = ReadMembershipDelta(link.Serializer, value); + Assert.Equal((1L, 3L), (value.FromVersion, value.ToVersion)); + Assert.Equal(new[] { peer, added }, delta.UpdatedEntries.Select(entry => entry.SiloAddress).Order()); + Assert.Equal(removed, Assert.Single(delta.RemovedSilos)); + AssertMembershipState(final, link.TargetManager.CurrentSnapshot); + } + + [Fact] + public async Task MembershipBroadcastCapturesBaselineBeforeAcknowledgment() + { + var local = CreateSilo(45101); + var peer = CreateSilo(45102); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + link.BeforeResponse = async (_, index, token) => + { + if (index == 1) + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + } + }; + try + { + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + link.SourceManager.CurrentSnapshot = new(initial.Version, initial.Entries.SetItem( + local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + Assert.True(link.Queue.Notify(peer, link.Source, DisseminationKey.Default)); + var flush = link.Queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + release.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, link.Batches.Count); + var delta = ReadMembershipDelta(link.Serializer, Assert.Single(GetBroadcastValues(link.Batches[1])).Value); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), Assert.Single(delta.UpdatedEntries).IAmAliveTime); + AssertMembershipState(link.SourceManager.CurrentSnapshot, link.TargetManager.CurrentSnapshot); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task MembershipBroadcastGapUsesFullAntiEntropyRepair() + { + var local = CreateSilo(45201); + var peer = CreateSilo(45202); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + var source = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.SetItem(local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + await using var link = new MembershipDeltaLink(local, peer, source, initial); + await link.Send(); + var probe = Assert.Single(GetBroadcastValues(Assert.Single(link.Batches))).Value; + Assert.Equal((3L, 3L), (probe.FromVersion, probe.ToVersion)); + Assert.Equal(1, link.TargetManager.CurrentSnapshot.Version.Value); + Assert.False(Assert.Single(link.Responses).AllVersionsAcknowledged); + + var repair = await link.SourceProtocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [link.Source.Name], + Digests = new() { [link.Source.Name] = [.. link.Target.Digests] }, + }, TestContext.Current.CancellationToken); + var full = Assert.Single(repair.Values[link.Source.Name]).Value; + Assert.Equal(0, full.FromVersion); + Assert.Null(Assert.IsType( + link.Serializer.Deserialize(full.Payload)).Delta); + Assert.Equal(DisseminationApplyResult.Applied, + await link.Target.ApplyValueAsync(full, TestContext.Current.CancellationToken)); + await link.Send(); + + AssertMembershipState(source, link.TargetManager.CurrentSnapshot); + Assert.All(link.Batches.SelectMany(GetBroadcastValues), item => Assert.True(item.Value.FromVersion > 0)); + Assert.True(link.Responses[^1].AllVersionsAcknowledged); + } + + [Fact] + public async Task MembershipDeltaRetransmitsCumulativeChangesAfterLostAcknowledgment() + { + var local = CreateSilo(45601); + var peer = CreateSilo(45602); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await using var link = new MembershipDeltaLink(local, peer, initial, initial); + await link.Send(); + link.BeforeResponse = (_, index, _) => index == 2 + ? Task.FromException(new InvalidOperationException("Acknowledgment lost.")) + : Task.CompletedTask; + link.SourceManager.CurrentSnapshot = new(initial.Version, initial.Entries.SetItem( + local, initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + await link.Send(); + link.SourceManager.CurrentSnapshot = new(initial.Version, link.SourceManager.CurrentSnapshot.Entries.SetItem( + peer, initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20)))); + + await link.Send(); + + Assert.Equal(3, link.Batches.Count); + var delta = ReadMembershipDelta(link.Serializer, Assert.Single(GetBroadcastValues(link.Batches[2])).Value); + Assert.Equal(new[] { local, peer }, delta.UpdatedEntries.Select(entry => entry.SiloAddress).Order()); + AssertMembershipState(link.SourceManager.CurrentSnapshot, link.TargetManager.CurrentSnapshot); + } + + [Fact] + public async Task AheadMembershipRelayBroadcastsLatestDeltaFromChildBaseline() + { + var root = CreateSilo(45801); + var relay = CreateSilo(45802); + var child = CreateSilo(45803); + var members = new[] { root, relay, child }; + var initial = CreateMembershipSnapshot(1, members.Select(silo => + CreateMembershipEntry(silo, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(silo => silo, _ => new FakeMembershipManager(initial)); + var namespaces = members.ToDictionary(silo => silo, silo => CreateMembershipNamespace(managers[silo], serializer)); + var transports = members.ToDictionary(silo => silo, silo => new FakeTransport(silo, members.Where(other => !other.Equals(silo)).ToArray())); + var protocols = members.ToDictionary(silo => silo, silo => CreateProtocol( + transports[silo], [namespaces[silo]], + options => options.Overlay.FanOutFactor = static _ => 1)); + var sent = new Dictionary> + { + [root] = [], + [relay] = [], + [child] = [], + }; + foreach (var silo in members) + { + transports[silo].SendBroadcastResponseHandler = (peer, batch, token) => + { + sent[silo].Add(batch); + return protocols[peer].ReceiveBroadcast(batch, token); + }; + } + + var token = TestContext.Current.CancellationToken; + try + { + Assert.True(await protocols[root].Publish(namespaces[root], DisseminationKey.Default, 1, token)); + await protocols[root].FlushPendingBroadcast(token); + await protocols[relay].FlushPendingBroadcast(token); + Assert.Single(sent[root]); + Assert.Single(sent[relay]); + + var second = new MembershipTableSnapshot(new MembershipVersion(2), + initial.Entries.SetItem(root, initial.Entries[root].WithStatus(SiloStatus.ShuttingDown))); + var third = new MembershipTableSnapshot(new MembershipVersion(3), + initial.Entries.SetItem(root, initial.Entries[root].WithStatus(SiloStatus.Stopping))); + managers[relay].CurrentSnapshot = third; + managers[root].CurrentSnapshot = second; + Assert.True(await protocols[root].Publish(namespaces[root], DisseminationKey.Default, 2, token)); + await protocols[root].FlushPendingBroadcast(token); + await protocols[relay].FlushPendingBroadcast(token); + + Assert.Equal(2, sent[root].Count); + Assert.Equal(2, sent[relay].Count); + var rootDelta = ReadMembershipDelta(serializer, Assert.Single(GetBroadcastValues(sent[root][1])).Value); + var relayDelta = ReadMembershipDelta(serializer, Assert.Single(GetBroadcastValues(sent[relay][1])).Value); + Assert.Equal((1L, 2L), (rootDelta.BaseVersion.Value, rootDelta.Version.Value)); + Assert.Equal((1L, 3L), (relayDelta.BaseVersion.Value, relayDelta.Version.Value)); + Assert.Equal(SiloStatus.Stopping, Assert.Single(relayDelta.UpdatedEntries).Status); + AssertMembershipState(third, managers[child].CurrentSnapshot); + Assert.Empty(sent[child]); + Assert.All(transports.Values, transport => Assert.Empty(transport.AntiEntropyRequests)); + } + finally + { + var canceled = new CancellationToken(canceled: true); + try + { + await Task.WhenAll(protocols.Values.Select(protocol => protocol.StopAsync(canceled))) + .WaitAsync(TimeSpan.FromSeconds(5), token); + } + catch (OperationCanceledException) when (canceled.IsCancellationRequested) + { + } + } + } + + [Fact] + public async Task MembershipDeltaRejectsIntermediateViewAndMergesTargetReplay() + { + var local = CreateSilo(45301); + var inactive = CreateSilo(45302); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(inactive, SiloStatus.Dead, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(new MembershipTableSnapshot(new MembershipVersion(2), initial.Entries)); + var ns = CreateMembershipNamespace(manager, serializer); + var updated = initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)); + var value = CreateMembershipDelta(serializer, 1, 3, [updated], [inactive]); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(2, manager.CurrentSnapshot.Version.Value); + Assert.Contains(inactive, manager.CurrentSnapshot.Entries.Keys); + + var canonical = initial.Entries[local].WithStatus(SiloStatus.ShuttingDown); + canonical.HostName = "canonical"; + manager.CurrentSnapshot = CreateMembershipSnapshot(3, canonical); + Assert.Equal(DisseminationApplyResult.Applied, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(SiloStatus.ShuttingDown, manager.CurrentSnapshot.Entries[local].Status); + Assert.Equal("canonical", manager.CurrentSnapshot.Entries[local].HostName); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), manager.CurrentSnapshot.Entries[local].IAmAliveTime); + Assert.DoesNotContain(inactive, manager.CurrentSnapshot.Entries.Keys); + Assert.Equal(DisseminationApplyResult.Duplicate, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + } + + [Theory] + [InlineData("duplicate-upsert")] + [InlineData("upsert-and-remove")] + [InlineData("same-view-removal")] + [InlineData("same-view-addition")] + [InlineData("wire-version-mismatch")] + public async Task MembershipDeltaRejectsInvalidShapeBeforeOwnerMutation(string scenario) + { + var local = CreateSilo(45911); + var added = CreateSilo(45912); + var initial = CreateMembershipSnapshot(1, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var ns = CreateMembershipNamespace(manager, serializer); + var entry = initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)); + var value = scenario switch + { + "duplicate-upsert" => CreateMembershipDelta(serializer, 1, 2, [entry, entry], []), + "upsert-and-remove" => CreateMembershipDelta(serializer, 1, 2, [entry], [local]), + "same-view-removal" => CreateMembershipDelta(serializer, 1, 1, [], [local]), + "same-view-addition" => CreateMembershipDelta(serializer, 1, 1, + [CreateMembershipEntry(added, SiloStatus.Joining, DateTime.UnixEpoch)], []), + "wire-version-mismatch" => new DisseminationValue(DisseminationKey.Default, 1, 3, + CreateMembershipDelta(serializer, 1, 2, [entry], []).Payload), + _ => throw new InvalidOperationException(scenario), + }; + var ownerCalls = 0; + manager.ProcessGossipSnapshotHandler = (_, _) => + { + ownerCalls++; + return Task.CompletedTask; + }; + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(0, ownerCalls); + Assert.Same(initial, manager.CurrentSnapshot); + } + + [Theory] + [InlineData(false, SiloStatus.Dead)] + [InlineData(true, SiloStatus.Dead)] + [InlineData(false, SiloStatus.Created)] + [InlineData(true, SiloStatus.Created)] + [InlineData(false, SiloStatus.Joining)] + [InlineData(true, SiloStatus.Joining)] + [InlineData(false, SiloStatus.Active)] + [InlineData(true, SiloStatus.Active)] + [InlineData(false, SiloStatus.ShuttingDown)] + [InlineData(true, SiloStatus.ShuttingDown)] + [InlineData(false, SiloStatus.Stopping)] + [InlineData(true, SiloStatus.Stopping)] + public async Task MembershipSameVersionPruningRemovesOnlyDeadRows(bool delta, SiloStatus removedStatus) + { + var local = CreateSilo(45401); + var first = CreateSilo(45402); + var second = CreateSilo(45403); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var sourceManager = new FakeMembershipManager(CreateMembershipSnapshot(7, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(30)), + CreateMembershipEntry(first, SiloStatus.Dead, DateTime.UnixEpoch))); + var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(7, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(40)), + CreateMembershipEntry(second, removedStatus, DateTime.UnixEpoch))); + var source = CreateMembershipNamespace(sourceManager, serializer); + var receiver = CreateMembershipNamespace(receiverManager, serializer); + + var sameVersion = delta + ? CreateMembershipDelta(serializer, 7, 7, [], [second]) + : GetMembershipRepair(source, 7); + Assert.Equal(removedStatus == SiloStatus.Dead ? DisseminationApplyResult.Applied : DisseminationApplyResult.Rejected, + await receiver.ApplyValueAsync(sameVersion, TestContext.Current.CancellationToken)); + Assert.Equal(removedStatus != SiloStatus.Dead, receiverManager.CurrentSnapshot.Entries.ContainsKey(second)); + Assert.DoesNotContain(first, receiverManager.CurrentSnapshot.Entries.Keys); + + sourceManager.CurrentSnapshot = CreateMembershipSnapshot(8, + sourceManager.CurrentSnapshot.Entries[local]); + var removal = delta + ? CreateMembershipDelta(serializer, 7, 8, [], [second]) + : GetMembershipRepair(source, 7); + Assert.Equal(DisseminationApplyResult.Applied, + await receiver.ApplyValueAsync(removal, TestContext.Current.CancellationToken)); + Assert.Equal(local, Assert.Single(receiverManager.CurrentSnapshot.Entries).Key); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(40), receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + Assert.Equal(8, receiverManager.CurrentSnapshot.Version.Value); + } + + [Fact] + public async Task MembershipDeltaCanPruneAllDeadRowsWithoutResurrection() + { + var first = CreateSilo(45921); + var second = CreateSilo(45922); + var initial = CreateMembershipSnapshot(7, + CreateMembershipEntry(first, SiloStatus.Dead, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Dead, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var sourceManager = new FakeMembershipManager(initial); + var source = CreateMembershipNamespace(sourceManager, serializer); + var receiverManager = new FakeMembershipManager(initial); + var receiver = CreateMembershipNamespace(receiverManager, serializer); + var request = new DisseminationRepairRequest(DisseminationKey.Default, 7, 1024 * 1024, 1024 * 1024); + var baseline = source.CreateBroadcast(request, baseline: null).BroadcastState; + var staleFull = source.CreateRepair(request).Value; + sourceManager.CurrentSnapshot = CreateMembershipSnapshot(7); + var delta = source.CreateBroadcast(request, baseline).Value; + + Assert.Equal(new[] { first, second }, ReadMembershipDelta(serializer, delta).RemovedSilos.Order()); + Assert.Equal(DisseminationApplyResult.Applied, + await receiver.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Empty(receiverManager.CurrentSnapshot.Entries); + Assert.Equal(7, receiverManager.CurrentSnapshot.Version.Value); + Assert.Equal(DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(delta, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Duplicate, + await receiver.ApplyValueAsync(staleFull, TestContext.Current.CancellationToken)); + Assert.Empty(receiverManager.CurrentSnapshot.Entries); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task MembershipHeartbeatAcceptanceIsIndependentOfStartTime(bool delta, bool rejectOwnerUpdate) + { + var local = CreateSilo(45901); + var start = DateTime.UnixEpoch.AddYears(50); + var heartbeat = DateTime.UnixEpoch.AddSeconds(10); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, start, DateTime.MinValue)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + if (rejectOwnerUpdate) + { + manager.ProcessGossipSnapshotHandler = (_, _) => Task.CompletedTask; + } + + var ns = CreateMembershipNamespace(manager, serializer); + var fingerprint = Assert.Single(ns.Digests).Fingerprint; + var entry = initial.Entries[local].WithIAmAliveTime(heartbeat); + var value = delta + ? CreateMembershipDelta(serializer, 1, 1, [entry], []) + : new DisseminationValue(DisseminationKey.Default, 0, 1, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = CreateMembershipSnapshot(1, entry) })); + + var result = await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(rejectOwnerUpdate ? DisseminationApplyResult.Rejected : DisseminationApplyResult.Applied, result); + Assert.Equal(start, manager.CurrentSnapshot.Entries[local].StartTime); + Assert.Equal(rejectOwnerUpdate ? DateTime.MinValue : heartbeat, manager.CurrentSnapshot.Entries[local].IAmAliveTime); + if (rejectOwnerUpdate) + { + Assert.Equal(fingerprint, Assert.Single(ns.Digests).Fingerprint); + } + else + { + Assert.NotEqual(fingerprint, Assert.Single(ns.Digests).Fingerprint); + Assert.True(manager.CurrentSnapshot.IsSuccessorTo(initial)); + } + } + + [Fact] + public async Task MembershipDeltaDoesNotAcknowledgeUnrelatedOwnerRefresh() + { + var local = CreateSilo(45501); + var initial = CreateMembershipSnapshot(1, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + manager.ProcessGossipSnapshotHandler = (_, _) => + { + manager.CurrentSnapshot = new(new MembershipVersion(2), initial.Entries); + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var value = CreateMembershipDelta(serializer, 1, 2, + [initial.Entries[local].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], []); + + Assert.Equal(DisseminationApplyResult.Rejected, + await ns.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal(DateTime.UnixEpoch, manager.CurrentSnapshot.Entries[local].IAmAliveTime); + + var peer = CreateSilo(45502); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + SupportsCompactAcknowledgments = true, + Values = new() { [ns.Name] = [new() { Value = value, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }, TestContext.Current.CancellationToken); + Assert.False(response.AllVersionsAcknowledged); + Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task MembershipAntiEntropyRejectsDeltaResponses() + { + var local = CreateSilo(45701); + var peer = CreateSilo(45702); + var initial = CreateMembershipSnapshot(1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(initial); + var applied = 0; + manager.ProcessGossipSnapshotHandler = (_, _) => + { + applied++; + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var delta = CreateMembershipDelta(serializer, 1, 2, + [initial.Entries[peer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10))], []); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() { [ns.Name] = [new() { Value = delta, TimeToLive = TimeSpan.FromSeconds(30) }] }, + }); + var protocol = CreateProtocol(transport, [ns]); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Single(transport.AntiEntropyRequests); + Assert.Equal(0, applied); + Assert.Same(initial, manager.CurrentSnapshot); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + private static MembershipTableSnapshotDelta ReadMembershipDelta(Serializer serializer, DisseminationValue value) + { + var update = Assert.IsType(serializer.Deserialize(value.Payload)); + Assert.Null(update.Snapshot); + var delta = Assert.IsType(update.Delta); + Assert.Equal(value.FromVersion, delta.BaseVersion.Value); + Assert.Equal(value.ToVersion, delta.Version.Value); + return delta; + } + + private static DisseminationValue CreateMembershipDelta( + Serializer serializer, + long from, + long to, + ImmutableArray entries, + ImmutableArray removed) => + new(DisseminationKey.Default, from, to, serializer.SerializeToArray(new MembershipTableSnapshotUpdate + { + Delta = new(new MembershipVersion(from), new MembershipVersion(to), entries, removed), + })); + + private sealed class MembershipDeltaLink : IAsyncDisposable + { + private readonly ServiceProvider _services; + private readonly SiloAddress _peer; + private readonly DisseminationProtocol _targetProtocol; + + public MembershipDeltaLink( + SiloAddress local, + SiloAddress peer, + MembershipTableSnapshot source, + MembershipTableSnapshot target) + { + _peer = peer; + _services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + Serializer = _services.GetRequiredService(); + SourceManager = new(source); + TargetManager = new(target); + Source = CreateMembershipNamespace(SourceManager, Serializer); + Target = CreateMembershipNamespace(TargetManager, Serializer); + var transport = new FakeTransport(local, peer); + SourceProtocol = CreateProtocol(transport, [Source], timeProvider: Clock); + _targetProtocol = CreateProtocol(new FakeTransport(peer, local), [Target], timeProvider: Clock); + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + Batches.Add(batch); + var response = await _targetProtocol.ReceiveBroadcast(batch, token); + Responses.Add(response); + if (BeforeResponse is { } before) + { + await before(batch, Batches.Count, token); + } + + return response; + }; + Queue = CreateBroadcastQueue(transport, [Source], timeProvider: Clock); + } + + public Serializer Serializer { get; } + public FakeTimeProvider Clock { get; } = new(); + public FakeMembershipManager SourceManager { get; } + public FakeMembershipManager TargetManager { get; } + public MembershipDisseminationNamespace Source { get; } + public MembershipDisseminationNamespace Target { get; } + public DisseminationProtocol SourceProtocol { get; } + public DisseminationBroadcastQueue Queue { get; } + public List Batches { get; } = []; + public List Responses { get; } = []; + public Func? BeforeResponse { get; set; } + + public async Task Send() + { + var token = TestContext.Current.CancellationToken; + var flush = BeforeBroadcastPumpsRun(() => + { + Assert.True(Queue.Notify(_peer, Source, DisseminationKey.Default)); + return Queue.FlushPendingBroadcast(token); + }); + await flush.WaitAsync(TimeSpan.FromSeconds(5), token); + } + + public async ValueTask DisposeAsync() + { + var token = new CancellationToken(canceled: true); + var stops = new[] + { + Queue.StopAsync(token), + SourceProtocol.StopAsync(token), + _targetProtocol.StopAsync(token), + }; + try + { + await Task.WhenAll(stops).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + } + finally + { + _services.Dispose(); + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs new file mode 100644 index 00000000000..8a526c98c13 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReset.cs @@ -0,0 +1,148 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OlderFullMembershipValueDoesNotReachOwner(bool antiEntropy) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41011); + var peer = CreateSilo(41012); + var current = CreateMembershipSnapshot(100, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var notification = CreateMembershipSnapshot(2, current.Entries.Values.ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(current); + var validations = 0; + manager.ProcessGossipSnapshotHandler = (snapshot, token) => + { + token.ThrowIfCancellationRequested(); + validations++; + return Task.CompletedTask; + }; + var ns = CreateMembershipNamespace(manager, serializer); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, [ns]); + var values = CreateValueGroups(ns.Name, CreateDisseminationValue(peer, new( + DisseminationKey.Default, 0, 2, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = notification })))); + try + { + if (antiEntropy) + { + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = peer, Values = values }); + await protocol.RunAntiEntropyRound(cancellationToken); + } + else + { + var response = await protocol.ReceiveBroadcast(new() { Sender = peer, Values = values }, cancellationToken); + Assert.Equal(100, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + + Assert.Equal(0, validations); + Assert.Same(current, manager.CurrentSnapshot); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AntiEntropyDoesNotSendOlderMembershipToAnAheadPeer() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41013); + var peer = CreateSilo(41014); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = CreateMembershipNamespace(new FakeMembershipManager(CreateMembershipSnapshot( + 2, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))), serializer); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns]); + try + { + var response = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = [new(DisseminationKey.Default, 100)] }, + }, cancellationToken); + + Assert.Empty(GetAntiEntropyResponseValues(response)); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + [Fact] + public void MembershipRepairMaterializesTheCurrentOwnerSnapshot() + { + var local = CreateSilo(41001); + var snapshot = CreateMembershipSnapshot(2, CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = CreateMembershipNamespace(new FakeMembershipManager(snapshot), serializer); + + var repair = ns.CreateRepair(new( + DisseminationKey.Default, 100, 1024 * 1024, 1024 * 1024)); + + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + Assert.Equal(2, repair.Version); + var value = repair.Value; + Assert.Equal(0, value.FromVersion); + Assert.Equal(2, value.ToVersion); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + var repaired = Assert.IsType(update.Snapshot); + Assert.Equal(snapshot.Version, repaired.Version); + Assert.Equal(local, Assert.Single(repaired.Entries).Key); + } + + [Fact] + public async Task MembershipCurrentViewSupersedesDelayedPublication() + { + var members = CreateSilos(20); + var old = CreateMembershipSnapshot(1, members.Select( + member => CreateMembershipEntry(member, SiloStatus.Active, DateTime.UnixEpoch)).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(old); + var ns = CreateMembershipNamespace(manager, serializer); + Assert.Equal(1, Assert.Single(ns.Digests).Version); + var current = CreateMembershipSnapshot(100, old.Entries.Values.Select(entry => + { + var result = entry.Copy(); + result.HostName = "current-view"; + return result; + }).ToArray()); + manager.CurrentSnapshot = current; + Assert.Equal(100, Assert.Single(ns.Digests).Version); + + var publications = new FakeDisseminationService(); + Assert.True(await ns.PublishAsync(publications, old, TestContext.Current.CancellationToken)); + Assert.Equal(100, Assert.Single(publications.Values).ToVersion); + var repair = ns.CreateRepair(new( + DisseminationKey.Default, 1, 1024 * 1024, 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var value = repair.Value; + Assert.Equal(0, value.FromVersion); + Assert.Equal(100, value.ToVersion); + var payload = Assert.IsType( + serializer.Deserialize(value.Payload)); + var repaired = Assert.IsType(payload.Snapshot); + Assert.Equal(members.Length, repaired.Entries.Count); + Assert.All(repaired.Entries.Values, entry => Assert.Equal("current-view", entry.HostName)); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs new file mode 100644 index 00000000000..c069eaf3f90 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MembershipReview.cs @@ -0,0 +1,334 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Core.Diagnostics; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(1, true)] + [InlineData(2, true)] + public async Task MembershipNamespaceFullSnapshotPreservesMixedLivenessInRealManager( + long incomingVersion, bool complementaryPeers) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(39500); + var newer = CreateSilo(39501); + var stale = CreateSilo(39502); + var previous = CreateMembershipSnapshot( + 1, + CreateMembershipEntry(newer, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10)), + CreateMembershipEntry(stale, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20))); + var incoming = CreateMembershipSnapshot( + incomingVersion, + previous.Entries[newer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30)), + previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(5))); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(local, out _); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, + 0, + incomingVersion, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incoming })); + + Assert.True(incoming.IsSuccessorTo(previous)); + if (complementaryPeers) + { + var complementary = CreateMembershipSnapshot(incomingVersion, + previous.Entries[newer].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(15)), + previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(40))); + var otherValue = new DisseminationValue(DisseminationKey.Default, 0, incomingVersion, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = complementary })); + var transport = new FakeTransport(local, newer, stale); + transport.ExchangeAntiEntropyHandler = (peer, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(peer, peer.Equals(newer) ? value : otherValue)), + }); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2); + try + { + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + else + { + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + } + + var expected = CreateMembershipSnapshot( + incomingVersion, + incoming.Entries[newer], + complementaryPeers + ? previous.Entries[stale].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(40)) + : previous.Entries[stale]); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(5), incoming.Entries[stale].IAmAliveTime); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(10), previous.Entries[newer].IAmAliveTime); + + // Repairs must contain the manager's merged state, not the unmerged incoming snapshot. + var repair = GetMembershipRepair(ns, incomingVersion); + Assert.Equal(0, repair.FromVersion); + var repaired = Assert.IsType( + serializer.Deserialize(repair.Payload)); + AssertMembershipState(expected, Assert.IsType(repaired.Snapshot)); + Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task MembershipNamespaceFullSnapshotPrunesDeadRowsAndRepairsDownstream( + bool advanceVersion, bool receiverAlreadyAtTargetVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var retired = CreateSilo(39511); + var successor = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var staleGenerationSnapshot = CreateMembershipSnapshot(1, + CreateMembershipEntry(retired, SiloStatus.Active, DateTime.UnixEpoch)); + var beforeCleanup = CreateMembershipSnapshot(2, + staleGenerationSnapshot.Entries[retired].WithStatus(SiloStatus.Dead), + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(20))); + using var sourceManager = CreateMembershipReviewManager(CreateSilo(39510), out _); + using var receiverManager = CreateMembershipReviewManager(CreateSilo(39512), out _); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + var sourceBaseline = Assert.Single(source.Digests); + var receiverBaseline = Assert.Single(receiver.Digests); + Assert.Equal(sourceBaseline, receiverBaseline); + + var afterCleanup = MembershipTableSnapshot.Update(beforeCleanup, CreateMembershipSnapshot( + beforeCleanup.Version.Value + (advanceVersion ? 1 : 0), + beforeCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(10)))); + Assert.Equal(beforeCleanup.Version.Value + (advanceVersion ? 1 : 0), afterCleanup.Version.Value); + Assert.Equal(successor, Assert.Single(afterCleanup.Entries).Key); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(20), afterCleanup.Entries[successor].IAmAliveTime); + Assert.True(afterCleanup.IsSuccessorTo(beforeCleanup)); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); + var target = CreateMembershipSnapshot(afterCleanup.Version.Value, + afterCleanup.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(30))); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); + if (receiverAlreadyAtTargetVersion) + { + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(afterCleanup, cancellationToken); + Assert.Equal(target.Entries.Keys, receiverManager.MembershipTableSnapshot.Entries.Keys); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(20), + receiverManager.MembershipTableSnapshot.Entries[successor].IAmAliveTime); + } + + var value = GetMembershipRepair(source, sourceBaseline.Version); + Assert.Equal((0L, target.Version.Value), (value.FromVersion, value.ToVersion)); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + var snapshot = Assert.IsType(update.Snapshot); + AssertMembershipState(target, snapshot); + Assert.Equal(successor, Assert.Single(snapshot.Entries).Key); + + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(value, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + Assert.False(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.Equal(SiloStatus.Active, receiverManager.MembershipTableSnapshot.GetSiloStatus(successor)); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(value, cancellationToken)); + + var obsolete = new DisseminationValue( + DisseminationKey.Default, + 0, + staleGenerationSnapshot.Version.Value, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = staleGenerationSnapshot })); + Assert.Equal(DisseminationApplyResult.Obsolete, await receiver.ApplyValueAsync(obsolete, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + + using var downstreamManager = CreateMembershipReviewManager(CreateSilo(39513), out _); + await ((IMembershipManager)downstreamManager).ProcessGossipSnapshot(beforeCleanup, cancellationToken); + var downstream = CreateMembershipNamespace((IMembershipManager)downstreamManager, serializer); + var forwarded = GetMembershipRepair(receiver, sourceBaseline.Version); + Assert.Equal(0, forwarded.FromVersion); + Assert.Equal(target.Version.Value, forwarded.ToVersion); + Assert.Equal(DisseminationApplyResult.Applied, await downstream.ApplyValueAsync(forwarded, cancellationToken)); + AssertMembershipState(target, downstreamManager.MembershipTableSnapshot); + } + + [Fact] + public async Task MembershipNamespaceFullSnapshotPreservesRealManagerLocalDeathEntry() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(39520); + var peer = CreateSilo(39521); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var manager = CreateMembershipReviewManager(local, out var fatalErrorHandler); + var localEntry = manager.MembershipTableSnapshot.Entries[local].WithStatus(SiloStatus.Active); + var previous = CreateMembershipSnapshot( + 1, + localEntry, + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + await ((IMembershipManager)manager).ProcessGossipSnapshot(previous, cancellationToken); + var ns = CreateMembershipNamespace((IMembershipManager)manager, serializer); + var update = new MembershipTableSnapshotUpdate + { + Snapshot = CreateMembershipSnapshot(2, previous.Entries[peer]), + }; + var value = new DisseminationValue(DisseminationKey.Default, 0, 2, serializer.SerializeToArray(update)); + + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + + var expected = CreateMembershipSnapshot(2, localEntry.WithStatus(SiloStatus.Dead), previous.Entries[peer]); + AssertMembershipState(expected, manager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, manager.CurrentStatus); + fatalErrorHandler.Received(1).OnFatalException(manager, Arg.Any(), null); + var repair = GetMembershipRepair(ns, 2); + Assert.Equal(0, repair.FromVersion); + var repaired = Assert.IsType( + serializer.Deserialize(repair.Payload)); + AssertMembershipState(expected, Assert.IsType(repaired.Snapshot)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipNamespaceDeadRowCleanupConvergesWithoutHeartbeatWithFullRepair(bool advanceVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var retired = CreateSilo(39541); + var successor = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + var receiverBaseline = CreateMembershipSnapshot( + 1, + CreateMembershipEntry(retired, SiloStatus.Dead, DateTime.UnixEpoch), + CreateMembershipEntry(successor, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(10))); + var target = CreateMembershipSnapshot( + receiverBaseline.Version.Value + (advanceVersion ? 1 : 0), + receiverBaseline.Entries[successor]); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + using var sourceManager = CreateMembershipReviewManager(CreateSilo(39540), out _); + using var receiverManager = CreateMembershipReviewManager(CreateSilo(39542), out _); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(receiverBaseline, cancellationToken); + await ((IMembershipManager)receiverManager).ProcessGossipSnapshot(receiverBaseline, cancellationToken); + var source = CreateMembershipNamespace((IMembershipManager)sourceManager, serializer); + var receiver = CreateMembershipNamespace((IMembershipManager)receiverManager, serializer); + _ = Assert.Single(source.Digests); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(target, cancellationToken); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.True(receiverManager.MembershipTableSnapshot.Entries.ContainsKey(retired)); + Assert.Equal(receiverBaseline.Version, receiverManager.MembershipTableSnapshot.Version); + Assert.Equal(target.Version, sourceManager.MembershipTableSnapshot.Version); + Assert.Equal(receiverBaseline.Entries[successor].IAmAliveTime, target.Entries[successor].IAmAliveTime); + + var transport = new FakeTransport(CreateSilo(39540), CreateSilo(39542)); + var protocol = CreateProtocol(transport, [source], timeProvider: new FakeTimeProvider()); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + var full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, target.Version.Value), (full.FromVersion, full.ToVersion)); + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); + AssertMembershipState(target, receiverManager.MembershipTableSnapshot); + Assert.Equal(successor, Assert.Single(receiverManager.MembershipTableSnapshot.Entries).Key); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + Assert.Equal(DisseminationApplyResult.Duplicate, await receiver.ApplyValueAsync(full, cancellationToken)); + var converged = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(converged)); + + var heartbeatTarget = CreateMembershipSnapshot( + target.Version.Value, + target.Entries[successor].WithIAmAliveTime(DateTime.UnixEpoch.AddSeconds(20))); + await ((IMembershipManager)sourceManager).ProcessGossipSnapshot(heartbeatTarget, cancellationToken); + response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + full = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, target.Version.Value), (full.FromVersion, full.ToVersion)); + Assert.Equal(DisseminationApplyResult.Applied, await receiver.ApplyValueAsync(full, cancellationToken)); + AssertMembershipState(heartbeatTarget, receiverManager.MembershipTableSnapshot); + Assert.Equal(SiloStatus.Dead, receiverManager.MembershipTableSnapshot.GetSiloStatus(retired)); + Assert.Equal(Assert.Single(source.Digests), Assert.Single(receiver.Digests)); + + response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = transport.Peers[0], + Digests = new() { [source.Name] = [Assert.Single(receiver.Digests)] }, + }, cancellationToken); + Assert.Empty(GetAntiEntropyResponseValues(response)); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + private static MembershipTableManager CreateMembershipReviewManager( + SiloAddress local, + out IFatalErrorHandler fatalErrorHandler, + IMembershipTable membershipTable = null!) + { + var timer = Substitute.For(); + var timers = Substitute.For(); + timers.Create(default, default!, default!).ReturnsForAnyArgs(timer); + fatalErrorHandler = Substitute.For(); + return new MembershipTableManager( + new FakeLocalSiloDetails(local), + Options.Create(new ClusterMembershipOptions()), + membershipTable ?? Substitute.For(), + fatalErrorHandler, + Substitute.For(), + NullLogger.Instance, + timers, + new SiloLifecycleSubject(NullLogger.Instance), + new FakeTimeProvider()); + } + + private static MembershipDisseminationNamespace CreateMembershipNamespace(IMembershipManager manager, Serializer serializer) + { + var options = new ClusterMembershipOptions(); + options.Dissemination.Enabled = true; + return new(manager, new TestOptionsMonitor(options), serializer); + } + + private static DisseminationValue GetMembershipRepair(MembershipDisseminationNamespace ns, long? fromVersion) + { + var repair = ns.CreateRepair(new(DisseminationKey.Default, fromVersion, 1024 * 1024, 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + return repair.Value; + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs new file mode 100644 index 00000000000..52922f8fbc1 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.MetricBoundaries.cs @@ -0,0 +1,178 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData("broadcast-receive")] + [InlineData("broadcast-send")] + [InlineData("repair-receive")] + [InlineData("repair-send")] + [InlineData("publication")] + [InlineData("repair-failure")] + [InlineData("broadcast-failure")] + public async Task MetricFailuresPreserveProtocolResults(string operation) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(41021); + var peer = CreateSilo(41022); + var healthy = CreateSilo(41023); + var transport = operation == "repair-failure" ? new FakeTransport(local, peer, healthy) : new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("metric-boundary-" + operation)); + var logger = new Phase6ProtocolLogger(); + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2, new FakeTimeProvider(), logger); + var metric = operation switch + { + "broadcast-receive" => "orleans-dissemination-broadcast-received", + "broadcast-send" => "orleans-dissemination-broadcast-sent", + "publication" => DisseminationInstruments.PublicationsName, + "repair-failure" => DisseminationInstruments.AntiEntropyFailuresName, + "broadcast-failure" => DisseminationInstruments.BroadcastSendFailuresName, + _ => "orleans-dissemination-anti-entropy-exchanges", + }; + var failure = new InvalidOperationException("Metric observer failure."); + var failures = 0; + var broadcastFailureThread = 0; + var listenerArmed = 1; + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, owner) => + { + if (Volatile.Read(ref listenerArmed) != 0 + && instrument.Meter.Name == DisseminationInstruments.MeterName && instrument.Name == metric) + { + owner.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + if (Volatile.Read(ref listenerArmed) == 0) + { + return; + } + + // Failure counters have no namespace tag. This completed fake transport failure is observed + // synchronously on its throwing thread, so unrelated background failures cannot claim it. + if (operation == "broadcast-failure") + { + var thread = Environment.CurrentManagedThreadId; + if (Interlocked.CompareExchange(ref broadcastFailureThread, 0, thread) != thread) + { + return; + } + } + + if (operation is "broadcast-receive" or "broadcast-send" or "publication") + { + var matches = false; + foreach (var tag in tags) + { + matches |= tag.Key == "namespace" && Equals(tag.Value, ns.Name); + } + if (!matches) + { + return; + } + } + + Interlocked.Increment(ref failures); + throw failure; + }); + listener.Start(); + try + { + switch (operation) + { + case "broadcast-receive": + { + var response = await protocol.ReceiveBroadcast(new() + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.CreateItem(peer, "value", 2)), + }, cancellationToken); + Assert.Equal(2, ns.GetVersion("value")); + Assert.Equal(2, Assert.Single(response.Acknowledgments[ns.Name]).Version); + break; + } + case "broadcast-send": + case "publication": + ns.SetValue("value", 1); + Assert.True(await protocol.Publish(ns, "value", 1, cancellationToken)); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Single(transport.BroadcastBatches); + break; + case "repair-receive": + { + ns.SetValue("value", 2); + var response = await protocol.ReceiveAntiEntropy(new() + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = new() { [ns.Name] = [new("value", 1)] }, + }, cancellationToken); + Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(response)).Value.ToVersion); + break; + } + case "repair-send": + case "repair-failure": + ns.SetValue("value", 1); + transport.ExchangeAntiEntropyHandler = (target, _, _) => + operation == "repair-failure" && target.Equals(peer) + ? ValueTask.FromException(new InvalidOperationException("Transport failure.")) + : ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.Name, ns.CreateItem(target, "value", 2)), + }); + await protocol.RunAntiEntropyRound(cancellationToken); + Assert.Equal(2, ns.GetVersion("value")); + break; + case "broadcast-failure": + ns.SetValue("value", 1); + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Error); + Assert.Equal(0, Volatile.Read(ref failures)); + transport.SendBroadcastResponseHandler = (target, batch, _) => + { + transport.BroadcastBatches.Add((target, batch)); + if (transport.BroadcastBatches.Count == 1) + { + Volatile.Write(ref broadcastFailureThread, Environment.CurrentManagedThreadId); + return Task.FromException(new InvalidOperationException("Transport failure.")); + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + Assert.True(await protocol.Publish(ns, "value", 1, cancellationToken)); + await protocol.FlushPendingBroadcast(cancellationToken); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.All(transport.BroadcastBatches, batch => + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + await protocol.FlushPendingBroadcast(cancellationToken); + Assert.Equal(2, transport.BroadcastBatches.Count); + break; + default: + throw new InvalidOperationException(operation); + } + + Assert.Equal(1, Volatile.Read(ref failures)); + Assert.Equal(0, Volatile.Read(ref broadcastFailureThread)); + if (operation is not ("broadcast-send" or "broadcast-failure")) + { + Assert.Contains(logger.Entries, entry => ReferenceEquals(entry.Exception, failure)); + } + } + finally + { + Volatile.Write(ref listenerArmed, 0); + listener.Dispose(); + ns.Options.Enabled = false; + await protocol.StopAsync(cancellationToken); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs new file mode 100644 index 00000000000..112169db208 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.OwnerCancellationReview.cs @@ -0,0 +1,73 @@ +#nullable enable + +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using NSubstitute; +using Orleans; +using Orleans.Runtime; +using Orleans.Runtime.MembershipService; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task MembershipOwnerRechecksCancellationAfterSharedRefreshCompleted() + { + var read = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var table = Substitute.For(); + table.ReadAllAsync(Arg.Any()).Returns(read.Task); + using var manager = CreateMembershipReviewManager(CreateSilo(39601), out _, table); + var refresh = manager.Refresh(cancellationToken: TestContext.Current.CancellationToken); + var context = new MembershipReviewContinuationContext(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var incoming = CreateMembershipSnapshot( + 2, CreateMembershipEntry(CreateSilo(39602), SiloStatus.Active, DateTime.UnixEpoch)); + Task gossip; + var previousContext = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(context); + try + { + gossip = ((IMembershipManager)manager).ProcessGossipSnapshot(incoming, cancellation.Token); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + Assert.False(gossip.IsCompleted); + read.SetResult(new MembershipTableData(new TableVersion(1, "1"))); + var continuation = await context.TakeContinuation(TestContext.Current.CancellationToken); + await refresh.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var committed = manager.MembershipTableSnapshot; + Assert.Equal(new MembershipVersion(1), committed.Version); + + cancellation.Cancel(); + continuation.Callback(continuation.State); + + var exception = await Assert.ThrowsAnyAsync( + () => gossip.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Same(committed, manager.MembershipTableSnapshot); + Assert.DoesNotContain(CreateSilo(39602), manager.MembershipTableSnapshot.Entries.Keys); + } + + private sealed class MembershipReviewContinuationContext : SynchronizationContext + { + private readonly Channel<(SendOrPostCallback Callback, object? State)> _continuations = + Channel.CreateUnbounded<(SendOrPostCallback, object?)>(); + + public override void Post(SendOrPostCallback callback, object? state) => + _continuations.Writer.TryWrite((callback, state)); + + public async Task<(SendOrPostCallback Callback, object? State)> TakeContinuation(CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5)); + return await _continuations.Reader.ReadAsync(deadline.Token); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs new file mode 100644 index 00000000000..e7d7ebbaff3 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PerformanceReview.cs @@ -0,0 +1,161 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task LoadRepairReusesOnlySuccessfullyAppliedPayloads() + { + var cancellationToken = TestContext.Current.CancellationToken; + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(new()), serializer); + var first = CreatePhase5Statistics(1); + var value = new DisseminationValue(harness.ActiveOne, 0, first.DateTime.Ticks, serializer.SerializeToArray(first)); + + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.True(value.Payload.Equals(Repair().Payload)); + + var duplicate = new DisseminationValue(value.Key, 0, value.ToVersion, value.Payload.ToArray()); + Assert.Equal(DisseminationApplyResult.Duplicate, await ns.ApplyValueAsync(duplicate, cancellationToken)); + Assert.True(value.Payload.Equals(Repair().Payload)); + + var second = CreatePhase5Statistics(2); + var replacement = new DisseminationValue(value.Key, 0, second.DateTime.Ticks, serializer.SerializeToArray(second)); + Assert.Equal(DisseminationApplyResult.Applied, await ns.ApplyValueAsync(replacement, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + Assert.Equal(DisseminationApplyResult.Obsolete, await ns.ApplyValueAsync(value, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + + var malformed = new DisseminationValue(value.Key, 0, replacement.ToVersion + 1, replacement.Payload); + Assert.Equal(DisseminationApplyResult.Rejected, await ns.ApplyValueAsync(malformed, cancellationToken)); + Assert.True(replacement.Payload.Equals(Repair().Payload)); + + DisseminationValue Repair() + { + var repair = ns.CreateRepair(new(value.Key, null, 1024, 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + return repair.Value; + } + } + + [Fact] + public void MembershipKeyInventoryRequiresNoSnapshotOrFingerprint() + { + var manager = Substitute.For(); + manager.CurrentSnapshot.Returns(_ => throw new InvalidOperationException("Key enumeration must not read membership state.")); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var ns = CreateMembershipNamespace(manager, services.GetRequiredService()); + + Assert.Equal(DisseminationKey.Default, Assert.Single(ns.Keys)); + } + + [Fact] + public async Task QueuePruningUsesKeyInventoryWithoutBuildingDigests() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40601); + var peer = CreateSilo(40602); + var transport = new FakeTransport(local, peer); + var ns = new InventoryOnlyNamespace(local); + ns.Inner.SetValue("value", 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var membership = new DisseminationMembership( + transport.MembershipManager, + new FakeLocalSiloDetails(local), + Microsoft.Extensions.Options.Options.Create(new DisseminationOptions())); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + await queue.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await queue.Prune(membership.CurrentSnapshots, cancellationToken); + + Assert.Equal(1, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + } + finally + { + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + [Fact] + public async Task ReceiveBudgetRetainsCursorForZeroByteSuffixAndIgnoresEmptyNamespaces() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40611); + var peer = CreateSilo(40612); + var ns = new ProtocolReviewNamespace(local); + var empty = new FakeNamespace(local, "empty"); + ns.ApplyHandler = (value, _) => + { + ns.Inner.SetValue(value.Key, value.ToVersion); + return new(DisseminationApplyResult.Applied); + }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns, empty], options => + { + options.MaxBatchBytes = 8; + options.MaxBatchItems = 2; + }); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [ns.Name] = + [ + CreateDisseminationValue(peer, new("first", 0, 1, new byte[8])), + CreateDisseminationValue(peer, new("second", 0, 1, ReadOnlyMemory.Empty)), + ], + [empty.Name] = [], + }, + }; + try + { + var first = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal("first", Assert.IsType(Assert.Single(first.Acknowledgments[ns.Name]).Key.Value)); + Assert.False(first.Acknowledgments.ContainsKey(empty.Name)); + Assert.Equal(0, ns.GetVersion("second")); + + var second = await protocol.ReceiveBroadcast(batch, cancellationToken); + Assert.Equal("second", Assert.IsType(Assert.Single(second.Acknowledgments[ns.Name]).Key.Value)); + Assert.Equal(new DisseminationKey[] { "first", "second" }, ns.Attempts.Select(static value => value.Key)); + Assert.Equal(1, ns.GetVersion("second")); + } + finally + { + await protocol.StopAsync(cancellationToken); + } + } + + private sealed class InventoryOnlyNamespace(SiloAddress local) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local); + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Keys => Inner.Digests.Select(static digest => digest.Key); + public IEnumerable Digests => throw new InvalidOperationException("Pruning must use key inventory."); + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => Inner.CreateRepair(request); + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + Inner.ApplyValueAsync(value, cancellationToken); + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs new file mode 100644 index 00000000000..060578ff730 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.ProtocolReview.cs @@ -0,0 +1,987 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task ProtocolReviewPayloadObserverFailureDoesNotBlockLaterRepair() + { + var local = CreateSilo(39501); + var peer = CreateSilo(39502); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue("oversize", 1); + ns.Inner.SetValue("healthy", 2); + ns.Options.MaxPayloadBytes = 8; + ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") + ? DisseminationRepairResult.Produced(new(request.Key, 0, 1, new byte[9])) + : ns.Inner.CreateRepair(request); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); + var observer = new ProtocolReviewPayloadObserver(local, ns.Name); + using var subscription = DisseminationEvents.Listener.Subscribe( + observer, static name => name == "Dissemination.PayloadDrop"); + try + { + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("oversize", 0), ("healthy", 0)), + }; + var response = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal(new DisseminationKey("healthy"), value.Key); + Assert.Equal(2, value.ToVersion); + Assert.False(response.Truncated); + Assert.Equal(1, observer.Count); + var diagnostic = Assert.Single(logger.Entries); + Assert.Same(observer.Failure, diagnostic.Exception); + Assert.Equal(ns.Name, diagnostic.State["Namespace"]); + Assert.Equal(new DisseminationKey("oversize"), diagnostic.State["Key"]); + + ns.RepairHandler = null; + response = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + Assert.Equal(new[] { "oversize", "healthy" }, + GetAntiEntropyResponseValues(response).Select(static item => item.Value.Key.ToString())); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewValueMetricFailuresDoNotBlockRepairOrApplication() + { + var local = CreateSilo(39503); + var peer = CreateSilo(39504); + var ns = new ProtocolReviewNamespace(local, "protocol-review-metric-isolation"); + ns.Inner.SetValue("oversize", 1); + ns.Inner.SetValue("healthy", 2); + ns.Options.MaxPayloadBytes = 8; + ns.RepairHandler = request => request.Key == new DisseminationKey("oversize") + ? DisseminationRepairResult.Produced(new(request.Key, 0, 1, new byte[9])) + : ns.Inner.CreateRepair(request); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(new FakeTransport(local, peer), ns, logger); + var failures = new List(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name is "orleans-dissemination-payload-dropped" or "orleans-dissemination-values-applied") + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, _, tags, _) => + { + foreach (var tag in tags) + { + if (tag.Key == "namespace" && Equals(tag.Value, ns.Name)) + { + failures.Add(instrument.Name); + throw new InvalidOperationException("Metric observer failure."); + } + } + }); + listener.Start(); + try + { + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("oversize", 0), ("healthy", 0)), + }, TestContext.Current.CancellationToken); + Assert.Equal(new DisseminationKey("healthy"), Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key); + var acknowledgment = await protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = CreateValueGroups(ns.Name, ns.Inner.CreateItem(peer, "new", 3)), + }, TestContext.Current.CancellationToken); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[ns.Name]).Version); + Assert.Equal(3, ns.GetVersion("new")); + Assert.Equal( + new[] { "orleans-dissemination-payload-dropped", "orleans-dissemination-values-applied" }, + failures); + Assert.Equal(2, logger.Entries.Count); + Assert.All(logger.Entries, static entry => Assert.Equal("Metric observer failure.", entry.Exception.Message)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewMalformedLoadPayloadRejectsAndContinues(bool antiEntropy) + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.Enabled = true; + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(options), serializer); + var statistics = CreatePhase5Statistics(1); + var malformed = new DisseminationValue(harness.ActiveOne, 0, statistics.DateTime.Ticks, new byte[] { 0xff }); + var valid = ns.CreateValue(harness.ActiveTwo, statistics); + var transport = new FakeTransport(harness.Local, harness.ActiveOne, harness.ActiveTwo); + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger, settings => settings.Overlay.AntiEntropyPeerCount = 1); + using var observer = new Phase6ApplyObserver(ns.Name, malformed.Key, harness.Local, harness.ActiveOne); + try + { + var values = CreateValueGroups(ns.Name, + CreateDisseminationValue(harness.ActiveOne, malformed), + CreateDisseminationValue(harness.ActiveOne, valid)); + if (antiEntropy) + { + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = harness.ActiveOne, Values = values }); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + else + { + var acknowledgment = await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = harness.ActiveOne, Values = values }, + TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 0, valid.ToVersion }, + acknowledgment.Acknowledgments[ns.Name].Select(static digest => digest.Version)); + } + + Assert.Equal(0, ns.GetVersion(malformed.Key)); + Assert.Equal(valid.ToVersion, ns.GetVersion(valid.Key)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal(statistics.ActivationCount, + harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + Assert.Single(observer.Events); + var failure = Assert.Single(logger.Entries); + Assert.Equal(malformed.Key, failure.State["Key"]); + Assert.Equal(malformed.ToVersion, failure.State["Version"]); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public void LoadNamespaceDigestsPruneInactiveCacheEntriesAndReuseActivePayloads() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, + new TestOptionsMonitor(new()), + serializer); + var statistics = CreatePhase5Statistics(1); + var cached = new Dictionary(); + foreach (var silo in new[] { harness.Local, harness.ActiveOne, harness.ActiveTwo }) + { + harness.Publisher.PeriodicStatistics[silo] = statistics; + cached.Add(silo, ns.CreateValue(silo, statistics)); + } + + Assert.Equal( + cached.Keys.Order(), + ns.Digests.Select(static digest => Assert.IsType(digest.Key.Value)).Order()); + Assert.All(cached, entry => Assert.True(entry.Value.Payload.Equals(ns.CreateValue(entry.Key, statistics).Payload))); + + harness.StatusOracle.SetStatus(harness.ActiveOne, SiloStatus.Dead); + harness.StatusOracle.SetStatus(harness.ActiveTwo, SiloStatus.Joining); + var restarted = SiloAddress.New(harness.ActiveOne.Endpoint, harness.ActiveOne.Generation + 1); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Active); + var digests = ns.Digests.OrderBy(static digest => Assert.IsType(digest.Key.Value)).ToArray(); + Assert.Equal( + new[] { harness.Local, restarted }.Order(), + digests.Select(static digest => Assert.IsType(digest.Key.Value))); + Assert.Equal(statistics.DateTime.Ticks, digests.Single(digest => digest.Key == new DisseminationKey(harness.Local)).Version); + Assert.Equal(0, digests.Single(digest => digest.Key == new DisseminationKey(restarted)).Version); + Assert.True(cached[harness.Local].Payload.Equals(ns.CreateValue(harness.Local, statistics).Payload)); + Assert.False(cached[harness.ActiveOne].Payload.Equals(ns.CreateValue(harness.ActiveOne, statistics).Payload)); + Assert.False(cached[harness.ActiveTwo].Payload.Equals(ns.CreateValue(harness.ActiveTwo, statistics).Payload)); + + harness.StatusOracle.SetStatus(harness.Local, SiloStatus.Stopping); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Dead); + Assert.Empty(ns.Digests); + Assert.False(cached[harness.Local].Payload.Equals(ns.CreateValue(harness.Local, statistics).Payload)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewApplicationWaitIsLocallyBounded(bool cancelCaller) + { + var local = CreateSilo(39511); + var peer = CreateSilo(39512); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.ExpectedKeys.Add("blocked"); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken applicationToken = default; + ns.ApplyHandler = (_, token) => + { + applicationToken = token; + entered.TrySetResult(); + return new(release.Task); + }; + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.Inner.CreateItem(peer, "blocked", 1)), + }); + var protocol = CreateProtocol(transport, [ns], timeProvider: clock); + using var caller = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + try + { + var round = protocol.RunAntiEntropyRound(caller.Token); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(round.IsCompleted); + if (cancelCaller) + { + await caller.CancelAsync(); + var exception = await Assert.ThrowsAnyAsync( + () => round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(caller.Token, exception.CancellationToken); + } + else + { + clock.Advance(TimeSpan.FromSeconds(1)); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.True(applicationToken.IsCancellationRequested); + Assert.False(release.Task.IsCompleted); + Assert.Equal(0, ns.GetVersion("blocked")); + ns.ApplyHandler = null; + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion("blocked")); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + Assert.Equal(0, Assert.Single(transport.AntiEntropyRequests[1].Request.Digests[ns.Name]).Version); + release.SetResult(DisseminationApplyResult.Applied); + Assert.Equal(1, ns.GetVersion("blocked")); + Assert.Equal(1, ns.Inner.ApplyCounts["blocked"]); + } + finally + { + release.TrySetResult(DisseminationApplyResult.Rejected); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewSlowPeerDoesNotExpireCompletedResponseApplication() + { + var local = CreateSilo(39521); + var good = CreateSilo(39522); + var slow = CreateSilo(39523); + var transport = new FakeTransport(local, good, slow); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Inner.ExpectedKeys.Add("value"); + var slowEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var applyEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var applyRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + applyEntered.TrySetResult(); + await applyRelease.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + transport.ExchangeAntiEntropyHandler = (peer, _, _) => + { + if (peer.Equals(slow)) + { + slowEntered.TrySetResult(); + return new(slowRelease.Task); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = good, + Values = CreateValueGroups(ns.Inner.CreateItem(good, "value", 1)), + }); + }; + var protocol = CreateProtocol(transport, [ns], options => options.Overlay.AntiEntropyPeerCount = 2, clock); + try + { + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await slowEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromSeconds(1)); + await applyEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(round.IsCompleted); + Assert.False(slowRelease.Task.IsCompleted); + applyRelease.SetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion("value")); + Assert.Equal(1, ns.Inner.ApplyCounts["value"]); + } + finally + { + applyRelease.TrySetResult(); + slowRelease.TrySetResult(new DisseminationAntiEntropyResponse { Sender = slow }); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1)] + public async Task ProtocolReviewBroadcastUsesLocalReceiptAge(int followingTtlSeconds) + { + var local = CreateSilo(39531); + var peer = CreateSilo(39532); + var clock = new FakeTimeProvider(); + var ns = new ProtocolReviewNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(10); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ns.ApplyHandler = async (value, token) => + { + entered.TrySetResult(); + await release.Task.WaitAsync(token); + return await ns.Inner.ApplyValueAsync(value, token); + }; + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], timeProvider: clock); + try + { + var receive = protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, + ns.Inner.CreateItem(peer, "first", 1), + new DisseminationBroadcastValue + { + Value = ns.Inner.CreateValue("later", 1), + TimeToLive = TimeSpan.FromSeconds(followingTtlSeconds), + }), TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromSeconds(1)); + release.SetResult(); + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 1, 0 }, response.Acknowledgments[ns.Name].Select(static digest => digest.Version)); + Assert.Equal(new[] { new DisseminationKey("first") }, ns.Attempts.Select(static value => value.Key)); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(0, ns.GetVersion("later")); + } + finally + { + release.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewExpiredQueuedLoadDoesNotMutateOwnerState() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.Enabled = true; + options.Dissemination.StaleItemTtl = TimeSpan.FromSeconds(1); + var ns = new DeploymentLoadStatisticsDisseminationNamespace( + harness.Publisher, new TestOptionsMonitor(options), + services.GetRequiredService()); + var clock = new FakeTimeProvider(); + var ownerEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseOwner = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, _) => + { + if (silo.Equals(harness.Local)) + { + ownerEntered.TrySetResult(); + releaseOwner.Task.GetAwaiter().GetResult(); + } + })); + var protocol = CreateProtocol(new FakeTransport(harness.Local, harness.ActiveOne), [ns], timeProvider: clock); + var owner = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + try + { + await ownerEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var value = ns.CreateValue(harness.ActiveOne, CreatePhase5Statistics(2)); + var receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = harness.ActiveOne, + Values = CreateValueGroups(ns.Name, CreateDisseminationValue(harness.ActiveOne, value)), + }, TestContext.Current.CancellationToken); + Assert.False(receive.IsCompleted); + clock.Advance(TimeSpan.FromSeconds(1)); + var response = await receive.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, Assert.Single(response.Acknowledgments[ns.Name]).Version); + Assert.False(owner.IsCompleted); + releaseOwner.SetResult(); + await owner.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await harness.Publisher.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(true); + }, TestContext.Current.CancellationToken); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal(0, ns.GetVersion(harness.ActiveOne)); + } + finally + { + releaseOwner.TrySetResult(); + await owner.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(2, 1024)] + [InlineData(10, 16)] + [InlineData(2, 16)] + public async Task ProtocolReviewBroadcastCapsAllNamespacesAndAcknowledgesOnlyPrefix(int maxItems, int maxBytes) + { + var local = CreateSilo(39541); + var peer = CreateSilo(39542); + var first = new ProtocolReviewNamespace(local); + var second = new ProtocolReviewNamespace(local, "second"); + var protocol = CreateProtocol(new FakeTransport(local, peer), [first, second], options => + { + options.MaxBatchItems = maxItems; + options.MaxBatchBytes = maxBytes; + }); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [first.Name] = [first.Inner.CreateItem(peer, "chain", 1), first.Inner.CreateItem(peer, "chain", 2)], + [second.Name] = [second.Inner.CreateItem(peer, "later", 3)], + }, + }; + try + { + var acknowledgment = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + Assert.Single(acknowledgment.Acknowledgments); + Assert.Equal(2, Assert.Single(acknowledgment.Acknowledgments[first.Name]).Version); + Assert.Equal(new long[] { 1, 2 }, first.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(16, first.Attempts.Sum(static value => value.Payload.Length)); + Assert.Empty(second.Attempts); + Assert.Equal(0, second.GetVersion("later")); + + acknowledgment = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + Assert.Single(acknowledgment.Acknowledgments); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[second.Name]).Version); + Assert.Equal(2, first.Attempts.Count); + Assert.Single(second.Attempts); + Assert.Equal(8, second.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(3, second.GetVersion("later")); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewReceiveCursorPassesRejectedHeadAndAppliesFullUpdates(bool antiEntropy) + { + var local = CreateSilo(39551); + var peer = CreateSilo(39552); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.ExpectedKeys.UnionWith(new DisseminationKey[] { "bad", "chain" }); + ns.Options.ExpectedUpdateCadence = TimeSpan.Zero; + var values = CreateValueGroups( + CreateDisseminationValue(peer, new DisseminationValue("bad", 0, 1, new byte[9])), + ns.Inner.CreateItem(peer, "chain", 1), + ns.Inner.CreateItem(peer, "chain", 2)); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse { Sender = peer, Values = values }); + var protocol = CreateProtocol(transport, [ns], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }); + try + { + for (var round = 0; round < 3; round++) + { + if (antiEntropy) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + else + { + var response = await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = peer, Values = values }, + TestContext.Current.CancellationToken); + if (round == 0) + { + Assert.Empty(response.Acknowledgments); + } + else + { + Assert.Equal(round, Assert.Single(response.Acknowledgments[ns.Name]).Version); + } + } + + Assert.Equal(round, ns.GetVersion("chain")); + } + + Assert.Equal(0, ns.GetVersion("bad")); + Assert.Equal(new long[] { 1, 2 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(new long[] { 0, 0 }, ns.Attempts.Select(static value => value.FromVersion)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(2, 1024)] + [InlineData(10, 16)] + [InlineData(2, 16)] + public async Task ProtocolReviewAntiEntropyCapsEachResponseBeforeApplication(int maxItems, int maxBytes) + { + var local = CreateSilo(39561); + var badPeer = CreateSilo(39562); + var goodPeer = CreateSilo(39563); + var ns = new ProtocolReviewNamespace(local); + var other = new ProtocolReviewNamespace(local, "other"); + var transport = new FakeTransport(local, badPeer, goodPeer); + transport.ExchangeAntiEntropyHandler = (peer, _, _) => + { + var first = peer.Equals(badPeer) + ? CreateDisseminationValue(peer, new DisseminationValue("chain", 0, 1, BitConverter.GetBytes(99L))) + : ns.Inner.CreateItem(peer, "chain", 1); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() + { + [ns.Name] = [first, ns.Inner.CreateItem(peer, "chain", 2)], + [other.Name] = [other.Inner.CreateItem(peer, "omitted", 3)], + }, + }); + }; + var protocol = CreateProtocol(transport, [ns, other], options => + { + options.Overlay.AntiEntropyPeerCount = 2; + options.MaxBatchItems = maxItems; + options.MaxBatchBytes = maxBytes; + }); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + Assert.Equal(2, ns.GetVersion("chain")); + Assert.Equal(new long[] { 1, 2, 2 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(24, ns.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(1, ns.Inner.ApplyCounts["chain"]); + Assert.Empty(other.Attempts); + Assert.Equal(0, other.GetVersion("omitted")); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(3, other.GetVersion("omitted")); + Assert.Equal(2, other.Attempts.Count); + Assert.Equal(16, other.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(3, ns.Attempts.Count); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ProtocolReviewLargerResponderOptionsRepairThroughLocalCaps(bool ignoresBudgets) + { + var local = CreateSilo(39571); + var peer = CreateSilo(39572); + var receiver = new ProtocolReviewNamespace(local); + var source = new FakeNamespace(peer); + DisseminationKey[] keys = ["first", "second", "last"]; + receiver.Inner.ExpectedKeys.UnionWith(keys); + foreach (var key in keys) + { + source.SetValue(key, 1); + } + + var responses = new List(); + var remote = CreateProtocol(new FakeTransport(peer, local), source, options => + { + options.MaxBatchItems = 10; + options.MaxBatchBytes = 1024; + }); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = async (_, request, token) => + { + Assert.Equal(2, request.MaxResponseItems); + Assert.Equal(16, request.MaxResponseBytes); + if (ignoresBudgets) + { + request = new DisseminationAntiEntropyRequest + { + Sender = request.Sender, + Digests = request.Digests, + SupportedNamespaces = request.SupportedNamespaces, + }; + } + + var response = await remote.ReceiveAntiEntropy(request, token); + responses.Add(response); + return response; + }; + var protocol = CreateProtocol(transport, [receiver], options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }); + try + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(ignoresBudgets ? 3 : 2, GetAntiEntropyResponseValues(Assert.Single(responses)).Count()); + Assert.Equal(ignoresBudgets ? 24 : 16, GetAntiEntropyResponseValues(responses[0]).Sum(static item => item.Value.Payload.Length)); + Assert.Equal(keys.Take(2), receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(16, receiver.Attempts.Sum(static value => value.Payload.Length)); + Assert.Equal(0, receiver.GetVersion(keys[2])); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + Assert.Equal(keys[2], Assert.Single(GetAntiEntropyResponseValues(responses[1])).Value.Key); + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.All(keys, key => Assert.Equal(1, receiver.GetVersion(key))); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + await remote.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ProtocolReviewAdvancingInventoryDoesNotAliasResponderAndReceiverCursors() + { + var local = CreateSilo(39591); + var peer = CreateSilo(39592); + var clock = new FakeTimeProvider(); + var receiver = new ProtocolReviewNamespace(local); + receiver.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + var source = new FakeNamespace(peer); + DisseminationKey[] keys = ["A", "B", "C", "D"]; + receiver.Inner.ExpectedKeys.UnionWith(keys); + var responses = new List(); + var remote = CreateProtocol(new FakeTransport(peer, local), source, options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }, clock); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = async (_, request, token) => + { + var response = await remote.ReceiveAntiEntropy(request, token); + responses.Add(response); + return response; + }; + var protocol = CreateProtocol(transport, [receiver], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }, clock); + try + { + for (var round = 1; round <= keys.Length; round++) + { + foreach (var key in keys) + { + source.SetValue(key, round); + } + + clock.Advance(TimeSpan.FromSeconds(2)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(new long[] { 1, 2, 3, 4 }, keys.Select(receiver.GetVersion)); + Assert.Equal(4, responses.Count); + Assert.Equal(keys, responses.Select(static response => Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key)); + Assert.All(transport.AntiEntropyRequests, request => + { + Assert.Equal(1, request.Request.MaxResponseItems); + Assert.Equal(8, request.Request.MaxResponseBytes); + Assert.Equal(4, request.Request.Digests[receiver.Name].Count); + }); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + await remote.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(0, null, "MaxResponseItems")] + [InlineData(-1, null, "MaxResponseItems")] + [InlineData(null, 0, "MaxResponseBytes")] + [InlineData(null, -1, "MaxResponseBytes")] + public async Task ProtocolReviewInvalidResponseBudgetsHaveNoReceiveSideEffects( + int? maxResponseItems, + int? maxResponseBytes, + string parameterName) + { + var local = CreateSilo(39601); + var peer = CreateSilo(39602); + var ns = new FakeNamespace(local); + ns.SetValue("pending", 1); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + try + { + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = protocol.Publish(ns, "pending", 1, TestContext.Current.CancellationToken); + var repairRequests = ns.RepairRequestCount; + var exception = Assert.Throws(() => + { + _ = protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("pending", 1)), + MaxResponseItems = maxResponseItems, + MaxResponseBytes = maxResponseBytes, + }, TestContext.Current.CancellationToken); + }); + Assert.Equal(parameterName, exception.ParamName); + Assert.Equal(maxResponseItems ?? maxResponseBytes, Assert.IsType(exception.ActualValue)); + Assert.Equal(repairRequests, ns.RepairRequestCount); + Assert.Equal(peer, Assert.Single(protocol.GetUnconfirmedPeers(ns))); + return pending; + }); + Assert.True(await publication); + + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var sent = Assert.Single(transport.BroadcastBatches); + Assert.Equal(1, Assert.Single(GetBroadcastValues(sent.Batch)).Value.ToVersion); + Assert.Equal(peer, sent.Peer); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(null, null)] + [InlineData(1, null)] + [InlineData(null, 8)] + [InlineData(1, 8)] + [InlineData(10, 1024)] + public async Task ProtocolReviewResponseBudgetsRoundTripAndBoundEveryRepairProbe(int? maxResponseItems, int? maxResponseBytes) + { + var local = CreateSilo(39611); + var peer = CreateSilo(39612); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var ns = new ProtocolReviewNamespace(local); + DisseminationKey[] keys = ["A", "B", "C"]; + foreach (var key in keys) + { + ns.Inner.SetValue(key, 1); + } + + var requests = new List(); + ns.RepairHandler = request => + { + requests.Add(request); + return ns.Inner.CreateRepair(request); + }; + var request = new DisseminationAntiEntropyRequest + { + Sender = peer, + SupportedNamespaces = [ns.Name], + Digests = CreateAntiEntropyRequestDigest(ns.Name, ("A", 0), ("B", 0), ("C", 0)), + MaxResponseItems = maxResponseItems, + MaxResponseBytes = maxResponseBytes, + }; + var copy = Assert.IsType( + serializer.Deserialize(serializer.SerializeToArray(request))); + Assert.Equal(maxResponseItems, copy.MaxResponseItems); + Assert.Equal(maxResponseBytes, copy.MaxResponseBytes); + Assert.Equal(peer, copy.Sender); + Assert.Equal(ns.Name, Assert.Single(copy.SupportedNamespaces)); + Assert.Equal(keys, copy.Digests[ns.Name].Select(static digest => digest.Key)); + var protocol = CreateProtocol(new FakeTransport(local, peer), [ns], options => + { + options.MaxBatchItems = 2; + options.MaxBatchBytes = 16; + }); + try + { + var response = await protocol.ReceiveAntiEntropy(copy, TestContext.Current.CancellationToken); + var maxItems = Math.Min(2, maxResponseItems ?? int.MaxValue); + var maxBytes = Math.Min(16, maxResponseBytes ?? int.MaxValue); + var count = Math.Min(maxItems, maxBytes / sizeof(long)); + Assert.Equal(keys.Take(count), GetAntiEntropyResponseValues(response).Select(static item => item.Value.Key)); + Assert.Equal(count * sizeof(long), GetAntiEntropyResponseValues(response).Sum(static item => item.Value.Payload.Length)); + Assert.True(response.Truncated); + var probes = count + (count < maxItems ? 1 : 0); + Assert.Equal(probes, requests.Count); + for (var index = 0; index < probes; index++) + { + Assert.Equal(keys[index], requests[index].Key); + Assert.Equal(maxBytes, requests[index].MaxBatchBytes); + Assert.Equal(ns.Options.MaxPayloadBytes, requests[index].MaxPayloadBytes); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(1)] + [InlineData(10)] + public async Task LargerBroadcastSenderRetriesOnlyUnacknowledgedKeys(int senderLimit) + { + var sourceAddress = CreateSilo(39581); + var receiverAddress = CreateSilo(39582); + var clock = new FakeTimeProvider(); + var source = new FakeNamespace(sourceAddress); + DisseminationKey[] keys = ["first", "second", "third"]; + for (var index = 0; index < keys.Length; index++) + { + source.SetValue(keys[index], index + 1); + } + var receiver = new ProtocolReviewNamespace(receiverAddress); + var receivingProtocol = CreateProtocol(new FakeTransport(receiverAddress, sourceAddress), [receiver], options => + { + options.MaxBatchItems = 1; + options.MaxBatchBytes = 8; + }, clock); + var acknowledgments = new List(); + var compactResponses = new List(); + var sentItemCounts = new List(); + var transport = new FakeTransport(sourceAddress, receiverAddress); + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + sentItemCounts.Add(GetBroadcastValues(batch).Count()); + var acknowledgment = await receivingProtocol.ReceiveBroadcast(batch, token); + compactResponses.Add(acknowledgment.AllVersionsAcknowledged); + acknowledgments.Add(acknowledgment.AllVersionsAcknowledged + ? GetBroadcastValues(batch).Max(static item => item.Value.ToVersion) + : Assert.Single(acknowledgment.Acknowledgments[source.Name]).Version); + return acknowledgment; + }; + var sendingQueue = CreateBroadcastQueue(transport, [source], options => + { + options.MaxBatchItems = senderLimit; + options.MaxBatchBytes = 1024; + }, clock); + using var schedules = new BroadcastScheduleObserver(); + var retry = senderLimit > 1 ? WaitForRetry() : null; + try + { + Assert.True(sendingQueue.NotifyBatch(receiverAddress, source, + [.. keys.Select((key, index) => new DisseminationBroadcastQueue.KeyNotification(key, index + 1, true))])); + for (var round = 0; round < 2 && retry is not null; round++) + { + var scheduled = await retry; + Assert.Equal(TimeSpan.FromMilliseconds(100), scheduled.DueTime); + Assert.Equal(1, scheduled.Attempt); + Assert.Equal(Enumerable.Range(1, round + 1).Select(static version => (long)version), acknowledgments); + retry = round == 0 ? WaitForRetry() : null; + clock.Advance(scheduled.DueTime); + } + + await sendingQueue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new long[] { 1, 2, 3 }, acknowledgments); + int[] expectedCounts = senderLimit == 1 ? [1, 1, 1] : [3, 2, 1]; + bool[] expectedCompact = senderLimit == 1 ? [true, true, true] : [false, false, true]; + Assert.Equal(expectedCounts, sentItemCounts); + Assert.Equal(expectedCompact, compactResponses); + Assert.Equal(new long[] { 0, 0, 0 }, receiver.Attempts.Select(static value => value.FromVersion)); + Assert.Equal(new long[] { 1, 2, 3 }, receiver.Attempts.Select(static value => value.ToVersion)); + Assert.Equal(keys, receiver.Attempts.Select(static value => value.Key)); + Assert.Equal(new long[] { 1, 2, 3 }, keys.Select(receiver.GetVersion)); + } + finally + { + source.Options.Enabled = false; + await sendingQueue.StopAsync(TestContext.Current.CancellationToken); + await receivingProtocol.StopAsync(TestContext.Current.CancellationToken); + } + + Task WaitForRetry() => schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(sourceAddress) && scheduled.Peer.Equals(receiverAddress) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + private sealed class ProtocolReviewNamespace(SiloAddress local, DisseminationNamespace? name = null) : IDisseminationNamespace + { + public FakeNamespace Inner { get; } = new(local, name); + public DisseminationNamespace Name => Inner.Name; + public DisseminationNamespaceOptions Options => Inner.Options; + public IEnumerable Digests => Inner.Digests; + public List Attempts { get; } = []; + public Func>? ApplyHandler { get; set; } + public Func? RepairHandler { get; set; } + public long GetVersion(DisseminationKey key) => Inner.GetVersion(key); + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + RepairHandler is { } handler ? handler(request) : Inner.CreateRepair(request); + + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) + { + Attempts.Add(value); + return ApplyHandler is { } handler ? handler(value, cancellationToken) : Inner.ApplyValueAsync(value, cancellationToken); + } + } + + private sealed class ProtocolReviewPayloadObserver(SiloAddress local, DisseminationNamespace namespaceName) : IObserver> + { + public Exception Failure { get; } = new InvalidOperationException("Payload observer failure."); + public int Count { get; private set; } + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent dropped + && dropped.Namespace == namespaceName + && Equals(dropped.LocalSilo, local)) + { + Count++; + throw Failure; + } + } + + public void OnCompleted() { } + public void OnError(Exception error) { } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs new file mode 100644 index 00000000000..133e505f201 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.PumpDiagnostics.cs @@ -0,0 +1,325 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +// Pump failure metrics have no peer tag, so throwing listeners must not overlap other tests. +[Collection(DisseminationDiagnosticCollection.Name)] +public partial class DisseminationProtocolTests +{ + [Theory] + [InlineData(0, false)] + [InlineData(1, false)] + [InlineData(1, true)] + public Task PumpFailureMetricExceptionsPreserveAcceptedWorkAndCapacity(int failedTimerChanges, bool failScheduleMetric) => + VerifyPumpFailureMetricIsolation(failedTimerChanges, failScheduleMetric); + + [Fact] + public Task PumpFailureMetricExceptionsPreservePermanentFailureAndWaiters() => + VerifyPumpFailureMetricIsolation(failedTimerChanges: 2, failScheduleMetric: false); + + [Theory] + [InlineData("warning")] + [InlineData("pump-failure")] + [InlineData("retry-schedule")] + public Task RecoveryDiagnosticLoggerExceptionsFailPumpAndPendingWaiters(string failureStage) => + VerifyPumpFailureMetricIsolation( + failedTimerChanges: 1, failScheduleMetric: failureStage == "retry-schedule", recoveryLogFailureStage: failureStage); + + private static async Task VerifyPumpFailureMetricIsolation(int failedTimerChanges, bool failScheduleMetric, string? recoveryLogFailureStage = null) + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40951); + var peer = CreateSilo(40952); + var clock = new RecordingFakeTimeProvider(); + var logger = new PumpDiagnosticsLogger(); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + ns.SetValue("original", 7); + ns.SetValue("next", 9); + var transport = new FakeTransport(local, peer); + var started = Enumerable.Range(0, 3) + .Select(static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var release = Enumerable.Range(0, 3) + .Select(static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)).ToArray(); + var sent = new ConcurrentQueue<(string Key, long Version, long Payload)>(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + var value = Assert.Single(GetBroadcastValues(batch)).Value; + sent.Enqueue((value.Key.ToString(), value.ToVersion, BitConverter.ToInt64(value.Payload.Span))); + var attempt = Interlocked.Increment(ref sendCount); + if (attempt <= started.Length) + { + started[attempt - 1].TrySetResult(); + await release[attempt - 1].Task.WaitAsync(token); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var responseFailure = new InvalidOperationException("The first response observer fails before acknowledgment processing."); + var responseCount = 0; + var options = new DisseminationOptions { Enabled = true, MaxConcurrentSends = 1 }; + options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4); + var queue = new DisseminationBroadcastQueue( + clock, local, transport.GrainFactory, new TestOptionsMonitor(options), [ns], logger, + responseObserver: (_, _) => + { + if (Interlocked.Increment(ref responseCount) == 1) + { + throw responseFailure; + } + }); + using var schedules = new BroadcastScheduleObserver(); + using var listener = new MeterListener(); + var pumpMetricFailure = new InvalidOperationException("Pump failure metric callback fails."); + var scheduleMetricFailure = new InvalidOperationException("Retry schedule metric callback fails."); + var pumpStatuses = new ConcurrentQueue(); + var scheduleMetricFailures = 0; + var listenerArmed = 0; + listener.InstrumentPublished = (instrument, meterListener) => + { + if (Volatile.Read(ref listenerArmed) != 0 + && instrument.Meter.Name == DisseminationInstruments.MeterName + && (instrument.Name == DisseminationInstruments.PumpFailuresName + || failScheduleMetric && instrument.Name == DisseminationInstruments.BroadcastScheduledName)) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, _, tags, _) => + { + if (Volatile.Read(ref listenerArmed) == 0) + { + return; + } + + if (instrument.Name == DisseminationInstruments.PumpFailuresName) + { + pumpStatuses.Enqueue((string)Assert.Single(tags.ToArray(), static tag => tag.Key == "status").Value!); + throw pumpMetricFailure; + } + + if (tags.ToArray().Any(static tag => tag.Key == "reason" && Equals(tag.Value, "retry"))) + { + Interlocked.Increment(ref scheduleMetricFailures); + throw scheduleMetricFailure; + } + }); + var recoveryLogFailure = new InvalidOperationException("The recovery diagnostic logger fails."); + var pendingRecoveryFlush = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var recoveryLogFailures = 0; + if (recoveryLogFailureStage is not null) + { + logger.OnLog = (level, exception) => + { + if (level == LogLevel.Warning) + { + var pendingFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(pendingFlush.IsCompleted); + pendingRecoveryFlush.TrySetResult(pendingFlush); + } + + var shouldThrow = recoveryLogFailureStage switch + { + "warning" => level == LogLevel.Warning, + "pump-failure" => ReferenceEquals(exception, pumpMetricFailure), + "retry-schedule" => ReferenceEquals(exception, scheduleMetricFailure), + _ => false, + }; + if (pendingRecoveryFlush.Task.IsCompletedSuccessfully && shouldThrow + && Interlocked.CompareExchange(ref recoveryLogFailures, 1, 0) == 0) + { + throw recoveryLogFailure; + } + }; + } + + Task? stop = null; + try + { + var initialSchedule = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Immediate, + TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(queue.Notify(peer, ns, "original")); + await WaitForPhase(initialSchedule, "initial immediate send scheduled"); + var initial = await initialSchedule; + Assert.Equal(TimeSpan.Zero, initial.DueTime); + Assert.Equal(0, initial.Attempt); + await WaitForPhase(started[0].Task, "original send in flight"); + Assert.False(queue.Notify(peer, ns, "next")); + + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(firstFlush.IsCompleted); + var retrySchedule = failedTimerChanges < 2 && recoveryLogFailureStage is null + ? schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), cancellationToken) + : null; + Volatile.Write(ref listenerArmed, 1); + listener.Start(); + clock.ThrowOnNextTimerChanges(failedTimerChanges); + release[0].TrySetResult(); + + if (failedTimerChanges == 2 || recoveryLogFailureStage is not null) + { + var failedFlush = firstFlush; + if (recoveryLogFailureStage is not null) + { + await WaitForPhase(firstFlush, "active waiter completed before recovery diagnostics"); + await WaitForPhase(pendingRecoveryFlush.Task, "pending waiter attached before recovery logger throws"); + failedFlush = await pendingRecoveryFlush.Task; + } + + var failure = await Assert.ThrowsAsync( + () => WaitForPhase(failedFlush, "permanently failed flush waiter")); + var notificationFailure = Assert.Throws(() => queue.Notify(peer, ns, "next")); + Assert.Same(failure, notificationFailure.InnerException); + var laterFlushFailure = await Assert.ThrowsAsync( + () => WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "later flush observes terminal failure")); + Assert.Same(failure, laterFlushFailure.InnerException); + stop = queue.StopAsync(cancellationToken); + var stopFailure = await Assert.ThrowsAsync( + () => WaitForPhase(stop, "permanently failed drain waiter")); + Assert.Same(failure, stopFailure); + Assert.Equal(2, failure.InnerExceptions.Count); + Assert.Equal("The test timer fails one scheduled change.", failure.InnerExceptions[0].Message); + if (recoveryLogFailureStage is null) + { + Assert.Equal("The test timer fails one scheduled change.", failure.InnerExceptions[1].Message); + } + else + { + Assert.Same(recoveryLogFailure, failure.InnerExceptions[1]); + Assert.Equal(1, Volatile.Read(ref recoveryLogFailures)); + } + + Assert.Same(failure, Assert.Single(logger.Entries, static entry => entry.Level == LogLevel.Error).Exception); + var recoveredMetrics = recoveryLogFailureStage is "pump-failure" or "retry-schedule" ? 2 : 1; + Assert.Equal(Enumerable.Repeat("recovered", recoveredMetrics).Append("permanent"), pumpStatuses); + Assert.Equal(1, Volatile.Read(ref sendCount)); + } + else + { + await WaitForPhase(firstFlush, "failed iteration flush waiter released"); + await WaitForPhase(retrySchedule!, "recovered retry timer armed"); + var retry = await retrySchedule!; + Assert.Equal(failedTimerChanges + 1, retry.Attempt); + Assert.Equal(TimeSpan.FromMilliseconds(100 * (failedTimerChanges + 1)), retry.DueTime); + Assert.Equal(initial.Epoch, retry.Epoch); + Assert.Equal(1, Volatile.Read(ref sendCount)); + + clock.Advance(retry.DueTime); + await WaitForPhase(started[1].Task, "original accepted value retried"); + var retryFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(retryFlush.IsCompleted); + release[1].TrySetResult(); + await WaitForPhase(retryFlush, "original accepted value acknowledged"); + Assert.Equal(new[] { ("original", 7L, 7L), ("original", 7L, 7L) }, sent); + Assert.True(queue.Notify(peer, ns, "original", force: false)); + await WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "acknowledged duplicate suppressed"); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal(2, Volatile.Read(ref sendCount)); + + var nextSchedule = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Immediate && value.Epoch > retry.Epoch, + TimeSpan.FromSeconds(5), cancellationToken); + Assert.True(queue.Notify(peer, ns, "next")); + await WaitForPhase(nextSchedule, "freed capacity schedules the next key"); + var next = await nextSchedule; + Assert.Equal(TimeSpan.Zero, next.DueTime); + Assert.Equal(0, next.Attempt); + Assert.Equal(retry.Epoch + 1, next.Epoch); + await WaitForPhase(started[2].Task, "next key admitted to transport"); + var nextFlush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(nextFlush.IsCompleted); + release[2].TrySetResult(); + await WaitForPhase(nextFlush, "next key acknowledged"); + Assert.True(queue.Notify(peer, ns, "next", force: false)); + await WaitForPhase(queue.FlushPendingBroadcast(cancellationToken), "next acknowledged duplicate suppressed"); + Assert.Equal(new[] { ("original", 7L, 7L), ("original", 7L, 7L), ("next", 9L, 9L) }, sent); + Assert.Equal(3, ns.RepairRequestCount); + Assert.Equal(3, Volatile.Read(ref sendCount)); + Assert.Equal(Enumerable.Repeat("recovered", failedTimerChanges + 1), pumpStatuses); + Assert.Equal(failedTimerChanges, logger.Entries.Count(static entry => entry.Level == LogLevel.Warning)); + Assert.DoesNotContain(logger.Entries, static entry => entry.Level == LogLevel.Error); + } + + Assert.Same(responseFailure, Assert.Single(logger.Entries, entry => ReferenceEquals(entry.Exception, responseFailure)).Exception); + Assert.Equal(pumpStatuses.Count, logger.Entries.Count(entry => + entry.Level == LogLevel.Debug && ReferenceEquals(entry.Exception, pumpMetricFailure))); + Assert.Equal(failScheduleMetric ? 1 : 0, Volatile.Read(ref scheduleMetricFailures)); + Assert.Equal(scheduleMetricFailures, logger.Entries.Count(entry => + entry.Level == LogLevel.Debug && ReferenceEquals(entry.Exception, scheduleMetricFailure))); + } + finally + { + Volatile.Write(ref listenerArmed, 0); + listener.Dispose(); + clock.ThrowOnNextTimerChanges(0); + foreach (var completion in release) + { + completion.TrySetResult(); + } + + if (stop is null) + { + ns.Options.Enabled = false; + using var cleanup = new CancellationTokenSource(); + cleanup.Cancel(); + try + { + await WaitForPhase(queue.StopAsync(cleanup.Token), "queue cleanup"); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cleanup.Token) + { + // An assertion failure can leave accepted work behind a fake-time retry; cancel its drain. + } + } + } + + async Task WaitForPhase(Task task, string phase) + { + try + { + await task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Timed out during {phase} for {local} -> {peer}; sends={Volatile.Read(ref sendCount)}, timer failures={failedTimerChanges}.", + exception); + } + } + } + + private sealed class PumpDiagnosticsLogger : ILogger + { + public ConcurrentQueue<(LogLevel Level, Exception? Exception)> Entries { get; } = new(); + + public Action? OnLog { get; set; } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Entries.Enqueue((logLevel, exception)); + OnLog?.Invoke(logLevel, exception); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs new file mode 100644 index 00000000000..1d2e7c2b8d6 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.QueueReview.cs @@ -0,0 +1,547 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.Scheduler; +using Xunit; + +namespace UnitTests.Dissemination; + +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task SingleKeyNotificationAvoidsTemporaryAllocations() + { + var local = CreateSilo(40901); + var peer = CreateSilo(40902); + var ns = new FakeNamespace(local); + ns.SetValue("value", 1); + var transport = new FakeTransport(local, peer); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + var allocated = BeforeBroadcastPumpsRun(() => + { + Assert.True(queue.Notify(peer, ns, "value")); + for (var iteration = 0; iteration < 128; iteration++) + { + Assert.True(queue.Notify(peer, ns, "value", force: false)); + } + + const int iterations = 1024; + var accepted = true; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var iteration = 0; iteration < iterations; iteration++) + { + accepted &= queue.Notify(peer, ns, "value", force: false); + } + + var result = GC.GetAllocatedBytesForCurrentThread() - before; + Assert.True(accepted); + return result; + }); + Assert.True(allocated < 512, $"Repeated single-key notifications allocated {allocated} bytes."); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(1, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task BatchNotificationPreservesLaterUpdatesWhenAdmissionRejectsAKey() + { + var cancellationToken = TestContext.Current.CancellationToken; + var local = CreateSilo(40903); + var peer = CreateSilo(40904); + var ns = new FakeNamespace(local); + ns.Options.MaxPendingItemCount = 1; + ns.SetValue("value", 1); + ns.SetValue("rejected", 1); + var transport = new FakeTransport(local, peer); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + transport.BroadcastBatches.Add((peer, batch)); + if (transport.BroadcastBatches.Count == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task; + } + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + try + { + Assert.True(queue.Notify(peer, ns, "value")); + var firstFlush = queue.FlushPendingBroadcast(cancellationToken); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + ns.SetValue("value", 2); + Assert.False(queue.NotifyBatch(peer, ns, + [new("rejected", 1, true), new("value", 2, true)])); + releaseFirst.TrySetResult(); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + await queue.FlushPendingBroadcast(cancellationToken); + + Assert.Equal(new long[] { 1, 2 }, transport.BroadcastBatches.Select( + entry => Assert.Single(GetBroadcastValues(entry.Batch)).Value.ToVersion)); + Assert.All(transport.BroadcastBatches, entry => + Assert.Equal(new DisseminationKey("value"), Assert.Single(GetBroadcastValues(entry.Batch)).Value.Key)); + } + finally + { + releaseFirst.TrySetResult(); + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken); + } + } + + [Fact] + public void DisseminationKeysSortAcrossSupportedKinds() + { + var firstSilo = CreateSilo(40101); + var secondSilo = CreateSilo(40102); + DisseminationKey[] keys = [secondSilo, "b", DisseminationKey.Default, firstSilo, string.Empty, "a"]; + DisseminationKey[] expected = [DisseminationKey.Default, string.Empty, "a", "b", firstSilo, secondSilo]; + + Array.Sort(keys); + + Assert.Equal(expected, keys); + for (var left = 0; left < expected.Length; left++) + { + for (var right = 0; right < expected.Length; right++) + { + Assert.Equal(Math.Sign(left - right), Math.Sign(expected[left].CompareTo(expected[right]))); + Assert.Equal(Math.Sign(left - right), Math.Sign(((IComparable)expected[left]).CompareTo(expected[right]))); + } + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task RedundantNotificationAndFlushWakesRespectRetryBoundary(bool transportFailure, bool publishNewVersion) + { + var cancellationToken = TestContext.Current.CancellationToken; + var (local, peer, transport, ns) = CreatePeerFixture(41601, 41602); + var clock = new FakeTimeProvider(); + var versions = new List(); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + cancellationToken); + ns.SetValue("value", 1); + transport.SendBroadcastResponseHandler = (_, batch, _) => + { + versions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + if (versions.Count == 2) + { + secondAttempt.TrySetResult(); + } + + return versions.Count == 1 + ? transportFailure + ? Task.FromException(new InvalidOperationException("Initial send fails.")) + : Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 0)] }, + }) + : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + var initialFlush = BeforeBroadcastPumpsRun(() => + { + Assert.True(queue.Notify(peer, ns, "value")); + var flush = queue.FlushPendingBroadcast(cancellationToken); + Assert.False(flush.IsCompleted); + return flush; + }); + + await initialFlush.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var scheduled = await retry.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(TimeSpan.FromMilliseconds(100), scheduled.DueTime); + Assert.Equal(new long[] { 1 }, versions); + clock.Advance(TimeSpan.FromMilliseconds(99)); + Assert.Equal(new long[] { 1 }, versions); + Assert.False(secondAttempt.Task.IsCompleted); + if (publishNewVersion) + { + ns.SetValue("value", 2); + Assert.True(queue.Notify(peer, ns, "value")); + } + else + { + clock.Advance(TimeSpan.FromMilliseconds(1)); + } + + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + Assert.Equal(new long[] { 1, publishNewVersion ? 2 : 1 }, versions); + Assert.Equal(2, ns.RepairRequestCount); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ShutdownDrainRetriesUntilAcceptedWorkIsAcknowledged(bool transportFailure) + { + var local = CreateSilo(40111); + var peer = CreateSilo(40112); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, 2); + var sentVersions = new List(); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, batch, token) => + { + sentVersions.Add(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + if (sentVersions.Count == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(token); + if (transportFailure) + { + throw new InvalidOperationException("Transient send failure."); + } + + return new DisseminationBroadcastResponse(); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Peer.Equals(peer) + && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Task? stop = null; + try + { + Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + stop = queue.StopAsync(cancellation.Token); + releaseFirst.TrySetResult(); + var scheduled = await retry; + Assert.False(stop.IsCompleted); + Assert.False(queue.Notify(peer, ns, "after-stop")); + Assert.Equal(new[] { 2L }, sentVersions); + + clock.Advance(scheduled.DueTime); + await stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(new[] { 2L, 2L }, sentVersions); + } + finally + { + releaseFirst.TrySetResult(); + cancellation.Cancel(); + stop ??= queue.StopAsync(cancellation.Token); + try + { + await stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cancellation.Token) + { + } + } + } + + [Fact] + public async Task ShutdownDrainCancellationSurfacesIncompleteDelivery() + { + var local = CreateSilo(40121); + var peer = CreateSilo(40122); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, 1); + var sends = 0; + transport.SendBroadcastResponseHandler = (_, _, _) => + { + Interlocked.Increment(ref sends); + return Task.FromResult(new DisseminationBroadcastResponse()); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + value => value.LocalSilo.Equals(local) && value.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + Assert.True(queue.Notify(peer, ns, FakeNamespace.DefaultKey)); + var stop = queue.StopAsync(cancellation.Token); + try + { + await retry; + Assert.False(stop.IsCompleted); + } + finally + { + cancellation.Cancel(); + } + + var exception = await Assert.ThrowsAnyAsync( + () => stop.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(1, Volatile.Read(ref sends)); + } + + [Fact] + public async Task UnsupportedResponsePreservesPublicationsMadeDuringItsSend() + { + var local = CreateSilo(40131); + var peer = CreateSilo(40132); + var transport = new FakeTransport(local, peer); + var clock = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.SetValue("updated", 1); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var batches = new List(); + transport.SendBroadcastResponseHandler = async (_, batch, cancellationToken) => + { + batches.Add(batch); + if (batches.Count == 1) + { + entered.SetResult(); + await release.Task.WaitAsync(cancellationToken); + return new DisseminationBroadcastResponse { UnsupportedNamespaces = [ns.Name] }; + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + try + { + Assert.True(queue.Notify(peer, ns, "updated")); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.SetValue("updated", 2); + ns.SetValue("new", 7); + Assert.True(queue.Notify(peer, ns, "updated")); + Assert.True(queue.Notify(peer, ns, "new")); + var newerFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + release.SetResult(); + await Task.WhenAll(firstFlush, newerFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, batches.Count); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batches[0])).Value.ToVersion); + var delivered = GetBroadcastValues(batches[1]).ToDictionary(value => value.Value.Key, value => value.Value.ToVersion); + Assert.Equal(2, delivered.Count); + Assert.Equal(2, delivered["updated"]); + Assert.Equal(7, delivered["new"]); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(2, batches.Count); + } + finally + { + release.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task DisabledAntiEntropySleepsUntilOptionsChangeAndUnsubscribesOnStop() + { + var local = CreateSilo(40141); + var peer = CreateSilo(40142); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var clock = new ReviewTimeProvider(); + var options = new ReviewOptionsMonitor(new DisseminationOptions { Enabled = false }); + var exchanges = Channel.CreateUnbounded(); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (destination, _, _) => + { + exchanges.Writer.TryWrite(Interlocked.Increment(ref exchangeCount)); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = destination }); + }; + var localDetails = new FakeLocalSiloDetails(local); + var target = new DisseminationSystemTarget( + localDetails, + transport.GrainFactory, + new DisseminationMembership(transport.MembershipManager, localDetails, Options.Create(options.CurrentValue)), + options, + [ns], + clock, + NullLogger.Instance, + NullLogger.Instance, + CreatePhase4SystemTargetShared(local)); + var lifecycle = Substitute.For(); + ILifecycleObserver? observer = null; + lifecycle.Subscribe(Arg.Any(), Arg.Any(), Arg.Any()).Returns(call => + { + observer = call.ArgAt(2); + return new ReviewSubscription(static () => { }); + }); + ((ILifecycleParticipant)target).Participate(lifecycle); + Assert.NotNull(observer); + await observer.OnStart(TestContext.Current.CancellationToken); + try + { + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromDays(1)); + await target.RunOrQueueTask(_ => Task.FromResult(true), TestContext.Current.CancellationToken); + Assert.Equal(0, Volatile.Read(ref exchangeCount)); + + options.Set(new DisseminationOptions { Enabled = true }); + Assert.Equal(1, await exchanges.Reader.ReadAsync(TestContext.Current.CancellationToken)); + await clock.WaitForChange(options.CurrentValue.Overlay.AntiEntropyInterval, TestContext.Current.CancellationToken); + + options.Set(new DisseminationOptions { Enabled = false }); + await clock.WaitForChange(Timeout.InfiniteTimeSpan, TestContext.Current.CancellationToken); + clock.Advance(TimeSpan.FromDays(1)); + await target.RunOrQueueTask(_ => Task.FromResult(true), TestContext.Current.CancellationToken); + Assert.Equal(1, Volatile.Read(ref exchangeCount)); + + options.Set(new DisseminationOptions { Enabled = true }); + Assert.Equal(2, await exchanges.Reader.ReadAsync(TestContext.Current.CancellationToken)); + } + finally + { + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + shutdown.CancelAfter(TimeSpan.FromSeconds(5)); + await observer.OnStop(shutdown.Token); + } + + Assert.Equal(0, options.SubscriptionCount); + } + + private sealed class ReviewOptionsMonitor(DisseminationOptions initial) : IOptionsMonitor + { + private readonly object _lock = new(); + private readonly List> _listeners = []; + private readonly TaskCompletionSource _unsubscribed = new(TaskCreationOptions.RunContinuationsAsynchronously); + private DisseminationOptions _current = initial; + + public DisseminationOptions CurrentValue => Volatile.Read(ref _current); + public Task Unsubscribed => _unsubscribed.Task; + public DisseminationOptions Get(string? name) => CurrentValue; + public int SubscriptionCount + { + get + { + lock (_lock) + { + return _listeners.Count; + } + } + } + + public IDisposable OnChange(Action listener) + { + lock (_lock) + { + _listeners.Add(listener); + } + + return new ReviewSubscription(() => + { + lock (_lock) + { + _listeners.Remove(listener); + } + + _unsubscribed.TrySetResult(); + }); + } + + public void Set(DisseminationOptions value) + { + Volatile.Write(ref _current, value); + Action[] listeners; + lock (_lock) + { + listeners = [.. _listeners]; + } + + foreach (var listener in listeners) + { + listener(value, null); + } + } + } + + private sealed class ReviewSubscription(Action unsubscribe) : IDisposable + { + private Action? _unsubscribe = unsubscribe; + public void Dispose() => Interlocked.Exchange(ref _unsubscribe, null)?.Invoke(); + } + + private sealed class ReviewTimeProvider : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private readonly Channel _changes = Channel.CreateUnbounded(); + + public override long TimestampFrequency => _inner.TimestampFrequency; + public override long GetTimestamp() => _inner.GetTimestamp(); + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public void Advance(TimeSpan duration) => _inner.Advance(duration); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ReviewTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + _changes.Writer.TryWrite(dueTime); + return timer; + } + + public async Task WaitForChange(TimeSpan expected, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(5)); + while (await _changes.Reader.ReadAsync(deadline.Token) != expected) + { + } + } + + private sealed class ReviewTimer(ReviewTimeProvider owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + var changed = inner.Change(dueTime, period); + owner._changes.Writer.TryWrite(dueTime); + return changed; + } + + public void Dispose() => inner.Dispose(); + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs new file mode 100644 index 00000000000..b2709379711 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationProtocolTests.cs @@ -0,0 +1,7995 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using CsCheck; +#if NET10_0_OR_GREATER +using Microsoft.Accordant; +#endif +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Orleans.Runtime.MembershipService; +using Orleans.Runtime.Scheduler; +using Orleans.Serialization; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public partial class DisseminationProtocolTests +{ + [Fact] + public async Task PublishPropagatesCancellationBeforeQueueing() + { + var (_, peer, transport, ns) = CreatePeerFixture(39101, 39102); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + try + { + var exception = await Assert.ThrowsAnyAsync( + async () => await protocol.Publish(ns, FakeNamespace.DefaultKey, 1, cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + Assert.Equal(0, transport.GetTargetResolutionCount(peer)); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public void PushBroadcastUsesRequestResponseSemantics() + { + var method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PushBroadcast)); + + Assert.NotNull(method); + Assert.False(method.IsDefined(typeof(Orleans.Concurrency.OneWayAttribute), inherit: false)); + Assert.Equal("017AA907", method.GetCustomAttribute()?.Alias); + Assert.Equal(typeof(Task), method.ReturnType); + + method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.ExchangeAntiEntropy)); + Assert.NotNull(method); + Assert.Equal("EF58CB3D", method.GetCustomAttribute()?.Alias); + + method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PublishAggregated)); + Assert.NotNull(method); + Assert.False(method.IsDefined(typeof(Orleans.Concurrency.OneWayAttribute), inherit: false)); + Assert.Equal("8C810F59", method.GetCustomAttribute()?.Alias); + Assert.Equal(typeof(Task), method.ReturnType); + } + + [Fact] + public async Task PublishQueuesBroadcastToDeterministicTreeChildrenWithoutCapabilityProbing() + { + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(result); + var expectedChildren = GetOriginatorTreeTargets(local, peers, fanout: 2); + Assert.Equal(expectedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + Assert.All(transport.BroadcastBatches, batch => Assert.Equal(item, GetBroadcastValues(batch.Batch).Single().Value)); + Assert.Empty(transport.AntiEntropyRequests); + } + + [Fact] + public async Task BroadcastSendsHonorMaxConcurrentSends() + { + const int maxConcurrentSends = 2; + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 6).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var gate = new object(); + var limitReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSends = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var started = 0; + var observedMax = 0; + var sentPeers = new List(); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + lock (gate) + { + inFlight++; + started++; + observedMax = Math.Max(observedMax, inFlight); + if (started == maxConcurrentSends) + { + limitReached.TrySetResult(true); + } + } + + try + { + await releaseSends.Task.WaitAsync(cancellationToken); + lock (gate) + { + sentPeers.Add(target); + } + } + finally + { + lock (gate) + { + inFlight--; + } + } + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = maxConcurrentSends; + options.Overlay.FanOutFactor = static _ => 10; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + Assert.True(await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + try + { + await limitReached.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await Task.Delay(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(maxConcurrentSends, started); + Assert.Equal(maxConcurrentSends, inFlight); + Assert.Equal(maxConcurrentSends, observedMax); + } + } + finally + { + releaseSends.TrySetResult(true); + } + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(peers.OrderBy(static peer => peer), sentPeers.OrderBy(static peer => peer)); + Assert.True(observedMax <= maxConcurrentSends); + } + } + + [Fact] + public async Task PublishRejectsInvalidValuesBeforeQueueing() + { + var (_, _, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + ns.SetValue("obsolete", version: 10); + + ns.Options.MaxPayloadBytes = sizeof(long); + var oversized = new DisseminationValue( + "oversized", + fromVersion: 0, + toVersion: 1, + new byte[sizeof(long) + 1]); + var obsolete = ns.CreateValue("obsolete", sequence: 5); + + Assert.False(await PublishValue(protocol, ns, oversized, TestContext.Current.CancellationToken)); + Assert.False(await protocol.Publish(ns, "obsolete", version: 11, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task PublishReturnsFalseWhenRootIsMissingFromMembership() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + transport.PeerStatuses[local] = SiloStatus.Dead; + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.False(result); + Assert.Equal(1, transport.RefreshMembershipCallCount); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task PublishContinuesAfterPeerSendFailure() + { + var local = CreateSilo(11111); + var joining = CreateSilo(11112); + var active = CreateSilo(11113); + var transport = new FakeTransport(local, joining, active); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Equals(target, joining)) + { + throw new InvalidOperationException("joining peer is not yet reachable"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var value = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var result = await PublishValue(protocol, ns, value, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(result); + Assert.Equal(new[] { active }, transport.BroadcastBatches.Select(batch => batch.Peer)); + } + + [Fact] + public async Task NewPublicationWakesFailedPeerWithoutWaitingForBackoff() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + try + { + var firstResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken); + await retry; + Assert.Equal(1, sendCount); + var beforeNotification = timeProvider.GetTimestamp(); + var secondResult = await PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken); + await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(firstResult); + Assert.True(secondResult); + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(beforeNotification)); + Assert.Equal(2, sendCount); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(2, GetBroadcastValues(batch.Batch).Single().Value.ToVersion); + } + finally + { + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(4000, 100)] + [InlineData(50, 50)] + public async Task SendFailureRetriesAutomaticallyWithBoundedBackoff(int antiEntropyMilliseconds, int retryMilliseconds) + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + firstAttempt.TrySetResult(); + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + secondAttempt.TrySetResult(); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(antiEntropyMilliseconds), + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var scheduledRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var retry = await scheduledRetry; + Assert.Equal(TimeSpan.FromMilliseconds(retryMilliseconds), retry.DueTime); + + timeProvider.Advance(retry.DueTime - TimeSpan.FromMilliseconds(1)); + Assert.False(secondAttempt.Task.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Single(transport.BroadcastBatches); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NewNotificationResetsRetryBackoff() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var thirdAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + switch (Interlocked.Increment(ref sendCount)) + { + case 1: + firstAttempt.TrySetResult(); + throw new InvalidOperationException("first transient send failure"); + case 2: + secondAttempt.TrySetResult(); + throw new InvalidOperationException("second transient send failure"); + default: + transport.BroadcastBatches.Add((target, batch)); + thirdAttempt.TrySetResult(); + return Task.CompletedTask; + } + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var firstRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 1, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromMilliseconds(100), (await firstRetry).DueTime); + var secondRetry = schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry && e.Attempt == 2, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMilliseconds(100)); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.FromMilliseconds(200), (await secondRetry).DueTime); + + var beforeNotification = timeProvider.GetTimestamp(); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + await thirdAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(beforeNotification)); + Assert.Equal(3, sendCount); + Assert.Equal(2, Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NamespaceNotificationFlushesImmediately() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sent.TrySetResult(batch); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var start = timeProvider.GetTimestamp(); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + + var scheduled = await schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Immediate, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, scheduled.DueTime); + + var batch = await sent.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(TimeSpan.Zero, timeProvider.GetElapsedTime(start)); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerVersionAdvancesOnlyFromExplicitAcknowledgment() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var secondAttempt = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + var acknowledgedVersion = Interlocked.Increment(ref sendCount) == 1 ? 0 : 1; + if (acknowledgedVersion == 1) + { + secondAttempt.TrySetResult(); + } + + return Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, acknowledgedVersion), + ], + }, + }); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + using var schedule = new BroadcastScheduleObserver(); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + var retry = await schedule.WaitAsync( + e => e.LocalSilo.Equals(local) && e.Peer.Equals(peer) && e.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + + timeProvider.Advance(retry.DueTime); + await secondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Equal( + new[] { 1L, 1L }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task UnsupportedNamespaceResponseStopsAutomaticRetries() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + Interlocked.Increment(ref sendCount); + return Task.FromResult(new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [FakeNamespace.DefaultName], + }); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task UnsupportedNamespaceResponseCompletesQueuedFlushWaiter() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + return new DisseminationBroadcastResponse + { + UnsupportedNamespaces = [FakeNamespace.DefaultName], + }; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + + releaseFirstSend.TrySetResult(); + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task RemovedPendingKeyIsDroppedWithoutRetry() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + Interlocked.Increment(ref sendCount); + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue("removed", sequence: 1), TestContext.Current.CancellationToken); + ns.RemoveValue("removed"); + return pending; + }); + Assert.True(await publication); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var repairRequestCount = ns.RepairRequestCount; + timeProvider.Advance(TimeSpan.FromMinutes(1)); + await Task.Delay(TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + + Assert.Equal(0, sendCount); + Assert.Equal(repairRequestCount, ns.RepairRequestCount); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task BroadcastBatchingStopsPeerFlushAfterSendFailure() + { + var (local, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = new FakeTimeProvider(); + var sendCount = 0; + var attempted = new List(); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + attempted.Add(Assert.Single(GetBroadcastValues(batch)).Value.Key); + if (Interlocked.Increment(ref sendCount) == 1) + { + throw new InvalidOperationException("transient send failure"); + } + + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + ns.SetValue("first", 1); + ns.SetValue("second", 1); + ns.SetValue("third", 1); + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxBatchItems = 1, timeProvider); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + try + { + Assert.True(queue.NotifyBatch(peer, ns, [new("first", 1, true), new("second", 1, true), new("third", 1, true)])); + await retry; + Assert.Equal(1, sendCount); + Assert.Equal(new DisseminationKey[] { "first" }, attempted); + Assert.Equal(1, ns.RepairRequestCount); + Assert.Empty(transport.BroadcastBatches); + } + finally + { + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task BroadcastPeerFailureDoesNotBlockOtherPeerPumps() + { + var local = CreateSilo(11111); + var failedPeer = CreateSilo(11112); + var healthyPeer = CreateSilo(11113); + var transport = new FakeTransport(local, failedPeer, healthyPeer); + var timeProvider = new FakeTimeProvider(); + var failedPeerAttempts = 0; + var healthySent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + if (Equals(target, failedPeer) && Interlocked.Increment(ref failedPeerAttempts) == 1) + { + throw new InvalidOperationException("transient peer failure"); + } + + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((target, batch)); + } + + if (target.Equals(healthyPeer)) + { + healthySent.TrySetResult(); + } + + return Task.CompletedTask; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.FanOutFactor = static _ => 10; + }, timeProvider); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(failedPeer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); + await Task.WhenAll(retry, healthySent.Task).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, failedPeerAttempts); + Assert.Equal(new[] { healthyPeer }, GetSentBroadcastPeers(transport)); + + ClearBroadcastBatches(transport); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(2, failedPeerAttempts); + Assert.Equal(new[] { failedPeer, healthyPeer }.OrderBy(static peer => peer), GetSentBroadcastPeers(transport).OrderBy(static peer => peer)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipRefreshDropsPendingBroadcastForRemovedPeers() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + var publications = BeforeBroadcastPumpsRun(() => + { + var before = PublishValue(protocol, ns, ns.CreateValue("before-removal", sequence: 1), TestContext.Current.CancellationToken); + transport.Peers.Remove(peer); + var after = PublishValue(protocol, ns, ns.CreateValue("during-removal", sequence: 2), TestContext.Current.CancellationToken); + return (before, after); + }); + Assert.True(await publications.before); + Assert.True(await publications.after); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task StopDrainsPendingBroadcastPumps() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + var pending = BeforeBroadcastPumpsRun(() => ( + Publication: PublishValue(protocol, ns, ns.CreateValue("before-stop", sequence: 1), TestContext.Current.CancellationToken), + Stop: protocol.StopAsync(TestContext.Current.CancellationToken))); + Assert.True(await pending.Publication); + await pending.Stop; + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.Equal(new DisseminationKey("before-stop"), Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key); + } + + [Fact] + public async Task FlushPendingBroadcastWaitsForInFlightFlush() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + transport.BroadcastBatches.Add((target, batch)); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + try + { + Assert.False(flushTask.IsCompleted); + } + finally + { + releaseSend.TrySetResult(); + } + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Single(transport.BroadcastBatches); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task NotificationDuringInFlightSendRepairsLatestFullValueAfterAcknowledgment() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + releaseFirstSend.TrySetResult(); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { (From: 0L, To: 1L), (From: 0L, To: 2L) }, + transport.BroadcastBatches.Select(batch => + { + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + return (value.Value.FromVersion, value.Value.ToVersion); + })); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task MembershipRefreshCompletesFlushWaitersForRemovedPeers() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1, timeProvider); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("in-flight", sequence: 1), TestContext.Current.CancellationToken)); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("pending", sequence: 1), TestContext.Current.CancellationToken)); + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + + transport.Peers.Remove(peer); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("after-removal", sequence: 1), TestContext.Current.CancellationToken)); + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ReceiveBroadcastForwardsOnlyToLocalTreeChildren() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var root = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, root, peers, fanout: 2, sender: root); + Assert.Equal(expectedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + } + + [Fact] + public async Task ReceiveBroadcastAppliesAllValuesBeforeForwarding() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var sender = CreateSilo(11113); + var peer = CreateSilo(11114); + var transport = new FakeTransport(local, sender, peer); + DisseminationKey firstKey = new("first"); + DisseminationKey secondKey = new("second"); + var ns = new FakeNamespace(local); + var forwardingObserved = false; + transport.SendBroadcastHandler = (target, batch, cancellationToken) => + { + forwardingObserved = true; + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(2, ns.GetVersion(secondKey)); + transport.BroadcastBatches.Add((target, batch)); + return Task.CompletedTask; + }; + + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var first = ns.CreateItem(root, firstKey, sequence: 1); + var second = ns.CreateItem(root, secondKey, sequence: 2); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, first, second), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(forwardingObserved); + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(2, ns.GetVersion(secondKey)); + } + + [Fact] + public async Task ReceiveBroadcastContinuesAfterFailedValue() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var child = CreateSilo(11113); + var transport = new FakeTransport(local, root, child); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + DisseminationKey firstKey = new("first"); + DisseminationKey failedKey = new("failed"); + DisseminationKey secondKey = new("second"); + var first = ns.CreateItem(root, firstKey, sequence: 1); + var failed = CreateDisseminationValue( + root, + new DisseminationValue(failedKey, fromVersion: 0, toVersion: 1, Array.Empty())); + var second = ns.CreateItem(root, secondKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(root, first, failed, second), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(firstKey)); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(1, ns.GetVersion(secondKey)); + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + Assert.Equal( + new[] { firstKey, secondKey }.OrderBy(static key => key), + GetBroadcastValues(forwarded.Batch).Select(static item => item.Value.Key).OrderBy(static key => key)); + } + + [Fact] + public async Task ReceiveBroadcastDoesNotRefreshMembershipForRemovedOriginators() + { + var local = CreateSilo(11111); + var sender = CreateSilo(11112); + var firstOriginator = CreateSilo(11120); + var secondOriginator = CreateSilo(11121); + var transport = new FakeTransport(local, sender); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + var first = ns.CreateItem(firstOriginator, "first", sequence: 1); + var second = ns.CreateItem(secondOriginator, "second", sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, first, second), TestContext.Current.CancellationToken); + + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal(1, ns.GetVersion("first")); + Assert.Equal(1, ns.GetVersion("second")); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task DuplicateBroadcastDoesNotForwardAgain() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var root = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + var batch = CreateBroadcastBatch(root, item); + + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, root, peers, fanout: 2, sender: root); + Assert.Equal(expectedChildren.Count, transport.BroadcastBatches.Count); + Assert.Equal(1, ns.ApplyCounts[item.Value.Key]); + } + + [Fact] + public async Task DuplicateBroadcastSchedulesForwardingWhenNoPriorQueueStateExists() + { + var silos = Enumerable.Range(11111, 8).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + var sender = silos[0]; + var local = silos[1]; + var peers = silos.Where(silo => !Equals(silo, local)).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var item = ns.CreateItem(sender, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch(sender, item), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var expectedChildren = GetForwardingTreeTargets(local, sender, peers, fanout: 2, sender: sender); + Assert.Equal( + expectedChildren.OrderBy(static peer => peer), + transport.BroadcastBatches.Select(static batch => batch.Peer).OrderBy(static peer => peer)); + Assert.False(ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); + } + + [Theory] + [InlineData(1, 0, 3, 3, 1)] + [InlineData(2, 2, 3, 2, 0)] + [InlineData(1, 2, 3, 1, 0)] + [InlineData(1, -1, 3, 1, 0)] + [InlineData(1, 0, 0, 1, 0)] + [InlineData(1, 0, -1, 1, 0)] + public async Task ReceiveBroadcastAppliesOnlyPositiveFullValues( + long localVersion, long fromVersion, long toVersion, long expectedVersion, int expectedApplyCount) + { + var (_, root, transport, ns) = CreatePeerFixture(11112, 11111); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, localVersion); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, toVersion, fromVersion); + using var rejected = new Phase6ApplyObserver(ns.Name, FakeNamespace.DefaultKey, transport.LocalSilo, root); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch(root, item), + TestContext.Current.CancellationToken); + + Assert.Equal(expectedVersion, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount, ns.ApplyCounts.GetValueOrDefault(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount != 0, ns.ApplyCounts.ContainsKey(FakeNamespace.DefaultKey)); + Assert.Equal(expectedApplyCount == 0 ? 1 : 0, rejected.Events.Count); + var acknowledgment = Assert.Single(response.Acknowledgments[ns.Name]); + Assert.Equal(FakeNamespace.DefaultKey, acknowledgment.Key); + Assert.Equal(expectedVersion, acknowledgment.Version); + } + + [Fact] + public async Task ReceiveBroadcastDoesNotRefreshMembershipForMissingRoot() + { + var root = CreateSilo(11111); + var local = CreateSilo(11112); + var sender = CreateSilo(11113); + var peer = CreateSilo(11114); + var transport = new FakeTransport(local, sender, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal(new[] { peer }, transport.BroadcastBatches.Select(static batch => batch.Peer)); + } + + [Fact] + public async Task ReceiveBroadcastRoutesWithoutRefreshingForMissingRoot() + { + var local = CreateSilo(11111); + var sender = CreateSilo(11112); + var peer = CreateSilo(11113); + var root = CreateSilo(11120); + var transport = new FakeTransport(local, sender, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 2); + var item = ns.CreateItem(root, FakeNamespace.DefaultKey, sequence: 1); + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(sender, item), TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(0, transport.RefreshMembershipCallCount); + Assert.Equal( + GetForwardingTreeTargets(local, sender, new[] { sender, peer }, fanout: 2, sender: sender), + transport.BroadcastBatches.Select(static batch => batch.Peer)); + } + + [Fact] + public async Task TreeRoutingInvalidatesCachedTopologyWhenActivePeersChange() + { + var local = CreateSilo(11115); + var initialPeers = Enumerable.Range(11112, 3).Select(CreateSilo).ToList(); + var transport = new FakeTransport(local, initialPeers.ToArray()); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 2; + }); + var item = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1); + + var initialResult = await PublishValue(protocol, ns, item, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var initialChildren = GetOriginatorTreeTargets(local, transport.Peers, fanout: 2); + + foreach (var peer in Enumerable.Range(11116, 8).Select(CreateSilo)) + { + transport.Peers.Add(peer); + var updatedChildren = GetOriginatorTreeTargets(local, transport.Peers, fanout: 2); + if (!initialChildren.SequenceEqual(updatedChildren)) + { + transport.BroadcastBatches.Clear(); + var updatedItem = ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2); + + var updatedResult = await PublishValue(protocol, ns, updatedItem, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(initialResult); + Assert.True(updatedResult); + Assert.Equal(updatedChildren.OrderBy(static peer => peer), transport.BroadcastBatches.Select(batch => batch.Peer).OrderBy(static peer => peer)); + return; + } + } + + throw new InvalidOperationException("The test did not find a peer set which changes the local tree children."); + } + + [Fact] + public async Task BroadcastMaterializesLatestQueuedVersion() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + + var publications = BeforeBroadcastPumpsRun(() => ( + First: PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), TestContext.Current.CancellationToken), + Second: PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), TestContext.Current.CancellationToken))); + var first = await publications.First; + var second = await publications.Second; + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.True(first); + Assert.True(second); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + Assert.Equal(2, value.Value.ToVersion); + } + + [Fact] + public async Task BroadcastRepairsLatestFullValueAfterAcknowledgment() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 2), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal( + new[] { (From: 0L, To: 1L), (From: 0L, To: 2L) }, + transport.BroadcastBatches.Select(batch => + { + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + return (value.Value.FromVersion, value.Value.ToVersion); + })); + } + + [Theory] + [InlineData(3)] + [InlineData(4)] + public async Task BroadcastPublicationSupersedesMissingHistoryWithLatestFullValue(long latestVersion) + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns); + + ns.SetValue(FakeNamespace.DefaultKey, latestVersion); + Assert.True(await protocol.Publish(ns, FakeNamespace.DefaultKey, 1, TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)); + Assert.Equal(0, value.Value.FromVersion); + Assert.Equal(latestVersion, value.Value.ToVersion); + Assert.Equal(latestVersion, BitConverter.ToInt64(value.Value.Payload.Span)); + } + + [Fact] + public async Task BroadcastPeerSenderCachesSystemTarget() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns, options => options.Overlay.FanOutFactor = static _ => 1); + + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", sequence: 1), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("second", sequence: 2), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal(1, transport.GetTargetResolutionCount(peer)); + } + + [Fact] + public void FixedTreeRoutingSatisfiesReachabilityAndFanoutInvariants() + { + Gen.Select(Gen.Int[1, 64], Gen.Int[1, 8], Gen.Int[0, 63], static (count, fanout, rootSeed) => + { + var rootIndex = rootSeed % count; + return (Count: count, Fanout: fanout, RootIndex: rootIndex); + }).Sample(testCase => + { + var silos = CreateSilos(testCase.Count); + var root = silos[testCase.RootIndex]; + var options = new DisseminationOverlayOptions + { + FanOutFactor = _ => testCase.Fanout, + }; + var members = silos.ToImmutableArray(); + var snapshots = silos.ToDictionary( + static silo => silo, + silo => new DisseminationMembershipSnapshot( + new MembershipVersion(1), + silo, + members, + options)); + var directTargets = snapshots[root].OriginatorTreeTargets; + + Assert.DoesNotContain(root, directTargets); + Assert.Equal(directTargets.Length, directTargets.Distinct().Count()); + Assert.True(directTargets.Length <= Math.Min((testCase.Fanout * 2), Math.Max(0, testCase.Count - 1))); + + var reached = GetReachedParticipants(root, snapshots); + Assert.Equal(silos.OrderBy(static silo => silo), reached.OrderBy(static silo => silo)); + }); + } + + [Fact] + public void AntiEntropyPeerSelectionSamplesDistinctMembers() + { + Gen.Select(Gen.Int[2, 64], Gen.Int[0, 63], Gen.Int[1, 64], static (count, localSeed, peerCount) => + { + var localIndex = localSeed % count; + return (Count: count, LocalIndex: localIndex, PeerCount: peerCount); + }).Sample(testCase => + { + var silos = CreateSilos(testCase.Count); + var local = silos[testCase.LocalIndex]; + var transport = new FakeTransport(local, silos.Where(silo => !Equals(silo, local)).ToArray()); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AntiEntropyPeerCount = testCase.PeerCount; + }); + + protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken).GetAwaiter().GetResult(); + var peers = transport.AntiEntropyRequests.Select(static request => request.Peer).ToArray(); + + Assert.Equal(Math.Min(testCase.PeerCount, testCase.Count - 1), peers.Length); + Assert.Equal(peers.Length, peers.Distinct().Count()); + Assert.DoesNotContain(local, peers); + Assert.All(peers, peer => Assert.Contains(peer, silos)); + }); + } + + [Theory] + [InlineData(3, false)] + [InlineData(3, true)] + [InlineData(5, false)] + public async Task AntiEntropyResponseIncludesOnlyRequestedNewerValues(long peerVersion, bool hasUnrequestedKey) + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + var key = hasUnrequestedKey ? new DisseminationKey("requested") : FakeNamespace.DefaultKey; + ns.SetValue(key, version: 5); + if (hasUnrequestedKey) + { + ns.SetValue("omitted", version: 5); + } + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (key, peerVersion)), + }, TestContext.Current.CancellationToken); + + if (peerVersion == 5) + { + Assert.Empty(response.Values); + } + else + { + var item = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(key, item.Value.Key); + Assert.Equal(5, item.Value.ToVersion); + } + Assert.False(response.Truncated); + } + + [Fact] + public async Task OversizedAntiEntropyKeyDoesNotStarveLaterRepairs() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + ns.Options.MaxPayloadBytes = sizeof(long); + ns.PublishValue(new DisseminationValue( + "oversized", + fromVersion: 0, + toVersion: 1, + new byte[sizeof(long) + 1])); + ns.SetValue("valid", version: 2); + var protocol = CreateProtocol(transport, ns); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + ("oversized", 0), + ("valid", 0)), + }, TestContext.Current.CancellationToken); + + var value = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(new DisseminationKey("valid"), value.Value.Key); + Assert.False(response.Truncated); + } + + [Fact] + public async Task AntiEntropyDigestRequestsAreSuppressedUntilExpectedCadenceExpires() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = new TestTimeProvider(); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + await protocol.ReceiveBroadcast( + CreateBroadcastBatch(peer, ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)), + TestContext.Current.CancellationToken); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + var capabilityRequest = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Empty(capabilityRequest.Digests); + Assert.Equal([ns.Name], capabilityRequest.SupportedNamespaces); + transport.AntiEntropyRequests.Clear(); + + timeProvider.Advance(TimeSpan.FromSeconds(2) - TimeSpan.FromMilliseconds(1)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + capabilityRequest = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Empty(capabilityRequest.Digests); + Assert.Equal([ns.Name], capabilityRequest.SupportedNamespaces); + transport.AntiEntropyRequests.Clear(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + var request = Assert.Single(transport.AntiEntropyRequests).Request; + var digest = Assert.Single(request.Digests[ns.Name]); + Assert.Equal(FakeNamespace.DefaultKey, digest.Key); + } + + [Fact] + public async Task AntiEntropyCapabilityExchangeConfirmsNamespaceWithoutRepairDigests() + { + var (_, peer, transport, ns) = CreatePeerFixture(11113, 11114); + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + Assert.Empty(request.Digests); + Assert.Equal([ns.Name], request.SupportedNamespaces); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + SupportedNamespaces = [ns.Name], + }); + }; + var protocol = CreateProtocol(transport, ns); + + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task AntiEntropyCapabilityOnlyRoundUsesNamespaceStaleItemTtl() + { + var local = CreateSilo(11115); + var peer = CreateSilo(11116); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var exchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + Assert.Empty(request.Digests); + Assert.Equal([ns.Name], request.SupportedNamespaces); + exchangeStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + cancellationObserved.TrySetResult(); + } + + return new DisseminationAntiEntropyResponse { Sender = target }; + }; + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await exchangeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + timeProvider.Advance(ns.Options.StaleItemTtl); + + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + [Fact] + public async Task DuplicateBroadcastDoesNotPostponeAntiEntropyProbe() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = new TestTimeProvider(); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(2); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + var batch = CreateBroadcastBatch(peer, ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)); + + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromSeconds(2) - TimeSpan.FromMilliseconds(1)); + await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + var request = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Single(request.Digests[ns.Name]); + } + + [Fact] + public async Task AntiEntropySendsOneDigestRequestPerPeer() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + DisseminationKey firstKey = new("first"); + DisseminationKey secondKey = new("second"); + var firstNamespace = new FakeNamespace(local, new DisseminationNamespace("first-namespace")); + firstNamespace.ExpectedKeys.Add(firstKey); + var secondNamespace = new FakeNamespace(local, new DisseminationNamespace("second-namespace")); + secondNamespace.ExpectedKeys.Add(secondKey); + var protocol = CreateProtocol(transport, new IDisseminationNamespace[] { firstNamespace, secondNamespace }, options => + { + options.Overlay.AntiEntropyPeerCount = 1; + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + var request = Assert.Single(transport.AntiEntropyRequests).Request; + Assert.Equal( + new[] { firstNamespace.Name, secondNamespace.Name }.OrderBy(static name => name), + request.Digests.Keys.OrderBy(static name => name)); + var firstDigest = Assert.Single(request.Digests[firstNamespace.Name]); + Assert.Equal(firstKey, firstDigest.Key); + Assert.Equal(0, firstDigest.Version); + var secondDigest = Assert.Single(request.Digests[secondNamespace.Name]); + Assert.Equal(secondKey, secondDigest.Key); + Assert.Equal(0, secondDigest.Version); + } + + [Fact] + public async Task AntiEntropyExchangeFailureDoesNotBackOffPeer() + { + var (_, _, transport, ns) = CreatePeerFixture(); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + if (Interlocked.Increment(ref exchangeCount) == 1) + { + throw new InvalidOperationException("transient anti-entropy failure"); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = target }); + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.Overlay.AntiEntropyPeerCount = 1; + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(2, Volatile.Read(ref exchangeCount)); + Assert.Equal(2, transport.AntiEntropyRequests.Count); + } + + [Fact] + public async Task AntiEntropyExchangePropagatesCancellation() + { + var (_, _, transport, ns) = CreatePeerFixture(); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var exchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellation = new CancellationTokenSource(); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + exchangeStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return new DisseminationAntiEntropyResponse { Sender = target }; + }; + + var protocol = CreateProtocol(transport, ns, options => options.Overlay.AntiEntropyPeerCount = 1); + var exchangeTask = protocol.RunAntiEntropyRound(cancellation.Token); + await exchangeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await exchangeTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task AntiEntropyResponseHonorsCancellation() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest { Sender = peer }, + cancellation.Token)); + } + + [Fact] + public async Task AntiEntropyExchangesAreNotLimitedByMaxConcurrentSends() + { + var local = CreateSilo(11111); + var peers = Enumerable.Range(11112, 3).Select(CreateSilo).ToArray(); + var transport = new FakeTransport(local, peers); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var gate = new object(); + var allStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseExchanges = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var started = 0; + var observedMax = 0; + transport.ExchangeAntiEntropyHandler = async (target, request, _) => + { + lock (gate) + { + inFlight++; + started++; + observedMax = Math.Max(observedMax, inFlight); + if (started == peers.Length) + { + allStarted.TrySetResult(true); + } + } + + try + { + await releaseExchanges.Task; + return new DisseminationAntiEntropyResponse { Sender = target }; + } + finally + { + lock (gate) + { + inFlight--; + } + } + }; + + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyPeerCount = peers.Length; + }); + + var exchangeTask = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + try + { + await allStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + lock (gate) + { + Assert.Equal(peers.Length, observedMax); + } + } + finally + { + releaseExchanges.TrySetResult(true); + } + + await exchangeTask; + + Assert.Equal(peers.Length, transport.AntiEntropyRequests.Count); + } + + [Fact] + public async Task AntiEntropyAppliesCompletedResponsesWhenSlowPeerExceedsRoundLifetime() + { + var local = CreateSilo(11111); + var fastPeer = CreateSilo(11112); + var slowPeer = CreateSilo(11113); + var transport = new FakeTransport(local, fastPeer, slowPeer); + var timeProvider = new FakeTimeProvider(); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + ns.ApplyObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + var fastResponseReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowExchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowCancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timeoutMetric = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name == DisseminationInstruments.AntiEntropyFailuresName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + if (tags.ToArray().Any(static tag => tag.Key == "reason" && object.Equals(tag.Value, "timeout"))) + { + timeoutMetric.TrySetResult(measurement); + } + }); + listener.Start(); + var repair = ns.CreateItem(fastPeer, FakeNamespace.DefaultKey, sequence: 1); + transport.ExchangeAntiEntropyHandler = async (target, _, cancellationToken) => + { + if (target.Equals(slowPeer)) + { + slowExchangeStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + slowCancellationObserved.TrySetResult(); + } + } + + fastResponseReturned.TrySetResult(); + return new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repair), + }; + }; + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyPeerCount = 2, + timeProvider); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(0, ns.GetVersion(FakeNamespace.DefaultKey)); + + timeProvider.Advance(ns.Options.StaleItemTtl); + await slowCancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await ns.ApplyObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal( + 1, + await timeoutMetric.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task AntiEntropyAppliesStaggeredFullResponsesAfterExchangesComplete() + { + var local = CreateSilo(11111); + var fastPeer = CreateSilo(11112); + var slowerPeer = CreateSilo(11113); + var transport = new FakeTransport(local, fastPeer, slowerPeer); + var ns = new FakeNamespace(local); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var fastResponseReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var slowExchangeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSlowResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = async (target, request, cancellationToken) => + { + Assert.Equal(1, Assert.Single(request.Digests[ns.Name]).Version); + var responseVersion = target.Equals(fastPeer) ? 2 : 3; + if (target.Equals(slowerPeer)) + { + slowExchangeStarted.TrySetResult(); + await releaseSlowResponse.Task.WaitAsync(cancellationToken); + } + else + { + fastResponseReturned.TrySetResult(); + } + + return new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(ns.CreateItem( + target, + FakeNamespace.DefaultKey, + sequence: responseVersion)), + }; + }; + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyPeerCount = 2); + + var round = protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await Task.WhenAll(fastResponseReturned.Task, slowExchangeStarted.Task) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.GetVersion(FakeNamespace.DefaultKey)); + + releaseSlowResponse.TrySetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); + } + + [Fact] + public async Task AntiEntropyAppliesReturnedRepairItemsWithoutForwarding() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + var repairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 7); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repairItem), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(7, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Empty(transport.BroadcastBatches); + Assert.Single(transport.AntiEntropyRequests); + var digestByNamespace = Assert.Single(transport.AntiEntropyRequests[0].Request.Digests); + Assert.Equal(ns.Name, digestByNamespace.Key); + var digest = Assert.Single(digestByNamespace.Value); + Assert.Equal(FakeNamespace.DefaultKey, digest.Key); + Assert.Equal(0, digest.Version); + } + + [Fact] + public async Task AntiEntropyRejectsNonFullRepairAndAppliesLaterFullValue() + { + var (local, peer, transport, _) = CreatePeerFixture(); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue(FakeNamespace.DefaultKey, version: 1); + var protocol = CreateProtocol(transport, [ns]); + var second = ns.Inner.CreateItem( + peer, + FakeNamespace.DefaultKey, + sequence: 2, + fromVersion: 1); + var third = ns.Inner.CreateItem( + peer, + FakeNamespace.DefaultKey, + sequence: 3); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(second, third), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(1, ns.Inner.ApplyCounts[FakeNamespace.DefaultKey]); + Assert.Equal(new long[] { 3 }, ns.Attempts.Select(static value => value.ToVersion)); + Assert.Empty(transport.BroadcastBatches); + } + + [Theory] + [InlineData(2, 3, 2)] + [InlineData(3, 2, 1)] + [InlineData(2, 2, 1)] + public async Task AntiEntropyAppliesFullResponsesWithoutRegressingState( + long firstVersion, long secondVersion, int expectedApplyCount) + { + var local = CreateSilo(11111); + var peers = new[] { CreateSilo(11112), CreateSilo(11113) }; + var transport = new FakeTransport(local, peers); + var ns = new ProtocolReviewNamespace(local); + ns.Inner.SetValue(FakeNamespace.DefaultKey, version: 1); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + Assert.Equal(1, Assert.Single(request.Digests[ns.Name]).Version); + var responseVersion = Interlocked.Increment(ref exchangeCount) == 1 ? firstVersion : secondVersion; + var repair = ns.Inner.CreateItem( + target, + FakeNamespace.DefaultKey, + sequence: responseVersion); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(repair), + }); + }; + + var protocol = CreateProtocol( + transport, + [ns], + options => options.Overlay.AntiEntropyPeerCount = peers.Length); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(Math.Max(firstVersion, secondVersion), ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(peers.Length, exchangeCount); + Assert.Equal(expectedApplyCount, ns.Inner.ApplyCounts[FakeNamespace.DefaultKey]); + long[] expectedAttempts = secondVersion < firstVersion ? [firstVersion] : [firstVersion, secondVersion]; + Assert.Equal(expectedAttempts, ns.Attempts.Select(static value => value.ToVersion)); + } + + [Fact] + public async Task AntiEntropyTruncationRotatesPastContinuouslyAdvancingKey() + { + var local = CreateSilo(11111); + var member = CreateSilo(11112); + var requester = CreateSilo(11113); + var transport = new FakeTransport(local, member); + var ns = new FakeNamespace(local); + DisseminationKey hotKey = "hot"; + DisseminationKey waitingKey = "waiting"; + ns.SetValue(hotKey, version: 1); + ns.SetValue(waitingKey, version: 1); + var protocol = CreateProtocol( + transport, + ns, + options => options.MaxBatchItems = 1); + var request = new DisseminationAntiEntropyRequest + { + Sender = requester, + Digests = CreateAntiEntropyRequestDigest( + ns.Name, + (hotKey, 0), + (waitingKey, 0)), + }; + + var first = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = member, + Digests = request.Digests, + }, TestContext.Current.CancellationToken); + ns.SetValue(hotKey, version: 2); + var second = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + + Assert.True(first.Truncated); + Assert.True(second.Truncated); + Assert.Equal(hotKey, Assert.Single(GetAntiEntropyResponseValues(first)).Value.Key); + Assert.Equal(waitingKey, Assert.Single(GetAntiEntropyResponseValues(second)).Value.Key); + } + + [Fact] + public async Task AntiEntropyAppliesValidItemsAfterFailedRepairItem() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + var badRepairItem = new DisseminationBroadcastValue + { + Value = new DisseminationValue("bad", fromVersion: 0, toVersion: 1, Array.Empty()), + TimeToLive = TimeSpan.FromMinutes(1), + }; + var goodRepairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 3); + transport.ExchangeAntiEntropyHandler = (target, request, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = CreateValueGroups(badRepairItem, goodRepairItem), + }); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task AntiEntropyAppliesValidItemsAfterFailedRepairRound() + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var protocol = CreateProtocol(transport, ns); + ns.SetValue(FakeNamespace.DefaultKey, version: 1); + + var badRepairItem = new DisseminationBroadcastValue + { + Value = new DisseminationValue(FakeNamespace.DefaultKey, fromVersion: 1, toVersion: 2, Array.Empty()), + TimeToLive = TimeSpan.FromMinutes(1), + }; + var goodRepairItem = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 3); + var exchangeCount = 0; + transport.ExchangeAntiEntropyHandler = (target, request, _) => + { + var count = Interlocked.Increment(ref exchangeCount); + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = target, + Values = count switch + { + 1 => CreateValueGroups(badRepairItem), + 2 => CreateValueGroups(goodRepairItem), + _ => [], + }, + }); + }; + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(3, ns.GetVersion(FakeNamespace.DefaultKey)); + Assert.Equal(2, Volatile.Read(ref exchangeCount)); + } + +#if NET10_0_OR_GREATER + [Fact] + public async Task MonotonicDisseminationModelConformsToAccordantSpec() + { + var spec = CreateMonotonicDisseminationSpec(); + spec.ExecuteWith() + .BindAsync("Receive", static (harness, version) => harness.Receive(version, TestContext.Current.CancellationToken)) + .BindAsync("RepairPeer", static (harness, peerVersion) => harness.RepairPeer(peerVersion, TestContext.Current.CancellationToken)); + + var receive = spec.GetOperation("Receive"); + var repairPeer = spec.GetOperation("RepairPeer"); + var inputs = new InputSet + { + receive.With(1, "Receive version 1"), + receive.With(2, "Receive version 2"), + receive.With(3, "Receive version 3"), + repairPeer.With(0, "Repair peer with no value"), + repairPeer.With(1, "Repair peer at version 1"), + repairPeer.With(3, "Repair peer at version 3"), + }; + var initialState = new MonotonicDisseminationState(); + var testCases = spec.GenerateTests(initialState, inputs, new TestGenerationOptions { MaxDepth = 4 }); + var harness = new MonotonicDisseminationHarness(CreateSilo(11111)); + var context = spec.CreateTestingContext(); + context.Register(harness); + + var results = await spec.RunTests( + context, + initialState, + testCases, + new TestExecutionOptions + { + BeforeEachAsync = testContext => + { + testContext.Context.Get().Reset(); + return Task.CompletedTask; + }, + }); + + var failures = results.Where(static result => !result.Success).ToArray(); + Assert.Empty(failures); + } +#endif + + [Fact] + public async Task MembershipNamespaceRepairsCurrentFullSnapshotWithoutRetainingPeerBaseline() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var baseSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var updatedSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(baseSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var peerDigest = Assert.Single(sourceNamespace.Digests); + sourceManager.CurrentSnapshot = updatedSnapshot; + var localDigest = Assert.Single(sourceNamespace.Digests); + + var repair = sourceNamespace.CreateRepair(new DisseminationRepairRequest( + localDigest.Key, + peerDigest.Version, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024)); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var value = repair.Value; + Assert.Equal(0, value.FromVersion); + Assert.Equal(localDigest.Version, value.ToVersion); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + AssertMembershipState(updatedSnapshot, Assert.IsType(update.Snapshot)); + var receiverManager = new FakeMembershipManager(baseSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + var result = await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal(updatedSnapshot.Version, receiverManager.CurrentSnapshot.Version); + Assert.True(receiverManager.CurrentSnapshot.Entries.ContainsKey(peer)); + } + + [Fact] + public void MembershipNamespaceInvalidatesCachedPayloadForSameVersionUpdate() + { + var local = CreateSilo(11111); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var manager = new FakeMembershipManager(firstSnapshot); + var disseminationNamespace = CreateMembershipNamespace(manager, serializer); + var request = new DisseminationRepairRequest( + DisseminationKey.Default, + fromVersion: null, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024); + + var firstRepair = disseminationNamespace.CreateRepair(request); + manager.CurrentSnapshot = updatedSnapshot; + var updatedRepair = disseminationNamespace.CreateRepair(request); + + Assert.Equal(DisseminationRepairStatus.Produced, firstRepair.Status); + Assert.Equal(DisseminationRepairStatus.Produced, updatedRepair.Status); + var firstValue = firstRepair.Value; + var updatedValue = updatedRepair.Value; + var firstUpdate = Assert.IsType( + serializer.Deserialize(firstValue.Payload)); + var updatedUpdate = Assert.IsType( + serializer.Deserialize(updatedValue.Payload)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(1), + firstUpdate.Snapshot!.Entries[local].IAmAliveTime); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + updatedUpdate.Snapshot!.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipNamespaceRetainsPostApplySnapshotForRepair() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var currentSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var incomingSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch)); + var appliedSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch)); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var manager = new FakeMembershipManager(currentSnapshot); + manager.ProcessGossipSnapshotHandler = (_, _) => + { + manager.CurrentSnapshot = appliedSnapshot; + return Task.FromResult(true); + }; + var disseminationNamespace = CreateMembershipNamespace(manager, serializer); + var value = new DisseminationValue( + DisseminationKey.Default, + fromVersion: 0, + toVersion: incomingSnapshot.Version.Value, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = incomingSnapshot })); + + var result = await disseminationNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken); + var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, + fromVersion: null, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: 1024 * 1024)); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal(DisseminationRepairStatus.Produced, repair.Status); + var repairedValue = repair.Value; + var repairedUpdate = Assert.IsType( + serializer.Deserialize(repairedValue.Payload)); + var repairedSnapshot = Assert.IsType(repairedUpdate.Snapshot); + Assert.Equal(appliedSnapshot.Version, repairedSnapshot.Version); + Assert.True(repairedSnapshot.Entries.ContainsKey(peer)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(1), + repairedSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipNamespacePublishesSameVersionSuccessorAsFullSnapshot() + { + var local = CreateSilo(11111); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync( + disseminationService, + firstSnapshot, + TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = updatedSnapshot; + Assert.True(await sourceNamespace.PublishAsync( + disseminationService, + updatedSnapshot, + TestContext.Current.CancellationToken)); + + var update = Assert.Single(disseminationService.Values.Skip(1)); + Assert.Equal((0L, 1L), (update.FromVersion, update.ToVersion)); + var receiverManager = new FakeMembershipManager(firstSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + Assert.Equal( + DisseminationApplyResult.Applied, + await receiverNamespace.ApplyValueAsync(update, TestContext.Current.CancellationToken)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipAntiEntropyRepairsSameVersionSuccessor() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(1))); + var updatedSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceNamespace = CreateMembershipNamespace( + new FakeMembershipManager(updatedSnapshot), + serializer); + var receiverManager = new FakeMembershipManager(firstSnapshot); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + var peerDigest = Assert.Single(receiverNamespace.Digests); + var transport = new FakeTransport(local, peer); + var protocol = CreateProtocol( + transport, + new IDisseminationNamespace[] { sourceNamespace }); + + var response = await protocol.ReceiveAntiEntropy(new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = new() + { + [sourceNamespace.Name] = [peerDigest], + }, + }, TestContext.Current.CancellationToken); + + var value = Assert.Single(GetAntiEntropyResponseValues(response)).Value; + Assert.Equal((0L, 1L), (value.FromVersion, value.ToVersion)); + Assert.Equal( + DisseminationApplyResult.Applied, + await receiverNamespace.ApplyValueAsync(value, TestContext.Current.CancellationToken)); + Assert.Equal( + DateTime.UnixEpoch.AddSeconds(2), + receiverManager.CurrentSnapshot.Entries[local].IAmAliveTime); + } + + [Fact] + public async Task MembershipPublicationValidationProducesIndependentlyApplicableFullRepairs() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var thirdSnapshot = CreateMembershipSnapshot( + version: 3, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry( + peer, + SiloStatus.Active, + DateTime.UnixEpoch.AddSeconds(1), + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(3))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = thirdSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, thirdSnapshot, TestContext.Current.CancellationToken)); + var (first, second, third) = ( + disseminationService.Values[0], + disseminationService.Values[1], + disseminationService.Values[2]); + + Assert.Equal((0L, 1L), (first.FromVersion, first.ToVersion)); + Assert.Equal((0L, 2L), (second.FromVersion, second.ToVersion)); + Assert.Equal((0L, 3L), (third.FromVersion, third.ToVersion)); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(first.Payload)).Snapshot); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(second.Payload)).Snapshot); + Assert.NotNull(Assert.IsType( + serializer.Deserialize(third.Payload)).Snapshot); + + var receiverManager = new FakeMembershipManager(CreateMembershipSnapshot(MembershipVersion.MinValue.Value)); + var receiverNamespace = CreateMembershipNamespace(receiverManager, serializer); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(first, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(second, TestContext.Current.CancellationToken)); + Assert.Equal(DisseminationApplyResult.Applied, await receiverNamespace.ApplyValueAsync(third, TestContext.Current.CancellationToken)); + AssertMembershipState(thirdSnapshot, receiverManager.CurrentSnapshot); + + var lateReceiver = new FakeMembershipManager(CreateMembershipSnapshot(MembershipVersion.MinValue.Value)); + var lateNamespace = CreateMembershipNamespace(lateReceiver, serializer); + Assert.Equal(DisseminationApplyResult.Applied, await lateNamespace.ApplyValueAsync(third, TestContext.Current.CancellationToken)); + AssertMembershipState(thirdSnapshot, lateReceiver.CurrentSnapshot); + } + + [Theory] + [InlineData(false, 2)] + [InlineData(true, 3)] + public async Task MembershipPublicationValidationUsesCurrentSnapshotWhenNotificationIsSuperseded(bool baselineAccepted, long nextVersion) + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: nextVersion, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + disseminationService.Results.Enqueue(baselineAccepted); + disseminationService.Results.Enqueue(true); + + Assert.Equal(baselineAccepted, await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + + var value = disseminationService.Values[1]; + Assert.Equal((0L, nextVersion), (value.FromVersion, value.ToVersion)); + var update = Assert.IsType( + serializer.Deserialize(value.Payload)); + AssertMembershipState(secondSnapshot, Assert.IsType(update.Snapshot)); + } + + [Fact] + public async Task MembershipPublicationValidationIncludesFullInventoryWhenTopologyChanges() + { + var local = CreateSilo(11111); + var firstPeer = CreateSilo(11112); + var secondPeer = CreateSilo(11113); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(firstPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(firstPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(secondPeer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(2))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + + Assert.True(await sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken)); + sourceManager.CurrentSnapshot = secondSnapshot; + Assert.True(await sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken)); + + Assert.Equal(0, disseminationService.Values[1].FromVersion); + var update = Assert.IsType( + serializer.Deserialize(disseminationService.Values[1].Payload)); + AssertMembershipState(secondSnapshot, Assert.IsType(update.Snapshot)); + } + + [Fact] + public async Task MembershipNamespaceAllowsConcurrentPublicationNotifications() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var firstSnapshot = CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + var secondSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry( + local, + SiloStatus.Active, + DateTime.UnixEpoch, + iAmAliveTime: DateTime.UnixEpoch.AddSeconds(2)), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var sourceManager = new FakeMembershipManager(firstSnapshot); + var sourceNamespace = CreateMembershipNamespace(sourceManager, serializer); + var disseminationService = new FakeDisseminationService(); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var callCount = 0; + disseminationService.PublishHandler = async (value, cancellationToken) => + { + if (Interlocked.Increment(ref callCount) == 1) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(cancellationToken); + } + + return true; + }; + + var firstPublish = sourceNamespace.PublishAsync(disseminationService, firstSnapshot, TestContext.Current.CancellationToken).AsTask(); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + sourceManager.CurrentSnapshot = secondSnapshot; + var secondPublish = sourceNamespace.PublishAsync(disseminationService, secondSnapshot, TestContext.Current.CancellationToken).AsTask(); + try + { + await Task.Delay(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + Assert.Equal(2, disseminationService.Values.Count); + Assert.True(secondPublish.IsCompletedSuccessfully); + } + finally + { + releaseFirst.TrySetResult(); + } + + Assert.True(await firstPublish); + Assert.True(await secondPublish); + Assert.Equal(new long[] { 1, 2 }, disseminationService.Values.Select(static value => value.ToVersion).Order()); + } + + [Fact] + public void DisseminationKeyRoundTripsDefaultAndSiloAddressKeys() + { + var earlierPeer = CreateSilo(11111); + var peer = CreateSilo(11112); + using var serviceProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = serviceProvider.GetRequiredService(); + var membershipValue = new DisseminationValue(DisseminationKey.Default, fromVersion: 0, toVersion: 7, Array.Empty()); + var deploymentLoadValue = new DisseminationValue(peer, fromVersion: 0, toVersion: 11, Array.Empty()); + + var roundTripMembershipValue = serializer.Deserialize(serializer.SerializeToArray(membershipValue)); + var roundTripDeploymentLoadValue = serializer.Deserialize(serializer.SerializeToArray(deploymentLoadValue)); + + Assert.Equal(DisseminationKey.Default, roundTripMembershipValue.Key); + Assert.Equal(new DisseminationKey(peer), roundTripDeploymentLoadValue.Key); + + Span expected = stackalloc char[128]; + Span actual = stackalloc char[128]; + Assert.True(((ISpanFormattable)peer).TryFormat(expected, out var expectedLength, "H", null)); + Assert.True(roundTripDeploymentLoadValue.Key.TryFormat(actual, out var actualLength, "H", null)); + Assert.Equal(expected[..expectedLength].ToString(), actual[..actualLength].ToString()); + Assert.True(roundTripMembershipValue.Key.TryFormat(Span.Empty, out var nullKeyLength, default, null)); + Assert.Equal(0, nullKeyLength); + Assert.Equal( + Math.Sign(earlierPeer.CompareTo(peer)), + Math.Sign(new DisseminationKey(earlierPeer).CompareTo(new DisseminationKey(peer)))); + } + + [Fact] + public void DisseminationNamespaceWrapsNonEmptyStringsForDictionaryKeys() + { + DisseminationNamespace namespaceName = "load"; + var namespaces = new Dictionary + { + [namespaceName] = 1, + }; + + Assert.Equal("load", (string)namespaceName); + Assert.True(namespaces.TryGetValue(new DisseminationNamespace("load"), out var value)); + Assert.Equal(1, value); + Assert.True(namespaceName.CompareTo(new DisseminationNamespace("membership")) < 0); + Assert.ThrowsAny(() => new DisseminationNamespace(string.Empty)); + } + + [Fact] + public void DisseminationNamespaceDefaultValueSupportsKeyOperations() + { + var namespaceName = default(DisseminationNamespace); + var namespaces = new Dictionary { [namespaceName] = 1 }; + + Assert.True(namespaces.TryGetValue(default, out var value)); + Assert.Equal(1, value); + Assert.Equal(0, namespaceName.GetHashCode()); + Assert.Equal(0, namespaceName.CompareTo(default)); + Assert.True(namespaceName.CompareTo(new DisseminationNamespace("membership")) < 0); + Assert.Equal(string.Empty, namespaceName.ToString()); + Assert.True(namespaceName.TryFormat([], out var charsWritten, default, null)); + Assert.Equal(0, charsWritten); + Assert.Throws(() => namespaceName.Value); + } + + [Fact] + public void OptionsValidatorRejectsInvalidFanoutBounds() + { + var options = new DisseminationOptions(); + options.Overlay.MinFanOutFactor = 8; + options.Overlay.MaxFanOutFactor = 4; + var result = new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Fact] + public void NamespaceOptionsValidatorAllowsStaleTtlShorterThanPublicationPeriod() + { + var options = new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(1), + }; + options.Dissemination.StaleItemTtl = TimeSpan.FromMilliseconds(1); + + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.Equal(ValidateOptionsResult.Success, result); + } + + [Fact] + public void NamespaceOptionsValidatorRejectsNonPositiveStaleTtl() + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.StaleItemTtl = TimeSpan.Zero; + + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.Equal( + [ + $"{nameof(DeploymentLoadPublisherOptions)}.{nameof(DeploymentLoadPublisherOptions.Dissemination)}." + + $"{nameof(DisseminationNamespaceOptions.StaleItemTtl)} must be greater than 0.", + ], + result.Failures); + } + + [Fact] + public void ClusterMembershipNamespaceOptionsValidatorIncludesOwningOptionsType() + { + var options = new ClusterMembershipOptions(); + options.Dissemination.MaxPayloadBytes = 0; + + var result = new ClusterMembershipOptionsDisseminationValidator().Validate(Options.DefaultName, options); + + Assert.Equal( + [ + $"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.Dissemination)}." + + $"{nameof(DisseminationNamespaceOptions.MaxPayloadBytes)} must be greater than 0.", + ], + result.Failures); + } + + [Theory] + [InlineData(0)] + [InlineData(4294967295)] + public void OptionsValidatorRejectsUnsupportedAntiEntropyIntervals(long milliseconds) + { + var options = new DisseminationOptions(); + options.Overlay.AntiEntropyInterval = TimeSpan.FromMilliseconds(milliseconds); + var result = new DisseminationOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Theory] + [InlineData(1, true)] + [InlineData(0, false)] + public async Task ReceiveBroadcastUsesOnlyPositiveRelativeLifetime(int milliseconds, bool skewClock) + { + var (_, peer, transport, ns) = CreatePeerFixture(); + var timeProvider = skewClock + ? new FakeTimeProvider(DateTimeOffset.MaxValue - TimeSpan.FromSeconds(1)) + : TimeProvider.System; + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + var item = ns.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1); + item = new DisseminationBroadcastValue + { + Value = item.Value, + TimeToLive = TimeSpan.FromMilliseconds(milliseconds), + }; + + await protocol.ReceiveBroadcast(CreateBroadcastBatch(peer, item), TestContext.Current.CancellationToken); + + Assert.Equal(milliseconds > 0 ? 1 : 0, ns.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task BroadcastTransportCancelsWhenRelativeLifetimeExpires() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancellationObserved = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, _, cancellationToken) => + { + sendStarted.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + finally + { + cancellationObserved.TrySetResult(); + } + + return new DisseminationBroadcastResponse(); + }; + + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var flush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + timeProvider.Advance(ns.Options.StaleItemTtl); + await cancellationObserved.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + // Complete the failed attempt's requeue before shutdown wakes the disabled namespace. + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.Options.Enabled = false; + await protocol.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + [Fact] + public void OptionsValidatorRejectsInvalidExpectedUpdateCadence() + { + var options = new DeploymentLoadPublisherOptions(); + options.Dissemination.ExpectedUpdateCadence = TimeSpan.Zero; + var result = new DeploymentLoadPublisherOptionsValidator().Validate(Options.DefaultName, options); + + Assert.True(result.Failed); + } + + [Fact] + public void NamespaceOptionsUseExpectedUpdateCadenceDefaults() + { + Assert.Equal(TimeSpan.FromSeconds(5), new DeploymentLoadPublisherOptions().Dissemination.ExpectedUpdateCadence); + Assert.Equal(8192, new DeploymentLoadPublisherOptions().Dissemination.MaxPendingItemCount); + Assert.Equal(TimeSpan.FromSeconds(10), new ClusterMembershipOptions().Dissemination.ExpectedUpdateCadence); + } + + [Fact] + public void DisseminationOptionsUseBatchDefaults() + { + var options = new DisseminationOptions(); + + Assert.False(options.Enabled); + Assert.False(new DisseminationNamespaceOptions().Enabled); + Assert.False(new DeploymentLoadPublisherOptions().Dissemination.Enabled); + Assert.False(new ClusterMembershipOptions().Dissemination.Enabled); + Assert.Equal(1024 * 1024, options.MaxBatchBytes); + Assert.Equal(8 * 1024, options.MaxBatchItems); + Assert.Equal(options.MaxBatchBytes, new DisseminationNamespaceOptions().MaxPayloadBytes); + } + + [Fact] + public void DisseminationMembershipReturnsCachedSnapshotForSameMembershipVersion() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Joining, DateTime.UnixEpoch.AddSeconds(1)))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + + var first = membership.CurrentSnapshot; + var second = membership.CurrentSnapshot; + + Assert.Same(first, second); + Assert.Equal(new MembershipVersion(1), first.MembershipVersion); + Assert.Equal(new[] { local, peer }, first.Members); + } + + [Fact] + public void DisseminationMembershipRecomputesSnapshotWhenMembershipVersionChanges() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + var first = membership.CurrentSnapshot; + + membershipManager.CurrentSnapshot = CreateMembershipSnapshot( + version: 2, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(peer, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1))); + + var second = membership.CurrentSnapshot; + + Assert.NotSame(first, second); + Assert.Equal(new MembershipVersion(2), second.MembershipVersion); + Assert.Equal(new[] { local, peer }, second.Members); + } + + [Fact] + public async Task DisseminationMembershipRefreshDelegatesToMembershipManager() + { + var local = CreateSilo(11111); + var membershipManager = new FakeMembershipManager(CreateMembershipSnapshot( + version: 1, + CreateMembershipEntry(local, SiloStatus.Active, DateTime.UnixEpoch))); + var membership = new DisseminationMembership( + membershipManager, + new FakeLocalSiloDetails(local), + Options.Create(new DisseminationOptions())); + + await membership.RefreshMembership(TestContext.Current.CancellationToken); + + Assert.Equal(1, membershipManager.RefreshCallCount); + Assert.Null(Assert.Single(membershipManager.RefreshTargetVersions)); + Assert.False(Assert.Single(membershipManager.RefreshFreshnessRequirements)); + } + + [Fact] + public void DisseminationMembershipSnapshotRejectsDuplicateMembers() + { + var local = CreateSilo(11111); + + var exception = Assert.Throws(() => new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [local, local], + new DisseminationOverlayOptions())); + + Assert.Equal("members", exception.ParamName); + } + + [Fact] + public void DisseminationMembershipSnapshotUsesStoredTreeOrderForOriginatorTargets() + { + var root = CreateSilo(11113); + var first = CreateSilo(11115); + var second = CreateSilo(11112); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + root, + [root, first, second], + CreateOverlayOptions(fanout: 2)); + + var targets = snapshot.OriginatorTreeTargets; + + Assert.Equal(new[] { first, second }, targets); + } + + [Fact] + public void DisseminationMembershipSnapshotUsesStoredTreeOrderForForwardingTargets() + { + var local = CreateSilo(11111); + var root = CreateSilo(11112); + var sender = CreateSilo(11113); + var child = CreateSilo(11114); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [local, root, sender, child], + CreateOverlayOptions(fanout: 2)); + + var targets = snapshot.ForwardingTreeTargets; + + Assert.Equal(new[] { sender, child }, targets); + } + + [Fact] + public void DisseminationMembershipSnapshotDoesNotAddLocalSiloWhenMissing() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [peer], + CreateOverlayOptions(fanout: 1)); + + var targets = snapshot.OriginatorTreeTargets; + var antiEntropyPeers = snapshot.SelectAntiEntropyPeers(1); + + Assert.False(snapshot.ContainsMember(local)); + Assert.Single(snapshot.Members); + Assert.Empty(targets); + Assert.Empty(antiEntropyPeers); + } + + [Fact] + public void DisseminationMembershipSnapshotReturnsAllAntiEntropyCandidatesWhenRequestedCountIsLarge() + { + const int peerCount = 20; + const int localIndex = 7; + var silos = CreateSilos(12); + var local = silos[localIndex]; + var expected = silos.Where(silo => !Equals(silo, local)).OrderBy(static silo => silo); + var snapshot = new DisseminationMembershipSnapshot( + new MembershipVersion(1), + local, + [.. silos], + new DisseminationOverlayOptions()); + + var selectedPeers = snapshot.SelectAntiEntropyPeers(peerCount); + + Assert.Equal(silos.Length - 1, selectedPeers.Length); + Assert.Equal(selectedPeers.Length, selectedPeers.Distinct().Count()); + Assert.DoesNotContain(local, selectedPeers); + Assert.Equal(expected, selectedPeers.OrderBy(static silo => silo)); + } + + [Fact] + public async Task DisabledNamespaceBetweenPublishAndReschedule_UsesFiniteBoundedFallbackWithoutInfiniteTimerDueTime() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFailedSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = async (_, _, _) => + { + Interlocked.Increment(ref sendCount); + sendStarted.TrySetResult(); + await releaseFailedSend.Task; + throw new InvalidOperationException("The initial send fails after the namespace is disabled."); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + using var schedule = new BroadcastScheduleObserver(); + var retryScheduled = schedule.WaitAsync( + e => e.Peer.Equals(peer) + && e.Reason == DisseminationBroadcastScheduleReason.Retry + && e.Attempt == 1, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + ns.Options.Enabled = false; + releaseFailedSend.TrySetResult(); + + var retry = await retryScheduled; + Assert.Equal(TimeSpan.FromMilliseconds(100), retry.DueTime); + Assert.DoesNotContain(TimeSpan.MaxValue, timeProvider.TimerDueTimes); + + timeProvider.Advance(retry.DueTime); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await protocol.StopAsync(TestContext.Current.CancellationToken); + + Assert.Equal(1, sendCount); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task UnexpectedPeerPumpIterationFailure_IsRetriedAndCompletesFlushAndStopDrainWaiters() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var logger = new RecordingLogger(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retrySendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseRetrySend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastResponseHandler = async (_, batch, _) => + { + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task; + return new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, version: 0), + ], + }, + }; + } + + retrySendStarted.TrySetResult(); + await releaseRetrySend.Task; + transport.BroadcastBatches.Add((peer, batch)); + return FakeTransport.CreateAcknowledgment(batch); + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocolWithBroadcastLogger( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider, + logger); + using var schedule = new BroadcastScheduleObserver(); + var recoveredRetryScheduled = schedule.WaitAsync( + e => e.Peer.Equals(peer) + && e.Reason == DisseminationBroadcastScheduleReason.Retry + && e.Attempt == 2, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.False(flushTask.IsCompleted); + timeProvider.ThrowOnNextTimerChange(); + releaseFirstSend.TrySetResult(); + + await flushTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var retry = await recoveredRetryScheduled; + Assert.Equal(TimeSpan.FromMilliseconds(200), retry.DueTime); + Assert.Equal(1, logger.WarningCount); + Assert.DoesNotContain(TimeSpan.MaxValue, timeProvider.TimerDueTimes); + + timeProvider.Advance(retry.DueTime); + await retrySendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var stopTask = protocol.StopAsync(TestContext.Current.CancellationToken); + Assert.False(stopTask.IsCompleted); + releaseRetrySend.TrySetResult(); + + await stopTask.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, sendCount); + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(1, Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion); + } + + [Fact] + public async Task PermanentPeerPumpRecoveryFailure_FaultsFlushAndStopDrainWaiters() + { + var local = CreateSilo(11111); + var peer = CreateSilo(11112); + var transport = new FakeTransport(local, peer); + var timeProvider = new RecordingFakeTimeProvider(); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = async (_, _, _) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task; + return new DisseminationBroadcastResponse + { + Acknowledgments = new() + { + [FakeNamespace.DefaultName] = + [ + new DigestEntry(FakeNamespace.DefaultKey, version: 0), + ], + }, + }; + }; + + var ns = new FakeNamespace(local); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(4), + timeProvider); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + var flushTask = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + timeProvider.ThrowOnNextTimerChanges(2); + releaseFirstSend.TrySetResult(); + + var flushException = await Assert.ThrowsAsync(() => flushTask); + Assert.Contains("could not recover", flushException.Message, StringComparison.Ordinal); + + var stopException = await Assert.ThrowsAsync( + () => protocol.StopAsync(TestContext.Current.CancellationToken)); + Assert.Same(flushException, stopException); + } + + private static T BeforeBroadcastPumpsRun(Func action) + { + var context = new DeferredBroadcastContext(); + var previous = SynchronizationContext.Current; + SynchronizationContext.SetSynchronizationContext(context); + try + { + return action(); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previous); + context.Resume(); + } + } + + private sealed class DeferredBroadcastContext : SynchronizationContext + { + private readonly List<(SendOrPostCallback Callback, object? State)> _pending = []; + private bool _resumed; + + public override void Post(SendOrPostCallback callback, object? state) + { + lock (_pending) + { + if (!_resumed) + { + _pending.Add((callback, state)); + return; + } + } + + ThreadPool.QueueUserWorkItem(static item => item.Callback(item.State), (Callback: callback, State: state), preferLocal: false); + } + + public void Resume() + { + lock (_pending) + { + _resumed = true; + foreach (var item in _pending) + { + ThreadPool.QueueUserWorkItem(static work => work.Callback(work.State), item, preferLocal: false); + } + + _pending.Clear(); + } + } + } + + private static (SiloAddress Local, SiloAddress Peer, FakeTransport Transport, FakeNamespace Namespace) CreatePeerFixture( + int localPort = 11111, int peerPort = 11112) + { + var local = CreateSilo(localPort); + var peer = CreateSilo(peerPort); + return (local, peer, new FakeTransport(local, peer), new FakeNamespace(local)); + } + + private static DisseminationProtocol CreateProtocol( + FakeTransport transport, + FakeNamespace ns, + Action? configure = null, + TimeProvider? timeProvider = null) => + CreateProtocol(transport, new IDisseminationNamespace[] { ns }, configure, timeProvider); + + private static ValueTask PublishValue( + DisseminationProtocol protocol, + FakeNamespace disseminationNamespace, + DisseminationValue value, + CancellationToken cancellationToken) + { + disseminationNamespace.PublishValue(value); + return protocol.Publish( + disseminationNamespace, + value.Key, + value.ToVersion, + cancellationToken); + } + + private static DisseminationProtocol CreateProtocol( + FakeTransport transport, + IReadOnlyList namespaces, + Action? configure = null, + TimeProvider? timeProvider = null, + Microsoft.Extensions.Logging.ILogger? logger = null, + Microsoft.Extensions.Logging.ILogger? broadcastLogger = null) + { + var options = new DisseminationOptions { Enabled = true }; + configure?.Invoke(options); + var localSiloDetails = new FakeLocalSiloDetails(transport.LocalSilo); + return new DisseminationProtocol( + localSiloDetails, + transport.GrainFactory, + new DisseminationMembership( + transport.MembershipManager, + localSiloDetails, + Options.Create(options)), + new TestOptionsMonitor(options), + namespaces, + timeProvider ?? TimeProvider.System, + logger ?? NullLogger.Instance, + broadcastLogger ?? NullLogger.Instance); + } + + private static DisseminationBroadcastQueue CreateBroadcastQueue( + FakeTransport transport, + IReadOnlyList namespaces, + Action? configure = null, + TimeProvider? timeProvider = null) + { + var options = new DisseminationOptions { Enabled = true }; + configure?.Invoke(options); + return new DisseminationBroadcastQueue( + timeProvider ?? TimeProvider.System, + transport.LocalSilo, + transport.GrainFactory, + new TestOptionsMonitor(options), + namespaces, + NullLogger.Instance); + } + + private static void UpdateMaximum(ref int maximum, int candidate) + { + var current = Volatile.Read(ref maximum); + while (candidate > current) + { + var observed = Interlocked.CompareExchange(ref maximum, candidate, current); + if (observed == current) + { + return; + } + + current = observed; + } + } + + private static DisseminationProtocol CreateProtocolWithBroadcastLogger( + FakeTransport transport, + FakeNamespace ns, + Action configure, + TimeProvider timeProvider, + Microsoft.Extensions.Logging.ILogger broadcastLogger) => + CreateProtocol(transport, [ns], configure, timeProvider, broadcastLogger: broadcastLogger); + + private static DisseminationOverlayOptions CreateOverlayOptions(int fanout) => new() + { + FanOutFactor = _ => fanout, + }; + + private static SiloAddress CreateSilo(int port) => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, port), port); + + private static SiloAddress[] CreateSilos(int count) => + Enumerable.Range(11111, count).Select(CreateSilo).OrderBy(static silo => silo).ToArray(); + + private static Dictionary> CreateAntiEntropyRequestDigest( + DisseminationNamespace namespaceName, + params (DisseminationKey Key, long Version)[] versions) => + new() + { + [namespaceName] = versions + .Select(static entry => new DigestEntry(entry.Key, entry.Version)) + .ToList(), + }; + + private static DisseminationBroadcastBatch CreateBroadcastBatch(SiloAddress sender, params DisseminationBroadcastValue[] values) => new() + { + Sender = sender, + Values = CreateBroadcastValueGroups(values), + }; + + private static IEnumerable GetBroadcastValues(DisseminationBroadcastBatch batch) => + batch.Values.Values.SelectMany(static values => values); + + private static SiloAddress[] GetSentBroadcastPeers(FakeTransport transport) + { + lock (transport.BroadcastBatches) + { + return [.. transport.BroadcastBatches.Select(static batch => batch.Peer)]; + } + } + + private static void ClearBroadcastBatches(FakeTransport transport) + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Clear(); + } + } + + private static IEnumerable GetAntiEntropyResponseValues(DisseminationAntiEntropyResponse response) => + response.Values.Values.SelectMany(static values => values); + + private static Dictionary> CreateBroadcastValueGroups(params DisseminationBroadcastValue[] values) => + new() + { + [FakeNamespace.DefaultName] = [.. values], + }; + + private static Dictionary> CreateValueGroups(params DisseminationBroadcastValue[] values) => + CreateValueGroups(FakeNamespace.DefaultName, values); + + private static Dictionary> CreateValueGroups(DisseminationNamespace namespaceName, params DisseminationBroadcastValue[] values) => + new() + { + [namespaceName] = [.. values], + }; + + private static MembershipDisseminationNamespace CreateMembershipNamespace( + FakeMembershipManager membershipManager, + Serializer serializer) + { + var options = new ClusterMembershipOptions(); + options.Dissemination.Enabled = true; + return new( + membershipManager, + new TestOptionsMonitor(options), + serializer); + } + + private static DisseminationBroadcastValue CreateDisseminationValue(SiloAddress originator, DisseminationValue value) + { + _ = originator; + return new() + { + Value = value, + TimeToLive = TimeSpan.FromMinutes(1), + }; + } + + private static MembershipTableSnapshot CreateMembershipSnapshot(long version, params MembershipEntry[] entries) => + new(new MembershipVersion(version), entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + + private static MembershipEntry CreateMembershipEntry( + SiloAddress silo, + SiloStatus status, + DateTime startTime, + DateTime? iAmAliveTime = null) => new() + { + SiloAddress = silo, + Status = status, + ProxyPort = silo.Endpoint.Port, + HostName = "localhost", + SiloName = silo.ToParsableString(), + RoleName = "test", + StartTime = startTime, + IAmAliveTime = iAmAliveTime ?? startTime, + }; + + private static IReadOnlyList GetOriginatorTreeTargets( + SiloAddress root, + IEnumerable peers, + int fanout) + { + var participants = GetSortedParticipants(root, root, peers); + var rootIndex = participants.FindIndex(silo => Equals(silo, root)); + var result = new List(); + AddTopLevelTargets(participants, fanout, root, result); + AddFixedChildren(participants, rootIndex, fanout, root, except: null, result); + return result; + } + + private static IReadOnlyList GetForwardingTreeTargets( + SiloAddress local, + SiloAddress root, + IEnumerable peers, + int fanout, + SiloAddress sender) + { + var participants = GetSortedParticipants(local, root, peers); + var localIndex = participants.FindIndex(silo => Equals(silo, local)); + var result = new List(); + AddFixedChildren(participants, localIndex, fanout, root, sender, result); + return result; + } + + private static List GetSortedParticipants( + SiloAddress local, + SiloAddress root, + IEnumerable peers) => + peers + .Append(local) + .Append(root) + .Distinct() + .OrderBy(static silo => silo) + .ToList(); + + private static void AddTopLevelTargets( + IReadOnlyList participants, + int fanout, + SiloAddress root, + List result) + { + var count = Math.Min(fanout, participants.Count); + for (var i = 0; i < count; i++) + { + AddTarget(participants[i], root, except: null, result); + } + } + + private static void AddFixedChildren( + IReadOnlyList participants, + int index, + int fanout, + SiloAddress root, + SiloAddress? except, + List result) + { + if (index < 0) + { + return; + } + + var firstChild = (long)fanout * (index + 1); + for (var i = 0; i < fanout; i++) + { + var childIndex = firstChild + i; + if (childIndex >= participants.Count) + { + break; + } + + AddTarget(participants[(int)childIndex], root, except, result); + } + } + + private static void AddTarget(SiloAddress peer, SiloAddress root, SiloAddress? except, List result) + { + if (Equals(peer, root) || (except is { } excluded && Equals(peer, excluded)) || result.Contains(peer)) + { + return; + } + + result.Add(peer); + } + + private static HashSet GetReachedParticipants( + SiloAddress root, + IReadOnlyDictionary snapshots) + { + var reached = new HashSet { root }; + var pending = new Queue(snapshots[root].OriginatorTreeTargets); + while (pending.Count > 0) + { + var current = pending.Dequeue(); + if (!reached.Add(current)) + { + continue; + } + + foreach (var child in snapshots[current].ForwardingTreeTargets) + { + pending.Enqueue(child); + } + } + + return reached; + } + +#if NET10_0_OR_GREATER + private static Spec CreateMonotonicDisseminationSpec() + { + var spec = new Spec() + .WithJsonPrinters(); + + spec.Operation("Receive", static (version, state) => + { + if (version < state.Version) + { + return Expect.That( + response => response.Result == ModelApplyResult.Obsolete && response.Version == state.Version, + "Older values are obsolete and do not change state") + .SameState(); + } + + if (version == state.Version && state.Version > 0) + { + return Expect.That( + response => response.Result == ModelApplyResult.Duplicate && response.Version == state.Version, + "Equal versions are duplicates") + .SameState(); + } + + return Expect.That( + response => response.Result == ModelApplyResult.Applied && response.Version == version, + "Newer versions are applied") + .ThenState(nextState => nextState.Version = version); + }); + + spec.Operation("RepairPeer", static (peerVersion, state) => + { + if (state.Version > peerVersion) + { + return Expect.That( + response => response.HasValue && response.Version == state.Version, + "Repair returns the local newer value") + .SameState(); + } + + return Expect.That( + response => !response.HasValue && response.Version == 0, + "Repair returns no value when the peer is current or newer") + .SameState(); + }); + + return spec; + } +#endif + + private sealed class FakeNamespace : IDisseminationNamespace + { + public static readonly DisseminationNamespace DefaultName = new("fake-namespace"); + public static readonly DisseminationKey DefaultKey = new("value"); + private readonly object _lock = new(); + private readonly Dictionary _versions = new(); + private readonly Dictionary _publishedValues = new(); + private readonly DisseminationNamespace _name; + private int _repairRequestCount; + + public FakeNamespace( + SiloAddress localSilo, + DisseminationNamespace? name = null, + DisseminationMembershipScope membershipScope = DisseminationMembershipScope.AllMembers) + { + _ = localSilo; + _name = name ?? DefaultName; + MembershipScope = membershipScope; + } + + public Dictionary ApplyCounts { get; } = new(); + + public HashSet ExpectedKeys { get; } = new(); + + public DisseminationNamespace Name => _name; + + public DisseminationRoutingMode RoutingMode { get; set; } + + public DisseminationMembershipScope MembershipScope { get; set; } = DisseminationMembershipScope.AllMembers; + + public DisseminationNamespaceOptions Options { get; } = new() { Enabled = true }; + + public TimeSpan AggregationPeriod { get; set; } = TimeSpan.FromSeconds(1); + + public int RepairRequestCount => Volatile.Read(ref _repairRequestCount); + + public TaskCompletionSource? ApplyObserved { get; set; } + + public DisseminationValue CreateValue(DisseminationKey key, long sequence, long fromVersion = 0) => new( + key, + fromVersion, + sequence, + BitConverter.GetBytes(sequence)); + + public DisseminationBroadcastValue CreateItem(SiloAddress originator, DisseminationKey key, long sequence, long fromVersion = 0) => + CreateDisseminationValue(originator, CreateValue(key, sequence, fromVersion)); + + public void PublishValue(DisseminationValue value) + { + lock (_lock) + { + if (!_versions.TryGetValue(value.Key, out var version) || value.ToVersion >= version) + { + _publishedValues[value.Key] = value; + _versions[value.Key] = value.ToVersion; + } + } + } + + public void SetValue(DisseminationKey key, long version) => PublishValue(CreateValue(key, version)); + + public void RemoveValue(DisseminationKey key) + { + lock (_lock) + { + _versions.Remove(key); + _publishedValues.Remove(key); + } + } + + public void Clear() + { + lock (_lock) + { + _versions.Clear(); + _publishedValues.Clear(); + } + } + + public (DisseminationKey Key, long Version, long Payload)[] GetState() + { + lock (_lock) + { + return _versions + .OrderBy(static entry => entry.Key.ToString(), StringComparer.Ordinal) + .Select(entry => ( + entry.Key, + entry.Value, + BitConverter.ToInt64(_publishedValues[entry.Key].Payload.Span))) + .ToArray(); + } + } + + public long GetVersion(DisseminationKey key) + { + lock (_lock) + { + return _versions.TryGetValue(key, out var version) ? version : 0; + } + } + + public IEnumerable Digests + { + get + { + KeyValuePair[] versions; + lock (_lock) + { + versions = [.. _versions]; + } + + var present = new HashSet(versions.Length); + foreach (var (key, version) in versions) + { + present.Add(key); + yield return new DigestEntry(key, version); + } + + foreach (var key in ExpectedKeys) + { + if (!present.Contains(key)) + { + yield return new DigestEntry(key, 0); + } + } + } + } + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + Interlocked.Increment(ref _repairRequestCount); + lock (_lock) + { + if (!_versions.TryGetValue(request.Key, out var version)) + { + return DisseminationRepairResult.Unavailable(version: 0); + } + + if (request.FromVersion is { } peerVersion && peerVersion >= version) + { + return DisseminationRepairResult.Current(version); + } + + var value = _publishedValues[request.Key]; + return value.Payload.Length <= request.MaxPayloadBytes && value.Payload.Length <= request.MaxBatchBytes + ? DisseminationRepairResult.Produced(value) + : DisseminationRepairResult.InsufficientCapacity(version); + } + } + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + var version = BitConverter.ToInt64(value.Payload.Span); + if (version != value.ToVersion) + { + return ValueTask.FromResult(DisseminationApplyResult.Rejected); + } + + lock (_lock) + { + if (_versions.TryGetValue(value.Key, out var current)) + { + if (current > version) + { + return ValueTask.FromResult(DisseminationApplyResult.Obsolete); + } + + if (current == version) + { + return ValueTask.FromResult(DisseminationApplyResult.Duplicate); + } + } + + _versions[value.Key] = version; + _publishedValues[value.Key] = CreateValue(value.Key, version); + ApplyCounts[value.Key] = ApplyCounts.TryGetValue(value.Key, out var count) ? count + 1 : 1; + } + + ApplyObserved?.TrySetResult(); + return ValueTask.FromResult(DisseminationApplyResult.Applied); + } + } + + private sealed class FakeDisseminationService : IDisseminationService + { + private readonly Dictionary<(DisseminationNamespace Namespace, DisseminationKey Key), long> _knownVersions = []; + + public List Values { get; } = new(); + + public Queue Results { get; } = new(); + + public Func>? PublishHandler { get; set; } + + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) => + new(await Publish(disseminationNamespace, key, version, cancellationToken), disseminationNamespace.AggregationPeriod); + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + var stream = (disseminationNamespace.Name, key); + var repair = disseminationNamespace.CreateRepair(new DisseminationRepairRequest( + key, + _knownVersions.TryGetValue(stream, out var knownVersion) ? knownVersion : null, + maxBatchBytes: 1024 * 1024, + maxPayloadBytes: disseminationNamespace.Options.MaxPayloadBytes)); + if (repair.Status is not DisseminationRepairStatus.Produced) + { + return false; + } + + Values.Add(repair.Value); + var result = PublishHandler is null + ? Results.Count == 0 || Results.Dequeue() + : await PublishHandler(repair.Value, cancellationToken); + if (result) + { + _knownVersions[stream] = repair.Version; + } + + return result; + } + + public IReadOnlyList GetUnconfirmedPeers( + IDisseminationNamespace disseminationNamespace) => []; + } + + private sealed class FakeTransport + { + private readonly SiloAddress _localSilo; + private readonly List _peers; + private readonly Dictionary _targets = new(); + private readonly object _membershipVersionLock = new(); + private long _membershipFingerprint = long.MinValue; + private long _membershipVersion; + + public FakeTransport(SiloAddress localSilo, params SiloAddress[] peers) + { + _localSilo = localSilo; + _peers = peers.ToList(); + MembershipManager = new FakeMembershipManager(GetMembershipSnapshot, RefreshMembership); + GrainFactory = Substitute.For(); + GrainFactory + .GetSystemTarget(Constants.DisseminationSystemTargetType, Arg.Any()) + .Returns(callInfo => GetSystemTarget(callInfo.ArgAt(1))); + } + + public List<(SiloAddress Peer, DisseminationBroadcastBatch Batch)> BroadcastBatches { get; } = new(); + + public List<(SiloAddress Peer, DisseminationAntiEntropyRequest Request)> AntiEntropyRequests { get; } = new(); + + public List<(SiloAddress Peer, DisseminationPublicationRequest Request)> PublicationRequests { get; } = []; + + public Func> PublishAggregatedHandler { get; set; } = + static (_, _, _) => Task.FromResult(new DisseminationPublicationReceipt(true, TimeSpan.FromSeconds(1))); + + public IInternalGrainFactory GrainFactory { get; } + + public FakeMembershipManager MembershipManager { get; } + + public List Peers => _peers; + + public Dictionary PeerStatuses { get; } = new(); + + public Dictionary StartTimes { get; } = new(); + + public Dictionary TargetResolutionCounts { get; } = new(); + + public Func? SendBroadcastHandler { get; set; } + + public Func>? SendBroadcastResponseHandler { get; set; } + + public Func> ExchangeAntiEntropyHandler { get; set; } = + static (peer, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse { Sender = peer }); + + public Func? RefreshMembershipHandler { get; set; } + + public SiloAddress LocalSilo => _localSilo; + + public int RefreshMembershipCallCount { get; private set; } + + public int GetTargetResolutionCount(SiloAddress peer) + { + lock (_targets) + { + return TargetResolutionCounts.TryGetValue(peer, out var count) ? count : 0; + } + } + + private IDisseminationSystemTarget GetSystemTarget(SiloAddress peer) + { + lock (_targets) + { + TargetResolutionCounts[peer] = TargetResolutionCounts.TryGetValue(peer, out var count) ? count + 1 : 1; + if (!_targets.TryGetValue(peer, out var target)) + { + target = new FakeDisseminationSystemTarget(peer, this); + _targets.Add(peer, target); + } + + return target; + } + } + + private MembershipTableSnapshot GetMembershipSnapshot() + { + var entries = _peers.Append(_localSilo).Distinct().Select(peer => + { + var status = PeerStatuses.TryGetValue(peer, out var peerStatus) ? peerStatus : SiloStatus.Active; + var startTime = StartTimes.TryGetValue(peer, out var value) ? value : DateTime.UnixEpoch; + return CreateMembershipEntry(peer, status, startTime); + }).ToArray(); + + var fingerprint = ComputeMembershipVersion(entries); + long version; + lock (_membershipVersionLock) + { + if (fingerprint != _membershipFingerprint) + { + _membershipFingerprint = fingerprint; + _membershipVersion++; + } + + version = _membershipVersion; + } + + return new MembershipTableSnapshot( + new MembershipVersion(version), + entries.ToImmutableDictionary(static entry => entry.SiloAddress)); + } + + private Task RefreshMembership(CancellationToken cancellationToken) + { + RefreshMembershipCallCount++; + return RefreshMembershipHandler?.Invoke(cancellationToken) ?? Task.CompletedTask; + } + + public async Task SendBroadcast( + SiloAddress peer, + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) + { + if (SendBroadcastResponseHandler is not null) + { + return await SendBroadcastResponseHandler(peer, batch, cancellationToken); + } + + if (SendBroadcastHandler is not null) + { + await SendBroadcastHandler(peer, batch, cancellationToken); + } + else + { + lock (BroadcastBatches) + { + BroadcastBatches.Add((peer, batch)); + } + } + + return CreateAcknowledgment(batch); + } + + public ValueTask ExchangeAntiEntropy( + SiloAddress peer, + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) + { + lock (AntiEntropyRequests) + { + AntiEntropyRequests.Add((peer, request)); + } + + return ExchangeAntiEntropyHandler(peer, request, cancellationToken); + } + + public static DisseminationBroadcastResponse CreateAcknowledgment(DisseminationBroadcastBatch batch) + { + var acknowledgments = new Dictionary>(); + foreach (var (namespaceName, values) in batch.Values) + { + acknowledgments[namespaceName] = values + .GroupBy(static value => value.Value.Key) + .Select(static stream => new DigestEntry( + stream.Key, + stream.Max(static value => value.Value.ToVersion))) + .ToList(); + } + + return new DisseminationBroadcastResponse { Acknowledgments = acknowledgments }; + } + + private static long ComputeMembershipVersion(IEnumerable entries) + { + var result = 17L; + foreach (var entry in entries.OrderBy(static entry => entry.SiloAddress)) + { + result = unchecked((result * 31) + entry.SiloAddress.GetConsistentHashCode()); + result = unchecked((result * 31) + (int)entry.Status); + result = unchecked((result * 31) + entry.StartTime.Ticks); + } + + return result == MembershipVersion.MinValue.Value ? result + 1 : result; + } + } + + private sealed class FakeDisseminationSystemTarget(SiloAddress peer, FakeTransport transport) : IDisseminationSystemTarget + { + public Task PublishAggregated( + DisseminationPublicationRequest request, CancellationToken cancellationToken) + { + lock (transport.PublicationRequests) + { + transport.PublicationRequests.Add((peer, request)); + } + + return transport.PublishAggregatedHandler(peer, request, cancellationToken); + } + + public Task PushBroadcast( + DisseminationBroadcastBatch batch, + CancellationToken cancellationToken) => + transport.SendBroadcast(peer, batch, cancellationToken); + + public async Task ExchangeAntiEntropy( + DisseminationAntiEntropyRequest request, + CancellationToken cancellationToken) => + await transport.ExchangeAntiEntropy(peer, request, cancellationToken); + } + + private sealed class FakeLocalSiloDetails(SiloAddress siloAddress) : ILocalSiloDetails + { + public string Name => "test"; + + public string ClusterId => "test"; + + public string DnsHostName => "localhost"; + + public SiloAddress SiloAddress => siloAddress; + + public SiloAddress GatewayAddress => siloAddress; + } + +#if NET10_0_OR_GREATER + private sealed class MonotonicDisseminationHarness(SiloAddress localSilo) + { + private readonly FakeNamespace _topic = new(localSilo); + + public void Reset() => _topic.Clear(); + + public async Task Receive(long version, CancellationToken cancellationToken) + { + var value = _topic.CreateValue(FakeNamespace.DefaultKey, version); + var result = await _topic.ApplyValueAsync(value, cancellationToken); + return new ModelApplyResponse(ToModelResult(result), _topic.GetVersion(FakeNamespace.DefaultKey)); + } + + public Task RepairPeer(long peerVersion, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var localDigest = _topic.Digests.SingleOrDefault(); + if (localDigest.Version == 0) + { + return Task.FromResult(new ModelRepairResponse(false, 0)); + } + + if (localDigest.Version <= peerVersion) + { + return Task.FromResult(new ModelRepairResponse(false, 0)); + } + + var repair = _topic.CreateRepair(new DisseminationRepairRequest( + FakeNamespace.DefaultKey, + peerVersion, + maxBatchBytes: 1024, + maxPayloadBytes: 1024)); + return Task.FromResult(repair.Status is DisseminationRepairStatus.Produced + ? new ModelRepairResponse(true, repair.Version) + : new ModelRepairResponse(false, 0)); + } + + private static ModelApplyResult ToModelResult(DisseminationApplyResult result) => result switch + { + DisseminationApplyResult.Applied => ModelApplyResult.Applied, + DisseminationApplyResult.Duplicate => ModelApplyResult.Duplicate, + DisseminationApplyResult.Obsolete => ModelApplyResult.Obsolete, + _ => ModelApplyResult.Rejected, + }; + } +#endif + + private sealed class FakeMembershipManager : IMembershipManager + { + private readonly Func? _getCurrentSnapshot; + private readonly Func? _refresh; + private MembershipTableSnapshot _currentSnapshot; + + public FakeMembershipManager(MembershipTableSnapshot currentSnapshot) + { + _currentSnapshot = currentSnapshot; + } + + public FakeMembershipManager( + Func getCurrentSnapshot, + Func refresh) + { + _getCurrentSnapshot = getCurrentSnapshot; + _refresh = refresh; + _currentSnapshot = getCurrentSnapshot(); + } + + public MembershipTableSnapshot CurrentSnapshot + { + get => _getCurrentSnapshot?.Invoke() ?? _currentSnapshot; + set => _currentSnapshot = value; + } + + public int RefreshCallCount { get; private set; } + + public List RefreshTargetVersions { get; } = new(); + + public List RefreshFreshnessRequirements { get; } = new(); + + public Func? ProcessGossipSnapshotHandler { get; set; } + + public IAsyncEnumerable MembershipUpdates => EmptyUpdates(TestContext.Current.CancellationToken); + + public SiloStatus LocalSiloStatus => SiloStatus.Active; + + public Task UpdateLocalStatus(SiloStatus status, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task TryKillSilo(SiloAddress silo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task TrySuspectSilo(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken) => Task.FromResult(false); + + public Task Refresh(MembershipVersion? targetVersion, CancellationToken cancellationToken, bool requireFresh = false) + { + RefreshCallCount++; + RefreshTargetVersions.Add(targetVersion); + RefreshFreshnessRequirements.Add(requireFresh); + return _refresh?.Invoke(cancellationToken) ?? Task.CompletedTask; + } + + public Task ProcessGossipSnapshot(MembershipTableSnapshot snapshot, CancellationToken cancellationToken) + { + if (ProcessGossipSnapshotHandler is { } handler) + { + return handler(snapshot, cancellationToken); + } + + var previous = CurrentSnapshot; + if (snapshot.IsSuccessorTo(previous)) + { + CurrentSnapshot = MembershipTableSnapshot.Update(previous, snapshot); + } + + return Task.CompletedTask; + } + + public Task UpdateIAmAlive(CancellationToken cancellationToken) => Task.CompletedTask; + + public bool CheckHealth(DateTime lastCheckTime, out string reason) + { + reason = string.Empty; + return true; + } + + public void Participate(ISiloLifecycle lifecycle) + { + } + + private static async IAsyncEnumerable EmptyUpdates( + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.CompletedTask; + yield break; + } + } + + + + private sealed class TestOptionsMonitor(T currentValue) : IOptionsMonitor + { + public T CurrentValue => currentValue; + + public T Get(string? name) => currentValue; + + public IDisposable? OnChange(Action listener) => null; + } + + private sealed class TestTimeProvider : TimeProvider + { + private readonly object _lock = new(); + private DateTimeOffset _utcNow = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + private long _timestamp; + + public override DateTimeOffset GetUtcNow() + { + lock (_lock) + { + return _utcNow; + } + } + + public override long GetTimestamp() + { + lock (_lock) + { + return _timestamp; + } + } + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public void Advance(TimeSpan value) + { + lock (_lock) + { + _utcNow += value; + _timestamp += value.Ticks; + } + } + } + + private sealed class RecordingFakeTimeProvider : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private readonly object _lock = new(); + private readonly List _timerDueTimes = []; + private int _throwOnNextTimerChange; + + public IReadOnlyList TimerDueTimes + { + get + { + lock (_lock) + { + return [.. _timerDueTimes]; + } + } + } + + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + + public override long GetTimestamp() => _inner.GetTimestamp(); + + public override long TimestampFrequency => _inner.TimestampFrequency; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + RecordTimerChange(dueTime); + return new RecordingTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + } + + public void Advance(TimeSpan duration) => _inner.Advance(duration); + + public void ThrowOnNextTimerChange() => ThrowOnNextTimerChanges(1); + + public void ThrowOnNextTimerChanges(int count) => Interlocked.Exchange(ref _throwOnNextTimerChange, count); + + private bool ShouldThrowOnTimerChange() + { + while (true) + { + var current = Volatile.Read(ref _throwOnNextTimerChange); + if (current <= 0) + { + return false; + } + + if (Interlocked.CompareExchange(ref _throwOnNextTimerChange, current - 1, current) == current) + { + return true; + } + } + } + + private void RecordTimerChange(TimeSpan dueTime) + { + lock (_lock) + { + _timerDueTimes.Add(dueTime); + } + } + + private sealed class RecordingTimer(RecordingFakeTimeProvider owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + owner.RecordTimerChange(dueTime); + if (owner.ShouldThrowOnTimerChange()) + { + throw new InvalidOperationException("The test timer fails one scheduled change."); + } + + return inner.Change(dueTime, period); + } + + public void Dispose() => inner.Dispose(); + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + } + + private sealed class RecordingLogger : Microsoft.Extensions.Logging.ILogger + { + private int _warningCount; + + public int WarningCount => Volatile.Read(ref _warningCount); + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + + public void Log( + Microsoft.Extensions.Logging.LogLevel logLevel, + Microsoft.Extensions.Logging.EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == Microsoft.Extensions.Logging.LogLevel.Warning) + { + Interlocked.Increment(ref _warningCount); + } + } + } + + // Subscribes to the dissemination DiagnosticListener and lets a test deterministically await the moment a + // peer pump (re)arms its flush timer, replacing wall-clock Task.Delay bridges. Buffered, consume-once + // semantics mean an event emitted before the wait is registered is still observed. + private sealed class BroadcastScheduleObserver : IObserver>, IDisposable + { + private readonly object _lock = new(); + private readonly List _events = new(); + private readonly HashSet _consumed = new(); + private readonly List _waiters = new(); + private readonly IDisposable _subscription; + + public BroadcastScheduleObserver() + { + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == DisseminationEvents.BroadcastScheduledEventName); + } + + public Task WaitAsync( + Func predicate, + TimeSpan timeout, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + TaskCompletionSource completion; + lock (_lock) + { + foreach (var scheduled in _events) + { + if (!_consumed.Contains(scheduled) && predicate(scheduled)) + { + _consumed.Add(scheduled); + return Task.FromResult(scheduled); + } + } + + completion = new(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add(new Waiter(predicate, completion)); + } + + return completion.Task.WaitAsync(timeout, cancellationToken); + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is not DisseminationBroadcastScheduledEvent scheduled) + { + return; + } + + TaskCompletionSource? completion = null; + lock (_lock) + { + _events.Add(scheduled); + for (var i = 0; i < _waiters.Count; i++) + { + if (_waiters[i].Predicate(scheduled)) + { + completion = _waiters[i].Completion; + _waiters.RemoveAt(i); + _consumed.Add(scheduled); + break; + } + } + } + + completion?.TrySetResult(scheduled); + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + + private sealed record Waiter( + Func Predicate, + TaskCompletionSource Completion); + } + + private sealed class AdmissionRejectionObserver : IObserver>, IDisposable + { + private readonly IDisposable _subscription; + + public AdmissionRejectionObserver() + { + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == DisseminationEvents.QueueAdmissionRejectedEventName); + } + + public List Events { get; } = []; + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationQueueAdmissionRejectedEvent rejection) + { + Events.Add(rejection); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PublishRoutesOnlyToNamespaceMembershipScope(bool activeOnly) + { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; + var topology = CreateScopeRoutingTopology(); + var transport = topology.Transport; + var ns = new FakeNamespace(topology.Local, "scoped-publish", scope); + var protocol = CreateProtocol( + transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 32); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue("load", sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(topology.GetPeers(scope), GetBroadcastPeersForNamespace(transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReceiveBroadcastForwardsOnlyToNamespaceMembershipScope(bool activeOnly) + { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; + var topology = CreateForwardingScopeTopology(); + var ns = new FakeNamespace(topology.Local, "scoped-forward", scope); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 2); + + await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch + { + Sender = topology.Sender, + Values = CreateValueGroups(ns.Name, ns.CreateItem(topology.Sender, "load", sequence: 1)), + }, + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + SiloAddress[] expected = scope == DisseminationMembershipScope.ActiveMembers + ? [topology.LastActive] + : [topology.LastActive, topology.Joining]; + Assert.Equal(expected, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetBroadcastPeersForNamespace(topology.Transport, ns.Name)); + Assert.Equal(1, ns.GetVersion("load")); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AntiEntropyRoutesOnlyToNamespaceMembershipScope(bool activeOnly) + { + var scope = activeOnly ? DisseminationMembershipScope.ActiveMembers : DisseminationMembershipScope.AllMembers; + var topology = CreateScopeRoutingTopology(); + var ns = new FakeNamespace(topology.Local, "scoped-anti-entropy", scope); + ns.SetValue("load", version: 1); + var protocol = CreateProtocol( + topology.Transport, + [ns], + static options => options.Overlay.AntiEntropyPeerCount = 32); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Equal(topology.GetPeers(scope), GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + Assert.DoesNotContain(topology.Dead, GetAntiEntropyPeersForNamespace(topology.Transport, ns.Name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task AntiEntropyPreservesCursorFairnessAcrossNamespaceScopes() + { + var topology = CreateScopeRoutingTopology(); + var load = new FakeNamespace( + topology.Local, + new DisseminationNamespace("deployment-load-fairness"), + DisseminationMembershipScope.ActiveMembers); + var membership = new FakeNamespace( + topology.Local, + new DisseminationNamespace("membership-fairness"), + DisseminationMembershipScope.AllMembers); + load.SetValue("load", version: 1); + membership.SetValue("membership", version: 1); + var protocol = CreateProtocol(topology.Transport, [load, membership], options => + { + options.Overlay.AntiEntropyPeerCount = 1; + options.MaxBatchItems = 1; + options.MaxBatchBytes = sizeof(long); + }); + + for (var i = 0; i < 5; i++) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + } + + Assert.Equal(5, topology.Transport.AntiEntropyRequests.Count); + Assert.Equal( + new[] { topology.ActiveOne, topology.ActiveTwo }, + GetAntiEntropyPeersForNamespace(topology.Transport, load.Name).Distinct().OrderBy(static silo => silo)); + Assert.Equal( + new[] + { + topology.ActiveOne, + topology.ActiveTwo, + topology.Joining, + topology.ShuttingDown, + topology.Stopping, + }.OrderBy(static silo => silo), + GetAntiEntropyPeersForNamespace(topology.Transport, membership.Name).Distinct().OrderBy(static silo => silo)); + Assert.All( + topology.Transport.AntiEntropyRequests.Where(request => request.Request.Digests.ContainsKey(load.Name)), + request => Assert.Contains(request.Peer, new[] { topology.ActiveOne, topology.ActiveTwo })); + + var request = new DisseminationAntiEntropyRequest + { + Sender = topology.ActiveOne, + Digests = new() + { + [load.Name] = [new DigestEntry("load", version: 0)], + [membership.Name] = [new DigestEntry("membership", version: 0)], + }, + }; + var first = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var second = await protocol.ReceiveAntiEntropy(request, TestContext.Current.CancellationToken); + var responses = new[] { first, second }; + + Assert.All(responses, response => + { + var value = Assert.Single(GetAntiEntropyResponseValues(response)); + Assert.Equal(sizeof(long), value.Value.Payload.Length); + Assert.True(response.Truncated); + }); + Assert.Equal( + new[] { load.Name, membership.Name }.OrderBy(static name => name), + responses.SelectMany(static response => response.Values.Keys).OrderBy(static name => name)); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task QueuePrunePreservesPeerStateRequiredByAnotherNamespaceScope() + { + var local = CreateSilo(35001); + var peer = CreateSilo(35002); + var transport = new FakeTransport(local, peer); + var timeProvider = new FakeTimeProvider(); + var load = new FakeNamespace( + local, + new DisseminationNamespace("deployment-load-prune"), + DisseminationMembershipScope.ActiveMembers); + var membership = new FakeNamespace( + local, + new DisseminationNamespace("membership-prune"), + DisseminationMembershipScope.AllMembers); + var protocol = CreateProtocol( + transport, + [load, membership], + static options => options.Overlay.FanOutFactor = static _ => 1, + timeProvider); + var pending = BeforeBroadcastPumpsRun(() => + { + var loadPublication = PublishValue(protocol, load, load.CreateValue("load", 1), TestContext.Current.CancellationToken); + var membershipPublication = PublishValue(protocol, membership, membership.CreateValue("membership", 1), TestContext.Current.CancellationToken); + transport.PeerStatuses[peer] = SiloStatus.Joining; + var receive = protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = peer, + Values = [], + }, TestContext.Current.CancellationToken); + return (loadPublication, membershipPublication, receive); + }); + Assert.True(await pending.loadPublication); + Assert.True(await pending.membershipPublication); + await pending.receive; + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal(peer, batch.Peer); + Assert.DoesNotContain(load.Name, batch.Batch.Values.Keys); + var membershipValue = Assert.Single(batch.Batch.Values[membership.Name]); + Assert.Equal(new DisseminationKey("membership"), membershipValue.Value.Key); + Assert.Equal(1, membershipValue.Value.ToVersion); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueRejectsNewDistinctKeyAtNamespacePendingLimit() + { + var (_, peer, transport, ns) = CreatePeerFixture(36001, 36002); + ns.Options.MaxPendingItemCount = 2; + var queue = CreateBroadcastQueue(transport, [ns]); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("first", version: 1); + ns.SetValue("second", version: 1); + ns.SetValue("rejected", version: 1); + + queue.Notify(peer, ns, "first"); + queue.Notify(peer, ns, "second"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "rejected"); + releaseSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + Assert.Equal( + new DisseminationKey[] { "first", "second" }, + GetBroadcastValues(batch.Batch).Select(static value => value.Value.Key).OrderBy(static key => key.Value)); + Assert.DoesNotContain(GetBroadcastValues(batch.Batch), static value => value.Value.Key == new DisseminationKey("rejected")); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAllowsLatestReplacementForAdmittedKeyAtPendingLimit() + { + var (_, peer, transport, ns) = CreatePeerFixture(36011, 36012); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (Interlocked.Increment(ref sendCount) == 1) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + ns.SetValue("admitted", version: 1); + + queue.Notify(peer, ns, "admitted"); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + ns.SetValue("admitted", version: 2); + queue.Notify(peer, ns, "admitted"); + ns.SetValue("rejected", version: 1); + queue.Notify(peer, ns, "rejected"); + releaseFirstSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, sendCount); + Assert.Equal( + new long[] { 1, 2 }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.ToVersion)); + Assert.All( + transport.BroadcastBatches, + batch => Assert.Equal( + new DisseminationKey("admitted"), + Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key)); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmissionRejectionEmitsBoundedDiagnostic() + { + var local = CreateSilo(36021); + var peer = CreateSilo(36022); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("diagnostic-bound")); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + using var observer = new AdmissionRejectionObserver(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("admitted", version: 1); + ns.SetValue("sensitive-unbounded-key", version: 1); + + queue.Notify(peer, ns, "admitted"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "sensitive-unbounded-key"); + + var rejection = Assert.Single(observer.Events); + Assert.Equal(local, rejection.LocalSilo); + Assert.Equal(peer, rejection.Peer); + Assert.Equal(ns.Name, rejection.Namespace); + Assert.Equal(1, rejection.Limit); + Assert.Equal(DisseminationEvents.NamespacePendingLimitReason, rejection.Reason); + Assert.Null(rejection.GetType().GetProperty("Key")); + releaseSend.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmissionRejectionIncrementsBoundedReasonMetric() + { + const string instrumentName = DisseminationInstruments.QueueAdmissionRejectedName; + var local = CreateSilo(36031); + var peer = CreateSilo(36032); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local, new DisseminationNamespace("metric-bound")); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + var observation = new TaskCompletionSource<(long Measurement, KeyValuePair[] Tags)>( + TaskCreationOptions.RunContinuationsAsynchronously); + Instrument? publishedInstrument = null; + var observationCount = 0; + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName && instrument.Name == instrumentName) + { + publishedInstrument = instrument; + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var capturedTags = tags.ToArray(); + if (capturedTags.Any(tag => tag.Key == "namespace" && Equals(tag.Value, ns.Name))) + { + Interlocked.Increment(ref observationCount); + observation.TrySetResult((measurement, capturedTags)); + } + }); + listener.Start(); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + ns.SetValue("admitted", version: 1); + ns.SetValue("rejected", version: 1); + + queue.Notify(peer, ns, "admitted"); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(peer, ns, "rejected"); + var result = await observation.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.IsType>(publishedInstrument); + Assert.Equal("keys", publishedInstrument.Unit); + Assert.Equal(1, result.Measurement); + Assert.Equal(1, observationCount); + Assert.Equal( + new[] { "namespace", "reason" }, + result.Tags.Select(static tag => tag.Key).OrderBy(static key => key)); + Assert.Contains(result.Tags, tag => tag.Key == "namespace" && Equals(tag.Value, ns.Name)); + Assert.Contains( + result.Tags, + tag => tag.Key == "reason" && Equals(tag.Value, DisseminationEvents.NamespacePendingLimitReason)); + releaseSend.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public void DisseminationFailureMetricsUseBoundedDimensionsOnTheOrleansMeter() + { + var expectedInstruments = new HashSet + { + DisseminationInstruments.BroadcastSendFailuresName, + DisseminationInstruments.BroadcastScheduledName, + DisseminationInstruments.AntiEntropyFailuresName, + DisseminationInstruments.PumpFailuresName, + DisseminationInstruments.PublicationsName, + }; + var observations = new ConcurrentDictionary[] Tags)>(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && expectedInstruments.Contains(instrument.Name)) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + observations[instrument.Name] = (measurement, tags.ToArray())); + listener.Start(); + + DisseminationInstruments.OnBroadcastSendFailure(DisseminationFailureReason.Timeout); + DisseminationInstruments.OnBroadcastScheduled(DisseminationBroadcastScheduleReason.Retry); + DisseminationInstruments.OnAntiEntropyFailure(DisseminationFailureReason.Error); + DisseminationInstruments.OnPumpFailure(DisseminationPumpFailureStatus.Permanent); + DisseminationInstruments.OnPublication( + new DisseminationNamespace("test"), + accepted: false, + reason: "disabled"); + + Assert.Equal(expectedInstruments.Order(), observations.Keys.Order()); + AssertMetric(DisseminationInstruments.BroadcastSendFailuresName, ("reason", "timeout")); + AssertMetric(DisseminationInstruments.BroadcastScheduledName, ("reason", "retry")); + AssertMetric(DisseminationInstruments.AntiEntropyFailuresName, ("reason", "error")); + AssertMetric(DisseminationInstruments.PumpFailuresName, ("status", "permanent")); + AssertMetric( + DisseminationInstruments.PublicationsName, + ("namespace", new DisseminationNamespace("test")), + ("result", "rejected"), + ("reason", "disabled")); + + void AssertMetric(string name, params (string Name, object Value)[] expectedTags) + { + var observation = observations[name]; + Assert.Equal(1, observation.Measurement); + Assert.Equal(expectedTags.Length, observation.Tags.Length); + foreach (var expectedTag in expectedTags) + { + Assert.Contains( + observation.Tags, + tag => tag.Key == expectedTag.Name && Equals(tag.Value, expectedTag.Value)); + } + } + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterAcknowledgedDrain() + { + var (_, peer, transport, ns) = CreatePeerFixture(36041, 36042); + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns]); + ns.SetValue("first", version: 1); + + queue.Notify(peer, ns, "first"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + ns.SetValue("second", version: 1); + queue.Notify(peer, ns, "second"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, transport.BroadcastBatches.Count); + Assert.Equal( + new DisseminationKey[] { "first", "second" }, + transport.BroadcastBatches.Select(batch => Assert.Single(GetBroadcastValues(batch.Batch)).Value.Key)); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterMembershipPrune() + { + var local = CreateSilo(36051); + var peer = CreateSilo(36052); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace( + local, + new DisseminationNamespace("prune-recovery"), + DisseminationMembershipScope.ActiveMembers); + ns.Options.MaxPendingItemCount = 1; + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (GetBroadcastValues(batch).Any(static value => value.Value.Key == new DisseminationKey("before-prune"))) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + var protocol = CreateProtocol( + transport, + [ns], + static options => options.Overlay.FanOutFactor = static _ => 1, + new FakeTimeProvider()); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue("before-prune", sequence: 1), + TestContext.Current.CancellationToken)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + using var observer = new AdmissionRejectionObserver(); + Assert.False(await PublishValue( + protocol, + ns, + ns.CreateValue("rejected-before-prune", sequence: 1), + TestContext.Current.CancellationToken)); + Assert.Single(observer.Events); + transport.PeerStatuses[peer] = SiloStatus.Joining; + await protocol.ReceiveBroadcast( + new DisseminationBroadcastBatch { Sender = peer, Values = [] }, + TestContext.Current.CancellationToken); + transport.PeerStatuses[peer] = SiloStatus.Active; + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue("after-prune", sequence: 1), + TestContext.Current.CancellationToken)); + Assert.Single(observer.Events); + releaseFirstSend.TrySetResult(); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Contains( + transport.BroadcastBatches.SelectMany(batch => GetBroadcastValues(batch.Batch)), + static value => value.Value.Key == new DisseminationKey("after-prune") && value.Value.ToVersion == 1); + Assert.DoesNotContain( + transport.BroadcastBatches.SelectMany(batch => GetBroadcastValues(batch.Batch)), + static value => value.Value.Key == new DisseminationKey("rejected-before-prune")); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueHardBoundIsIndependentPerPeerAndNamespace() + { + var local = CreateSilo(36061); + var firstPeer = CreateSilo(36062); + var secondPeer = CreateSilo(36063); + var transport = new FakeTransport(local, firstPeer, secondPeer); + var firstNamespace = new FakeNamespace(local, new DisseminationNamespace("first-bound")); + var secondNamespace = new FakeNamespace(local, new DisseminationNamespace("second-bound")); + firstNamespace.Options.MaxPendingItemCount = 1; + secondNamespace.Options.MaxPendingItemCount = 1; + firstNamespace.SetValue("first-peer", version: 1); + firstNamespace.SetValue("rejected-first-peer", version: 1); + firstNamespace.SetValue("second-peer", version: 1); + secondNamespace.SetValue("other-namespace", version: 1); + var queue = CreateBroadcastQueue(transport, [firstNamespace, secondNamespace]); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (target, batch, cancellationToken) => + { + transport.BroadcastBatches.Add((target, batch)); + if (target.Equals(firstPeer) + && batch.Values.TryGetValue(firstNamespace.Name, out var values) + && values.Any(static value => value.Value.Key == new DisseminationKey("first-peer"))) + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellationToken); + } + }; + + queue.Notify(firstPeer, firstNamespace, "first-peer"); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(firstPeer, firstNamespace, "rejected-first-peer"); + queue.Notify(secondPeer, firstNamespace, "second-peer"); + queue.Notify(firstPeer, secondNamespace, "other-namespace"); + releaseFirstSend.TrySetResult(); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var firstPeerBatches = transport.BroadcastBatches.Where(batch => batch.Peer.Equals(firstPeer)).ToArray(); + Assert.Equal( + new[] { firstNamespace.Name, secondNamespace.Name }.OrderBy(static name => name.Value), + firstPeerBatches.SelectMany(static batch => batch.Batch.Values.Keys).Distinct().OrderBy(static name => name.Value)); + Assert.Equal( + new DisseminationKey("first-peer"), + Assert.Single( + firstPeerBatches.SelectMany(batch => batch.Batch.Values.GetValueOrDefault(firstNamespace.Name) ?? [])).Value.Key); + Assert.Equal( + new DisseminationKey("other-namespace"), + Assert.Single( + firstPeerBatches.SelectMany(batch => batch.Batch.Values.GetValueOrDefault(secondNamespace.Name) ?? [])).Value.Key); + var secondPeerBatch = Assert.Single(transport.BroadcastBatches, batch => batch.Peer.Equals(secondPeer)).Batch; + Assert.Equal( + new DisseminationKey("second-peer"), + Assert.Single(secondPeerBatch.Values[firstNamespace.Name]).Value.Key); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueMaterializesNamespaceRepairAtSendTime() + { + var (_, peer, transport, ns) = CreatePeerFixture(36071, 36072); + var queue = CreateBroadcastQueue(transport, [ns]); + ns.SetValue("latest", version: 1); + Assert.True(BeforeBroadcastPumpsRun(() => + { + var accepted = queue.Notify(peer, ns, "latest"); + ns.SetValue("latest", version: 2); + return accepted; + })); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var value = Assert.Single(GetBroadcastValues(Assert.Single(transport.BroadcastBatches).Batch)).Value; + Assert.Equal(new DisseminationKey("latest"), value.Key); + Assert.Equal(2, value.ToVersion); + Assert.Equal(1, ns.RepairRequestCount); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task PeerQueueAdvancesKnownVersionOnlyFromResponseAcknowledgment() + { + var local = CreateSilo(36081); + var peer = CreateSilo(36082); + var transport = new FakeTransport(local, peer); + var sentVersions = new List(); + var sendCount = 0; + transport.SendBroadcastResponseHandler = (target, batch, cancellationToken) => + { + sentVersions.Add(GetBroadcastValues(batch).Select(static value => value.Value.ToVersion).ToArray()); + return Task.FromResult(Interlocked.Increment(ref sendCount) == 1 + ? new DisseminationBroadcastResponse() + : FakeTransport.CreateAcknowledgment(batch)); + }; + var ns = new ProtocolReviewNamespace(local); + var baselines = new List(); + ns.RepairHandler = request => + { + baselines.Add(request.FromVersion); + return ns.Inner.CreateRepair(request); + }; + ns.Options.MaxPendingItemCount = 1; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + using var schedules = new BroadcastScheduleObserver(); + var retry = schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(local) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + ns.Inner.SetValue("ack", 1); + + queue.Notify(peer, ns, "ack"); + await retry; + Assert.Equal(1, sendCount); + ns.Inner.SetValue("ack", 2); + queue.Notify(peer, ns, "ack"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + ns.Inner.SetValue("ack", 3); + queue.Notify(peer, ns, "ack"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(3, sendCount); + Assert.Equal(new long[] { 1 }, sentVersions[0]); + Assert.Equal(new long[] { 2 }, sentVersions[1]); + Assert.Equal(new long[] { 3 }, sentVersions[2]); + Assert.Equal(new long?[] { null, null, 2 }, baselines); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task BlockedPeerPumpDoesNotPreventAnotherPeerPumpFromDraining() + { + var local = CreateSilo(36091); + var blockedPeer = CreateSilo(36092); + var healthyPeer = CreateSilo(36093); + var transport = new FakeTransport(local, blockedPeer, healthyPeer); + var blockedStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var healthyCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseBlocked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var inFlight = 0; + var observedMax = 0; + transport.SendBroadcastResponseHandler = async (target, batch, cancellationToken) => + { + var current = Interlocked.Increment(ref inFlight); + UpdateMaximum(ref observedMax, current); + var isHealthyPeer = target.Equals(healthyPeer); + try + { + if (target.Equals(blockedPeer)) + { + blockedStarted.TrySetResult(); + await releaseBlocked.Task.WaitAsync(cancellationToken); + } + + return FakeTransport.CreateAcknowledgment(batch); + } + finally + { + Interlocked.Decrement(ref inFlight); + if (isHealthyPeer) + { + healthyCompleted.TrySetResult(); + } + } + }; + var ns = new FakeNamespace(local); + ns.SetValue("blocked", version: 1); + ns.SetValue("healthy", version: 1); + var queue = CreateBroadcastQueue( + transport, + [ns], + static options => options.MaxConcurrentSends = 2); + + queue.Notify(blockedPeer, ns, "blocked"); + var blockedFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await blockedStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + queue.Notify(healthyPeer, ns, "healthy"); + var allFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await healthyCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.False(blockedFlush.IsCompleted); + Assert.False(allFlush.IsCompleted); + Assert.Equal(2, observedMax); + Assert.Equal(1, inFlight); + releaseBlocked.TrySetResult(); + await Task.WhenAll(blockedFlush, allFlush).WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + Assert.Equal(0, inFlight); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + + private static ScopeRoutingTopology CreateScopeRoutingTopology() + { + var local = CreateSilo(33001); + var activeOne = CreateSilo(33002); + var activeTwo = CreateSilo(33003); + var joining = CreateSilo(33004); + var shuttingDown = CreateSilo(33005); + var stopping = CreateSilo(33006); + var dead = CreateSilo(33007); + var transport = new FakeTransport( + local, + activeOne, + activeTwo, + joining, + shuttingDown, + stopping, + dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[shuttingDown] = SiloStatus.ShuttingDown; + transport.PeerStatuses[stopping] = SiloStatus.Stopping; + transport.PeerStatuses[dead] = SiloStatus.Dead; + return new(local, activeOne, activeTwo, joining, shuttingDown, stopping, dead, transport); + } + + private static ForwardingScopeTopology CreateForwardingScopeTopology() + { + var sender = CreateSilo(34001); + var local = CreateSilo(34002); + var activeTwo = CreateSilo(34003); + var activeThree = CreateSilo(34004); + var lastActive = CreateSilo(34005); + var joining = CreateSilo(34006); + var shuttingDown = CreateSilo(34007); + var stopping = CreateSilo(34008); + var dead = CreateSilo(34009); + var transport = new FakeTransport( + local, + sender, + activeTwo, + activeThree, + lastActive, + joining, + shuttingDown, + stopping, + dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[shuttingDown] = SiloStatus.ShuttingDown; + transport.PeerStatuses[stopping] = SiloStatus.Stopping; + transport.PeerStatuses[dead] = SiloStatus.Dead; + return new(sender, local, lastActive, joining, dead, transport); + } + + private static SiloAddress[] GetBroadcastPeersForNamespace( + FakeTransport transport, + DisseminationNamespace namespaceName) + { + lock (transport.BroadcastBatches) + { + return + [ + .. transport.BroadcastBatches + .Where(batch => batch.Batch.Values.ContainsKey(namespaceName)) + .Select(static batch => batch.Peer) + .OrderBy(static peer => peer), + ]; + } + } + + private static SiloAddress[] GetAntiEntropyPeersForNamespace( + FakeTransport transport, + DisseminationNamespace namespaceName) + { + lock (transport.AntiEntropyRequests) + { + return + [ + .. transport.AntiEntropyRequests + .Where(request => request.Request.Digests.ContainsKey(namespaceName)) + .Select(static request => request.Peer) + .OrderBy(static peer => peer), + ]; + } + } + + private readonly record struct ScopeRoutingTopology( + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo, + SiloAddress Joining, + SiloAddress ShuttingDown, + SiloAddress Stopping, + SiloAddress Dead, + FakeTransport Transport) + { + public SiloAddress[] GetPeers(DisseminationMembershipScope scope) => + scope == DisseminationMembershipScope.ActiveMembers + ? [ActiveOne, ActiveTwo] + : [ActiveOne, ActiveTwo, Joining, ShuttingDown, Stopping]; + } + + private readonly record struct ForwardingScopeTopology( + SiloAddress Sender, + SiloAddress Local, + SiloAddress LastActive, + SiloAddress Joining, + SiloAddress Dead, + FakeTransport Transport); + + [Fact] + public async Task QueueAcceptanceDoesNotConfirmPeerNamespaceSupport() + { + var local = CreateSilo(31001); + var peer = CreateSilo(31002); + var timeProvider = new FakeTimeProvider(); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + var protocol = CreateProtocol(transport, ns, timeProvider: timeProvider); + + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, 1), TestContext.Current.CancellationToken); + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + Assert.Empty(transport.BroadcastBatches); + return pending; + }); + Assert.True(await publication); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task InboundNamespaceTrafficConfirmsPeerSupport() + { + var local = CreateSilo(31003); + var peer = CreateSilo(31004); + var transport = new FakeTransport(local, peer); + var supported = new FakeNamespace(local); + var other = new FakeNamespace(local, "other"); + var protocol = CreateProtocol(transport, [supported, other]); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch(peer, supported.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 1)), + TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(supported)); + Assert.Equal([peer], protocol.GetUnconfirmedPeers(other)); + Assert.Equal(1, Assert.Single(response.Acknowledgments[supported.Name]).Version); + } + + [Fact] + public async Task ResponseAcknowledgmentConfirmsPeerNamespaceSupport() + { + var (_, _, transport, ns) = CreatePeerFixture(31005, 31006); + var protocol = CreateProtocol(transport, ns); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + Assert.Equal(1, Assert.Single(transport.BroadcastBatches).Batch.Values[ns.Name].Single().Value.ToVersion); + } + + [Fact] + public async Task UnsupportedNamespaceResponseRevokesPeerSupport() + { + var (_, peer, transport, ns) = CreatePeerFixture(31007, 31008); + var protocol = CreateProtocol(transport, ns); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + transport.SendBroadcastResponseHandler = (_, _, _) => Task.FromResult( + new DisseminationBroadcastResponse { UnsupportedNamespaces = [ns.Name] }); + + Assert.True(await PublishValue( + protocol, + ns, + ns.CreateValue(FakeNamespace.DefaultKey, sequence: 1), + TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal([peer], protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task AntiEntropyResponseRetainsUnrelatedPeerNamespaceConfirmations() + { + var local = CreateSilo(31009); + var peer = CreateSilo(31010); + var first = new FakeNamespace(local, "first"); + var second = new FakeNamespace(local, "second"); + first.SetValue(FakeNamespace.DefaultKey, version: 1); + second.SetValue(FakeNamespace.DefaultKey, version: 1); + var transport = new FakeTransport(local, peer); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult( + new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = new() + { + [first.Name] = [first.CreateItem(peer, FakeNamespace.DefaultKey, sequence: 2)], + }, + }); + var protocol = CreateProtocol( + transport, + [first, second], + options => options.Overlay.AntiEntropyPeerCount = 1); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = new() + { + [first.Name] = [new(FakeNamespace.DefaultKey, 0)], + [second.Name] = [new(FakeNamespace.DefaultKey, 0)], + }, + }, + TestContext.Current.CancellationToken); + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken); + + Assert.Empty(protocol.GetUnconfirmedPeers(first)); + Assert.Empty(protocol.GetUnconfirmedPeers(second)); + Assert.Equal(2, first.GetVersion(FakeNamespace.DefaultKey)); + } + + [Fact] + public async Task PeerNamespaceConfirmationPersistsUntilExplicitRevocationOrRemoval() + { + var local = CreateSilo(31011); + var peer = CreateSilo(31012); + var timeProvider = new FakeTimeProvider(); + var transport = new FakeTransport(local, peer); + var ns = new FakeNamespace(local); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(7); + var protocol = CreateProtocol( + transport, + ns, + options => options.Overlay.AntiEntropyInterval = TimeSpan.FromSeconds(5), + timeProvider); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = peer, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + + timeProvider.Advance(TimeSpan.FromDays(1)); + Assert.Empty(protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task GetUnconfirmedPeersReturnsOnlyActiveUnconfirmedPeersForNamespace() + { + var local = CreateSilo(31013); + var confirmed = CreateSilo(31014); + var unconfirmed = CreateSilo(31015); + var joining = CreateSilo(31016); + var dead = CreateSilo(31017); + var transport = new FakeTransport(local, confirmed, unconfirmed, joining, dead); + transport.PeerStatuses[joining] = SiloStatus.Joining; + transport.PeerStatuses[dead] = SiloStatus.Dead; + var ns = new FakeNamespace(local, "active-only", DisseminationMembershipScope.ActiveMembers); + var protocol = CreateProtocol(transport, [ns]); + await protocol.ReceiveAntiEntropy( + new DisseminationAntiEntropyRequest + { + Sender = confirmed, + Digests = CreateAntiEntropyRequestDigest(ns.Name, (FakeNamespace.DefaultKey, 0)), + }, + TestContext.Current.CancellationToken); + + Assert.Equal([unconfirmed], protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(local, protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(joining, protocol.GetUnconfirmedPeers(ns)); + Assert.DoesNotContain(dead, protocol.GetUnconfirmedPeers(ns)); + } + + [Fact] + public async Task DisseminationPushBroadcastWireContractPreservesAcknowledgmentsAndUnsupportedNamespaces() + { + var (_, peer, transport, ns) = CreatePeerFixture(31018, 31019); + var protocol = CreateProtocol(transport, ns); + var unsupported = new DisseminationNamespace("unsupported"); + var receivedNamespaces = new ConcurrentBag(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == DisseminationInstruments.MeterName + && instrument.Name == DisseminationInstruments.ValuesReceivedName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + receivedNamespaces.Add(tags.ToArray().Single(static tag => tag.Key == "namespace").Value); + }); + listener.Start(); + var batch = new DisseminationBroadcastBatch + { + Sender = peer, + Values = new() + { + [ns.Name] = [ns.CreateItem(peer, "supported-key", sequence: 3)], + [unsupported] = [ns.CreateItem(peer, "unsupported-key", sequence: 4)], + }, + }; + + var response = await protocol.ReceiveBroadcast(batch, TestContext.Current.CancellationToken); + + var method = typeof(IDisseminationSystemTarget).GetMethod(nameof(IDisseminationSystemTarget.PushBroadcast)); + Assert.Equal(typeof(Task), method!.ReturnType); + Assert.Equal( + [("Acknowledgments", 0), ("UnsupportedNamespaces", 1), ("AllVersionsAcknowledged", 2)], + typeof(DisseminationBroadcastResponse) + .GetProperties() + .Select(property => ( + property.Name, + Convert.ToInt32(property.GetCustomAttributesData() + .Single(attribute => attribute.AttributeType == typeof(IdAttribute)) + .ConstructorArguments.Single().Value))) + .OrderBy(static property => property.Item2)); + Assert.False(batch.SupportsCompactAcknowledgments); + Assert.False(response.AllVersionsAcknowledged); + var acknowledgment = Assert.Single(response.Acknowledgments); + Assert.Equal(ns.Name, acknowledgment.Key); + var digest = Assert.Single(acknowledgment.Value); + Assert.Equal(new DisseminationKey("supported-key"), digest.Key); + Assert.Equal(3, digest.Version); + Assert.Equal([unsupported], response.UnsupportedNamespaces); + Assert.Equal([ns.Name], receivedNamespaces); + } + + [Theory] + [InlineData("partially-confirmed")] + [InlineData("confirmed")] + [InlineData("inactive")] + [InlineData("unconfirmed")] + public async Task DeploymentLoadPublisherDirectFallbackRequiresUnconfirmedActivePeers(string support) + { + var harness = CreateDeploymentLoadPublisherHarness(); + SiloAddress[] unconfirmed = support switch + { + "partially-confirmed" => [harness.ActiveTwo], + "confirmed" => [], + "inactive" => [harness.Joining], + _ => [harness.ActiveOne, harness.ActiveTwo], + }; + harness.Dissemination.UnconfirmedPeers = unconfirmed; + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + SiloAddress[] expected = support is "confirmed" or "inactive" ? [] : [harness.ActiveOne, harness.ActiveTwo]; + AssertDirectRecipients(harness, expected); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Single(harness.Dissemination.QueryCalls); + Assert.True(Assert.Single(harness.Dissemination.PublishResults)); + Assert.Equal(unconfirmed, harness.Dissemination.UnconfirmedPeers.OrderBy(static peer => peer)); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationIsDisabled() + { + var harness = CreateDeploymentLoadPublisherHarness(namespaceEnabled: false); + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Empty(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationFails(bool throws) + { + var harness = CreateDeploymentLoadPublisherHarness(); + harness.Dissemination.PublishHandler = (_, _, _, _) => throws + ? ValueTask.FromException(new InvalidOperationException("test failure")) + : ValueTask.FromResult(false); + + await harness.PublishStatistics(TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherFallsBackToAllActivePeersWhenDisseminationTimesOut() + { + var timeProvider = new FakeTimeProvider(); + var harness = CreateDeploymentLoadPublisherHarness(timeProvider: timeProvider); + harness.Dissemination.PublishHandler = async (_, _, _, cancellationToken) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }; + + var publish = harness.PublishStatistics(TestContext.Current.CancellationToken); + await harness.Dissemination.PublishStarted.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + timeProvider.Advance(harness.RefreshTime); + Assert.False(publish.IsCompleted); + timeProvider.Advance(harness.RefreshTime); + await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + AssertDirectRecipients(harness, harness.ActiveOne, harness.ActiveTwo); + Assert.Single(harness.Dissemination.PublishCalls); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + [Fact] + public async Task DeploymentLoadPublisherCallerCancellationStopsDisseminationWithoutFallback() + { + var harness = CreateDeploymentLoadPublisherHarness(); + using var cancellation = new CancellationTokenSource(); + CancellationToken observedToken = default; + harness.Dissemination.PublishHandler = async (_, _, _, cancellationToken) => + { + observedToken = cancellationToken; + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return true; + }; + + var publish = harness.PublishStatistics(cancellation.Token); + await harness.Dissemination.PublishStarted.Task.WaitAsync( + TimeSpan.FromSeconds(5), + TestContext.Current.CancellationToken); + + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + async () => await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.True(observedToken.IsCancellationRequested); + AssertDirectRecipients(harness); + Assert.Empty(harness.Dissemination.QueryCalls); + } + + private static DeploymentLoadPublisherHarness CreateDeploymentLoadPublisherHarness( + bool namespaceEnabled = true, + FakeTimeProvider? timeProvider = null) + { + var local = CreateSilo(31101); + var activeOne = CreateSilo(31102); + var activeTwo = CreateSilo(31103); + var joining = CreateSilo(31104); + var statusOracle = new Phase4FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(activeOne, SiloStatus.Active); + statusOracle.SetStatus(activeTwo, SiloStatus.Active); + statusOracle.SetStatus(joining, SiloStatus.Joining); + var directTargets = new Dictionary + { + [activeOne] = new(), + [activeTwo] = new(), + [joining] = new(), + }; + var grainFactory = Substitute.For(); + grainFactory + .GetSystemTarget( + Constants.DeploymentLoadPublisherSystemTargetType, + Arg.Any()) + .Returns(call => directTargets[call.ArgAt(1)]); + var services = new Phase4MutableServiceProvider(); + timeProvider ??= new FakeTimeProvider(); + services.Add(timeProvider); + var refreshTime = TimeSpan.FromSeconds(5); + var options = new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = refreshTime, + }; + options.Dissemination.Enabled = namespaceEnabled; + var publisher = new DeploymentLoadPublisher( + new FakeLocalSiloDetails(local), + statusOracle, + Options.Create(options), + grainFactory, + NullLoggerFactory.Instance, + new ActivationDirectory(CreatePhase4Instruments()), + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + services, + CreatePhase4SystemTargetShared(local)); + var serializerProvider = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var disseminationNamespace = new DeploymentLoadStatisticsDisseminationNamespace( + publisher, + new TestOptionsMonitor(options), + serializerProvider.GetRequiredService()); + var dissemination = new Phase4FakeDisseminationService(); + services.Add(dissemination); + services.Add(disseminationNamespace); + + return new( + publisher, + dissemination, + local, + activeOne, + activeTwo, + joining, + refreshTime, + directTargets); + } + + private static void AssertDirectRecipients( + DeploymentLoadPublisherHarness harness, + params SiloAddress[] expectedRecipients) + { + var actualRecipients = harness.DirectTargets + .Where(static entry => entry.Value.Updates.Count > 0) + .Select(static entry => entry.Key) + .OrderBy(static peer => peer) + .ToArray(); + Assert.Equal(expectedRecipients.OrderBy(static peer => peer), actualRecipients); + foreach (var recipient in expectedRecipients) + { + var update = Assert.Single(harness.DirectTargets[recipient].Updates); + Assert.Equal(harness.Local, update.Source); + Assert.Equal(harness.Publisher.LocalRuntimeStatistics.DateTime, update.Statistics.DateTime); + } + + Assert.All( + harness.DirectTargets.Where(entry => !expectedRecipients.Contains(entry.Key)), + static entry => Assert.Empty(entry.Value.Updates)); + } + + private static SystemTargetShared CreatePhase4SystemTargetShared(SiloAddress local) => new( + runtimeClient: null!, + new FakeLocalSiloDetails(local), + NullLoggerFactory.Instance, + Options.Create(new SchedulingOptions()), + grainReferenceActivator: null!, + timerRegistry: null!, + activations: new ActivationDirectory(CreatePhase4Instruments()), + schedulerInstruments: CreatePhase4Instruments(), + grainInstruments: CreatePhase4Instruments(), + messagingInstruments: CreatePhase4Instruments(), + messagingProcessingInstruments: CreatePhase4Instruments()); + + private static T CreatePhase4Instruments() where T : class + { + var services = new ServiceCollection(); + services.AddMetrics(); + services.AddSingleton(); + services.AddSingleton(); + return services.BuildServiceProvider().GetRequiredService(); + } + + private sealed record DeploymentLoadPublisherHarness( + DeploymentLoadPublisher Publisher, + Phase4FakeDisseminationService Dissemination, + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo, + SiloAddress Joining, + TimeSpan RefreshTime, + Dictionary DirectTargets) + { + public Task PublishStatistics(CancellationToken cancellationToken) => + Publisher.RunOrQueueTask( + async token => + { + await Publisher.PublishStatistics(token); + return true; + }, + cancellationToken); + } + + private sealed class Phase4FakeDisseminationService : IDisseminationService + { + public List<(IDisseminationNamespace Namespace, DisseminationKey Key, long Version)> PublishCalls { get; } = []; + + public List PublishResults { get; } = []; + + public List QueryCalls { get; } = []; + + public IReadOnlyList UnconfirmedPeers { get; set; } = []; + + public TaskCompletionSource PublishStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Func>? AggregatedPublishHandler { get; set; } + + public async ValueTask PublishAggregated( + IDisseminationNamespace disseminationNamespace, DisseminationKey key, long version, CancellationToken cancellationToken) + { + if (AggregatedPublishHandler is null) + { + return new(await Publish(disseminationNamespace, key, version, cancellationToken), disseminationNamespace.AggregationPeriod); + } + + PublishCalls.Add((disseminationNamespace, key, version)); + PublishStarted.TrySetResult(); + var result = await AggregatedPublishHandler(disseminationNamespace, key, version, cancellationToken); + PublishResults.Add(result.Accepted); + return result; + } + + public Func> PublishHandler { get; set; } = + static (_, _, _, _) => ValueTask.FromResult(true); + + public async ValueTask Publish( + IDisseminationNamespace disseminationNamespace, + DisseminationKey key, + long version, + CancellationToken cancellationToken) + { + PublishCalls.Add((disseminationNamespace, key, version)); + PublishStarted.TrySetResult(); + var result = await PublishHandler(disseminationNamespace, key, version, cancellationToken); + PublishResults.Add(result); + return result; + } + + public IReadOnlyList GetUnconfirmedPeers(IDisseminationNamespace disseminationNamespace) + { + QueryCalls.Add(disseminationNamespace); + return UnconfirmedPeers; + } + } + + private sealed class Phase4MutableServiceProvider : IServiceProvider + { + private readonly Dictionary _services = []; + + public void Add(T service) where T : class => _services[typeof(T)] = service; + + public object? GetService(Type serviceType) => + _services.TryGetValue(serviceType, out var service) ? service : null; + } + + private sealed class Phase4FakeSiloStatusOracle : ISiloStatusOracle + { + private readonly ConcurrentDictionary _statuses = new(); + + public Func? GetStatusHandler { get; set; } + + public SiloStatus CurrentStatus => SiloStatus.Active; + + public string SiloName => "local"; + + public SiloAddress SiloAddress => _statuses.Keys.OrderBy(static silo => silo).First(); + + public void SetStatus(SiloAddress silo, SiloStatus status) => _statuses[silo] = status; + + public SiloAddress[] GetActiveSilos() => + [.. _statuses.Where(static entry => entry.Value == SiloStatus.Active).Select(static entry => entry.Key)]; + + public SiloStatus GetStoredStatus(SiloAddress siloAddress) => + _statuses.TryGetValue(siloAddress, out var status) ? status : SiloStatus.None; + + public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) => + GetStatusHandler?.Invoke(siloAddress) ?? GetStoredStatus(siloAddress); + + public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) => + _statuses + .Where(entry => !onlyActive || entry.Value == SiloStatus.Active) + .ToDictionary(static entry => entry.Key, static entry => entry.Value); + + public bool TryGetSiloName( + SiloAddress siloAddress, + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? siloName) + { + siloName = siloAddress.ToParsableString(); + return true; + } + + public bool IsFunctionalDirectory(SiloAddress siloAddress) => true; + + public bool IsDeadSilo(SiloAddress silo) => GetStoredStatus(silo) == SiloStatus.Dead; + + public bool SubscribeToSiloStatusEvents(ISiloStatusListener observer) => true; + + public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => true; + } + + private sealed class Phase4FakeEnvironmentStatisticsProvider : Orleans.Statistics.IEnvironmentStatisticsProvider + { + public Orleans.Statistics.EnvironmentStatistics GetEnvironmentStatistics() => default; + } + + private sealed class Phase4FakeActivationWorkingSet : IActivationWorkingSet + { + public int Count => 0; + + public void OnActivated(IActivationWorkingSetMember member) + { + } + + public void OnActive(IActivationWorkingSetMember member) + { + } + + public void OnDeactivating(IActivationWorkingSetMember member) + { + } + + public void OnDeactivated(IActivationWorkingSetMember member) + { + } + } + + private sealed class Phase4FakeDeploymentLoadPublisherTarget : IDeploymentLoadPublisher + { + public List<(SiloAddress Source, SiloRuntimeStatistics Statistics)> Updates { get; } = []; + + public Task UpdateRuntimeStatistics( + SiloAddress siloAddress, + SiloRuntimeStatistics siloStats, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Updates.Add((siloAddress, siloStats)); + return Task.CompletedTask; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task LoadNotificationAppliesQueuedUpdatesInPublicationOrder(bool interleaveSilos) + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + (SiloAddress Silo, int Version)[] publications = interleaveSilos + ? [(harness.ActiveOne, 10), (harness.ActiveTwo, 20), (harness.ActiveOne, 30), (harness.ActiveTwo, 40)] + : [(harness.ActiveOne, 10), (harness.ActiveOne, 20), (harness.ActiveOne, 30)]; + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + })); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task[] updates; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + updates = publications.Select(publication => harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + publication.Silo, CreatePhase5Statistics(publication.Version), TestContext.Current.CancellationToken)).ToArray(); + Assert.All(updates, static update => Assert.False(update.IsCompleted)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.All(await Task.WhenAll(updates.Prepend(firstUpdate)), + static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal(30, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + .. publications.Select(static publication => + Phase5Notification.Update(publication.Silo, CreatePhase5Statistics(publication.Version))), + ], + notifications); + if (interleaveSilos) + { + Assert.Equal(40, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + } + } + + [Fact] + public async Task LoadNotificationTerminalRemovalRejectsQueuedUpdate() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + }, + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task queuedUpdate; + Task removal; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + queuedUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + removal = SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + Assert.False(queuedUpdate.IsCompleted); + Assert.False(removal.IsCompleted); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Applied, await firstUpdate); + Assert.Equal(DisseminationApplyResult.Rejected, await queuedUpdate); + await removal; + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Fact] + public async Task ConcurrentLoadUpdateCannotOverwriteTerminalRemoval() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var statusCheckEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseStatusCheck = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var blockStatusCheck = 0; + harness.StatusOracle.GetStatusHandler = silo => + { + if (silo.Equals(harness.ActiveOne) && Interlocked.Exchange(ref blockStatusCheck, 0) == 1) + { + statusCheckEntered.TrySetResult(); + releaseStatusCheck.Task.GetAwaiter().GetResult(); + } + + return harness.StatusOracle.GetStoredStatus(silo); + }; + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken)); + Volatile.Write(ref blockStatusCheck, 1); + var concurrentUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + Task termination; + try + { + await statusCheckEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + termination = SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + Assert.False(termination.IsCompleted); + } + finally + { + releaseStatusCheck.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Rejected, await concurrentUpdate); + await termination; + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + DisseminationApplyResult.Rejected, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(30), TestContext.Current.CancellationToken)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Theory] + [InlineData(SiloStatus.Dead)] + [InlineData(SiloStatus.None)] + public async Task LoadNotificationDoesNotResurrectInactiveSilo(SiloStatus status) + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + await SetPhase5SiloStatusAsync( + harness, harness.ActiveOne, SiloStatus.Dead, TestContext.Current.CancellationToken); + harness.StatusOracle.SetStatus(harness.ActiveOne, status); + + var futureUpdate = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, + CreatePhase5Statistics(20), + TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Rejected, futureUpdate); + Assert.True(harness.Publisher.IsRuntimeStatisticsObsolete(harness.ActiveOne, long.MaxValue)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(harness.ActiveOne)); + Assert.Equal( + [ + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Removal(harness.ActiveOne), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationAllowsActiveNewerIncarnation() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + var terminated = harness.ActiveOne; + var restarted = SiloAddress.New(terminated.Endpoint, terminated.Generation + 1); + + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + terminated, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + notifications.Clear(); + await SetPhase5SiloStatusAsync( + harness, terminated, SiloStatus.Dead, TestContext.Current.CancellationToken); + harness.StatusOracle.SetStatus(restarted, SiloStatus.Active); + + var result = await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + restarted, + CreatePhase5Statistics(20), + TestContext.Current.CancellationToken); + + Assert.Equal(DisseminationApplyResult.Applied, result); + Assert.Equal(20, harness.Publisher.PeriodicStatistics[restarted].ActivationCount); + Assert.Equal( + [ + Phase5Notification.Removal(terminated), + Phase5Notification.Update(restarted, CreatePhase5Statistics(20)), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationCleansOlderGenerationAfterNewerTermination() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => notifications.Add(Phase5Notification.Update(silo, statistics)), + onRemove: silo => notifications.Add(Phase5Notification.Removal(silo)))); + var older = harness.ActiveOne; + var newer = SiloAddress.New(older.Endpoint, older.Generation + 1); + harness.StatusOracle.SetStatus(older, SiloStatus.Active); + harness.StatusOracle.SetStatus(newer, SiloStatus.Active); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + older, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + newer, CreatePhase5Statistics(20), TestContext.Current.CancellationToken)); + + await SetPhase5SiloStatusAsync( + harness, newer, SiloStatus.Dead, TestContext.Current.CancellationToken); + await SetPhase5SiloStatusAsync( + harness, older, SiloStatus.Dead, TestContext.Current.CancellationToken); + + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(older)); + Assert.False(harness.Publisher.PeriodicStatistics.ContainsKey(newer)); + Assert.Contains(Phase5Notification.Removal(older), notifications); + Assert.Contains(Phase5Notification.Removal(newer), notifications); + } + + [Fact] + public async Task LoadNotificationCallbacksRunOutsideSubscriberLock() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + notifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + })); + + var update = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + var registrationCompleted = new TaskCompletionSource<(bool Added, bool Removed)>( + TaskCreationOptions.RunContinuationsAsynchronously); + Task registration; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + var additionalSubscriber = new Phase5StatisticsListener(); + registration = Task.Run( + () => + { + var added = harness.Publisher.SubscribeToStatisticsChangeEvents(additionalSubscriber); + var removed = harness.Publisher.UnsubscribeStatisticsChangeEvents(additionalSubscriber); + registrationCompleted.TrySetResult((added, removed)); + }, + TestContext.Current.CancellationToken); + var result = await registrationCompleted.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + Assert.True(result.Added); + Assert.True(result.Removed); + Assert.False(update.IsCompleted); + Assert.Equal([Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1))], notifications); + } + finally + { + releaseCallback.TrySetResult(); + } + + await registration; + Assert.Equal(DisseminationApplyResult.Applied, await update); + Assert.Equal( + DisseminationApplyResult.Applied, + await harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken)); + Assert.Equal( + [ + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + ], + notifications); + } + + [Fact] + public async Task LoadNotificationsAreSerializedOnPublisherScheduler() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var notifications = new ConcurrentQueue(); + var callbackContexts = new ConcurrentQueue(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var callbackConcurrencyLock = new object(); + var currentCallbackCount = 0; + var maximumCallbackCount = 0; + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + lock (callbackConcurrencyLock) + { + currentCallbackCount++; + maximumCallbackCount = Math.Max(maximumCallbackCount, currentCallbackCount); + } + + try + { + notifications.Enqueue(Phase5Notification.Update(silo, statistics)); + callbackContexts.Enqueue(RuntimeContext.Current); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + } + finally + { + lock (callbackConcurrencyLock) + { + currentCallbackCount--; + } + } + })); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task firstProducer; + Task secondProducer; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + firstProducer = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + secondProducer = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + Assert.False(firstProducer.IsCompleted); + Assert.False(secondProducer.IsCompleted); + Assert.Equal(1, maximumCallbackCount); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.All(await Task.WhenAll(firstUpdate, firstProducer, secondProducer), + static result => Assert.Equal(DisseminationApplyResult.Applied, result)); + Assert.Equal(1, maximumCallbackCount); + Assert.Equal(0, currentCallbackCount); + Assert.Equal(3, notifications.Count); + Assert.All(callbackContexts, context => Assert.Same(harness.Publisher, context)); + Assert.Equal( + [harness.Local, harness.ActiveOne, harness.ActiveTwo], + notifications.Select(static notification => notification.Silo).OrderBy(static silo => silo)); + } + + [Fact] + public async Task LoadNotificationContinuesAfterListenerExceptions() + { + var harness = CreatePhase5DeploymentLoadPublisherHarness(); + var throwingListenerNotifications = new List(); + var observingListenerNotifications = new List(); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + { + throwingListenerNotifications.Add(Phase5Notification.Update(silo, statistics)); + if (silo.Equals(harness.Local)) + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + } + else if (silo.Equals(harness.ActiveOne)) + { + throw new InvalidOperationException("phase 5 listener failure"); + } + })); + harness.Publisher.SubscribeToStatisticsChangeEvents(new Phase5StatisticsListener( + onUpdate: (silo, statistics) => + observingListenerNotifications.Add(Phase5Notification.Update(silo, statistics)))); + + var firstUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.Local, CreatePhase5Statistics(1), TestContext.Current.CancellationToken); + Task failingUpdate; + Task laterUpdate; + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + failingUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveOne, CreatePhase5Statistics(10), TestContext.Current.CancellationToken); + laterUpdate = harness.Publisher.ApplyDisseminatedRuntimeStatisticsAsync( + harness.ActiveTwo, CreatePhase5Statistics(20), TestContext.Current.CancellationToken); + } + finally + { + releaseCallback.TrySetResult(); + } + + Assert.Equal(DisseminationApplyResult.Applied, await firstUpdate); + var exception = await Assert.ThrowsAsync(() => failingUpdate); + Assert.Equal("phase 5 listener failure", exception.Message); + Assert.Equal(DisseminationApplyResult.Applied, await laterUpdate); + var expected = new[] + { + Phase5Notification.Update(harness.Local, CreatePhase5Statistics(1)), + Phase5Notification.Update(harness.ActiveOne, CreatePhase5Statistics(10)), + Phase5Notification.Update(harness.ActiveTwo, CreatePhase5Statistics(20)), + }; + Assert.Equal(expected, throwingListenerNotifications); + Assert.Equal(expected, observingListenerNotifications); + Assert.Equal(10, harness.Publisher.PeriodicStatistics[harness.ActiveOne].ActivationCount); + Assert.Equal(20, harness.Publisher.PeriodicStatistics[harness.ActiveTwo].ActivationCount); + } + + private static Task SetPhase5SiloStatusAsync( + Phase5DeploymentLoadPublisherHarness harness, + SiloAddress silo, + SiloStatus status, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + harness.StatusOracle.SetStatus(silo, status); + harness.Publisher.SiloStatusChangeNotification(silo, status); + return harness.Publisher.RunOrQueueTask( + token => + { + token.ThrowIfCancellationRequested(); + return Task.FromResult(true); + }, + cancellationToken); + } + + private static Phase5DeploymentLoadPublisherHarness CreatePhase5DeploymentLoadPublisherHarness() + { + var local = CreateSilo(32101); + var activeOne = CreateSilo(32102); + var activeTwo = CreateSilo(32103); + var statusOracle = new Phase4FakeSiloStatusOracle(); + statusOracle.SetStatus(local, SiloStatus.Active); + statusOracle.SetStatus(activeOne, SiloStatus.Active); + statusOracle.SetStatus(activeTwo, SiloStatus.Active); + var publisher = new DeploymentLoadPublisher( + new FakeLocalSiloDetails(local), + statusOracle, + Options.Create(new DeploymentLoadPublisherOptions + { + DeploymentLoadPublisherRefreshTime = TimeSpan.FromSeconds(5), + }), + Substitute.For(), + NullLoggerFactory.Instance, + new ActivationDirectory(CreatePhase4Instruments()), + new Phase4FakeActivationWorkingSet(), + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + new Phase4MutableServiceProvider(), + CreatePhase4SystemTargetShared(local)); + return new(publisher, statusOracle, local, activeOne, activeTwo); + } + + private static SiloRuntimeStatistics CreatePhase5Statistics(int version) => new( + activationCount: version, + recentlyUsedActivationCount: version * 10, + new Phase4FakeEnvironmentStatisticsProvider(), + Options.Create(new LoadSheddingOptions()), + new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(version)); + + private sealed record Phase5DeploymentLoadPublisherHarness( + DeploymentLoadPublisher Publisher, + Phase4FakeSiloStatusOracle StatusOracle, + SiloAddress Local, + SiloAddress ActiveOne, + SiloAddress ActiveTwo); + + private readonly record struct Phase5Notification(string Kind, SiloAddress Silo, int ActivationCount) + { + public static Phase5Notification Update(SiloAddress silo, SiloRuntimeStatistics statistics) => + new("Update", silo, statistics.ActivationCount); + + public static Phase5Notification Removal(SiloAddress silo) => new("Remove", silo, -1); + } + + private sealed class Phase5StatisticsListener( + Action? onUpdate = null, + Action? onRemove = null) : ISiloStatisticsChangeListener + { + public void SiloStatisticsChangeNotification(SiloAddress updatedSilo, SiloRuntimeStatistics newStats) => + onUpdate?.Invoke(updatedSilo, newStats); + + public void RemoveSilo(SiloAddress removedSilo) => onRemove?.Invoke(removedSilo); + } + + [Theory] + [InlineData(false, false, 1, 2)] + [InlineData(false, true, 1, 2)] + [InlineData(true, false, 1, 3)] + [InlineData(false, false, 4, 5)] + [InlineData(false, false, 6, 7)] + public async Task ReceiveBroadcastRejectsAndDiagnosesApplyFailureWithoutBlockingLaterValues( + bool unrelatedCancellation, bool forward, long failedVersion, long validVersion) + { + var sender = CreateSilo(33001); + var local = CreateSilo(33002); + var child = CreateSilo(33003); + var transport = new FakeTransport(local, forward ? [sender, child] : [sender]); + var failedKey = new DisseminationKey("throws"); + var validKey = new DisseminationKey("valid"); + using var cancellation = new CancellationTokenSource(); + Exception failure = unrelatedCancellation + ? new OperationCanceledException("unrelated", cancellation.Token) + : new InvalidOperationException("phase 6 apply failure"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = (_, _) => throw failure, + }; + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol( + transport, ns, logger, + configure: static options => options.Overlay.FanOutFactor = static _ => 1); + using var observer = new Phase6ApplyObserver(ns.Name, failedKey, local, sender); + try + { + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, failedVersion), + ns.CreateItem(sender, validKey, validVersion)), + TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.False(cancellation.IsCancellationRequested); + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(failedKey)); + Assert.Equal(validVersion, ns.GetVersion(validKey)); + var acknowledgments = response.Acknowledgments[ns.Name].ToDictionary(static entry => entry.Key); + Assert.Equal(0, acknowledgments[failedKey].Version); + Assert.Equal(validVersion, acknowledgments[validKey].Version); + Assert.Empty(response.UnsupportedNamespaces); + + var rejection = Assert.Single(observer.Events); + Assert.Equal(ns.Name, rejection.Namespace); + Assert.Equal(local, rejection.LocalSilo); + Assert.Equal(sender, rejection.Peer); + Assert.Equal(failedKey, rejection.Key); + Assert.Equal(0, rejection.FromVersion); + Assert.Equal(failedVersion, rejection.ToVersion); + Assert.Equal(nameof(DisseminationApplyResult.Rejected), rejection.Result); + Assert.Equal(sizeof(long), rejection.PayloadBytes); + + var entry = Assert.Single(logger.Entries); + Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, entry.Level); + Assert.Equal(1360205587, entry.EventId.Id); + Assert.Equal("LogDebugDisseminationValueApplyFailed", entry.EventId.Name); + Assert.Same(failure, entry.Exception); + Assert.Equal(sender, entry.State["Sender"]); + Assert.Equal(ns.Name, entry.State["Namespace"]); + Assert.Equal(failedKey, entry.State["Key"]); + Assert.Equal(failedVersion, entry.State["Version"]); + + if (forward) + { + var forwarded = Assert.Single(transport.BroadcastBatches); + Assert.Equal(child, forwarded.Peer); + var forwardedValue = Assert.Single(GetBroadcastValues(forwarded.Batch)); + Assert.Equal(validKey, forwardedValue.Value.Key); + Assert.Equal(validVersion, forwardedValue.Value.ToVersion); + } + else + { + Assert.Empty(transport.BroadcastBatches); + } + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task ReceiveBroadcastPropagatesCallerCancellation() + { + var sender = CreateSilo(33008); + var local = CreateSilo(33009); + var transport = new FakeTransport(local, sender); + var canceledKey = new DisseminationKey("caller-cancellation"); + var laterKey = new DisseminationKey("must-not-apply"); + var applyStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ns = new Phase6ThrowingNamespace(local, canceledKey) + { + Failure = async (_, cancellationToken) => + { + applyStarted.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return DisseminationApplyResult.Applied; + }, + }; + var protocol = CreatePhase6Protocol(transport, ns); + using var cancellation = new CancellationTokenSource(); + + var receive = protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, canceledKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2)), + cancellation.Token); + await applyStarted.Task.WaitAsync(TestContext.Current.CancellationToken); + cancellation.Cancel(); + + var exception = await Assert.ThrowsAnyAsync(() => receive); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal([canceledKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(laterKey)); + Assert.Empty(transport.BroadcastBatches); + } + + [Fact] + public async Task ReceiveBroadcastContinuesWhenApplyDiagnosticObserverThrows() + { + var sender = CreateSilo(33014); + var local = CreateSilo(33015); + var transport = new FakeTransport(local, sender); + var failedKey = new DisseminationKey("throws"); + var validKey = new DisseminationKey("valid"); + var ns = new Phase6ThrowingNamespace(local, failedKey) + { + Failure = static (_, _) => throw new InvalidOperationException("phase 6 apply failure"), + }; + var logger = new Phase6ProtocolLogger(); + var protocol = CreatePhase6Protocol(transport, ns, logger); + using var subscription = DisseminationEvents.Listener.Subscribe( + new ThrowingApplyObserver(ns.Name, failedKey, local, sender), + static name => name == "Dissemination.ValueApply"); + + var response = await protocol.ReceiveBroadcast( + CreateBroadcastBatch( + sender, + ns.CreateItem(sender, failedKey, sequence: 8), + ns.CreateItem(sender, validKey, sequence: 9)), + TestContext.Current.CancellationToken); + + Assert.Equal([failedKey, validKey], ns.ApplyAttempts); + Assert.Equal(9, ns.GetVersion(validKey)); + Assert.Equal(9, response.Acknowledgments[ns.Name].Single(entry => entry.Key == validKey).Version); + Assert.Contains( + logger.Entries, + entry => entry.EventId.Name == "LogDebugDisseminationDiagnosticFailed" + && Equals(entry.State["Namespace"], ns.Name)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReceivingStopsBeforeNextValueWhenCallerIsCanceledAfterApply(bool antiEntropy) + { + var sender = CreateSilo(39001); + var local = CreateSilo(39002); + var transport = new FakeTransport(local, sender); + var cancelingKey = new DisseminationKey("canceling-apply"); + var laterKey = new DisseminationKey("must-not-apply"); + using var cancellation = new CancellationTokenSource(); + CancellationToken applyCancellation = default; + var ns = new Phase6ThrowingNamespace(local, cancelingKey) + { + Failure = (_, cancellationToken) => + { + applyCancellation = cancellationToken; + cancellation.Cancel(); + return ValueTask.FromResult(DisseminationApplyResult.Applied); + }, + }; + var values = CreateValueGroups( + ns.CreateItem(sender, cancelingKey, sequence: 1), + ns.CreateItem(sender, laterKey, sequence: 2)); + transport.ExchangeAntiEntropyHandler = (_, _, _) => ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = sender, + Values = values, + }); + var protocol = CreateProtocol(transport, [ns]); + + try + { + var exception = await Assert.ThrowsAnyAsync( + () => antiEntropy + ? protocol.RunAntiEntropyRound(cancellation.Token) + : protocol.ReceiveBroadcast(new() { Sender = sender, Values = values }, cancellation.Token)); + + Assert.True(cancellation.IsCancellationRequested); + Assert.True(applyCancellation.IsCancellationRequested); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal([cancelingKey], ns.ApplyAttempts); + Assert.Equal(0, ns.GetVersion(laterKey)); + Assert.Empty(transport.BroadcastBatches); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MembershipDuplicatesStopForwardingWhileNewLivenessStillPropagates(bool duplicateDuringSend) + { + var sender = CreateSilo(39021); + var local = CreateSilo(39022); + var child = CreateSilo(39023); + SiloAddress[] members = [sender, local, child]; + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var manager = new FakeMembershipManager(CreateSnapshot(1)); + var ns = CreateMembershipNamespace(manager, serializer); + var transport = new FakeTransport(local, sender, child); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sends = 0; + transport.SendBroadcastResponseHandler = async (peer, batch, cancellationToken) => + { + lock (transport.BroadcastBatches) + { + transport.BroadcastBatches.Add((peer, batch)); + } + + if (Interlocked.Increment(ref sends) == 1) + { + sendStarted.TrySetResult(); + if (duplicateDuringSend) + { + await releaseSend.Task.WaitAsync(cancellationToken); + } + } + + return new DisseminationBroadcastResponse + { + Acknowledgments = FakeTransport.CreateAcknowledgment(batch).Acknowledgments, + AllVersionsAcknowledged = true, + }; + }; + var protocol = CreateProtocol( + transport, + [ns], + options => options.Overlay.FanOutFactor = static _ => 1, + new FakeTimeProvider()); + + try + { + var original = CreateBatch(1); + await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + var firstFlush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + if (duplicateDuringSend) + { + await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + } + + releaseSend.TrySetResult(); + await firstFlush; + var acknowledgment = await protocol.ReceiveBroadcast(original, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(1, Assert.Single(acknowledgment.Acknowledgments[ns.Name]).Version); + Assert.Equal(child, Assert.Single(transport.BroadcastBatches).Peer); + + var updated = CreateBatch(2); + await protocol.ReceiveBroadcast(updated, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await protocol.ReceiveBroadcast(updated, TestContext.Current.CancellationToken); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Equal(2, Volatile.Read(ref sends)); + var forwarded = Assert.Single(GetBroadcastValues(transport.BroadcastBatches[1].Batch)); + var update = Assert.IsType( + serializer.Deserialize(forwarded.Value.Payload)); + Assert.Null(update.Snapshot); + var delta = Assert.IsType(update.Delta); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(2), + Assert.Single(delta.UpdatedEntries, entry => entry.SiloAddress.Equals(sender)).IAmAliveTime); + Assert.Equal(1, forwarded.Value.ToVersion); + } + finally + { + releaseSend.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + + MembershipTableSnapshot CreateSnapshot(int aliveSeconds) => + CreateMembershipSnapshot(1, members.Select(member => CreateMembershipEntry( + member, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(aliveSeconds))).ToArray()); + + DisseminationBroadcastBatch CreateBatch(int aliveSeconds) => new() + { + Sender = sender, + Values = CreateValueGroups(ns.Name, new DisseminationBroadcastValue + { + Value = new DisseminationValue( + DisseminationKey.Default, + 0, + 1, + serializer.SerializeToArray(new MembershipTableSnapshotUpdate { Snapshot = CreateSnapshot(aliveSeconds) })), + TimeToLive = ns.Options.StaleItemTtl, + }), + }; + } + + [Fact] + public async Task PeerQueueAdmitsNewKeyAfterOversizedRepair() + { + var (_, peer, transport, ns) = CreatePeerFixture(39003, 39004); + ns.Options.MaxPendingItemCount = 1; + ns.Options.MaxPayloadBytes = sizeof(long); + ns.PublishValue(new DisseminationValue("oversized", 0, 1, new byte[sizeof(long) + 1])); + ns.SetValue("valid", version: 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + + try + { + queue.Notify(peer, ns, "oversized"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Empty(transport.BroadcastBatches); + + queue.Notify(peer, ns, "valid"); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + var batch = Assert.Single(transport.BroadcastBatches); + var value = Assert.Single(GetBroadcastValues(batch.Batch)); + Assert.Equal(new DisseminationKey("valid"), value.Value.Key); + Assert.Equal(1, value.Value.ToVersion); + } + finally + { + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task DisablingDisseminationStopsPreviouslyQueuedBroadcasts() + { + var (_, _, transport, ns) = CreatePeerFixture(39025, 39026); + DisseminationOptions? options = null; + var protocol = CreateProtocol( + transport, ns, value => options = value, timeProvider: new FakeTimeProvider()); + + try + { + var publication = BeforeBroadcastPumpsRun(() => + { + var pending = PublishValue(protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken); + Assert.NotNull(options); + options.Enabled = false; + return pending; + }); + Assert.True(await publication); + + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + Assert.Empty(transport.BroadcastBatches); + } + finally + { + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublishReturnsFalseWhenPeerQueueRejectsNewKey() + { + var (_, _, transport, ns) = CreatePeerFixture(39005, 39006); + ns.Options.MaxPendingItemCount = 1; + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastHandler = async (_, _, cancellationToken) => + { + sendStarted.TrySetResult(); + await releaseSend.Task.WaitAsync(cancellationToken); + }; + var protocol = CreateProtocol(transport, ns, timeProvider: new FakeTimeProvider()); + + try + { + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", 1), TestContext.Current.CancellationToken)); + var flush = protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.False(await PublishValue(protocol, ns, ns.CreateValue("second", 1), TestContext.Current.CancellationToken)); + Assert.True(await PublishValue(protocol, ns, ns.CreateValue("first", 2), TestContext.Current.CancellationToken)); + releaseSend.TrySetResult(); + await flush; + } + finally + { + releaseSend.TrySetResult(); + await protocol.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task BroadcastDeadlineCompletesFlushBeforeRemoteCancellationAcknowledgment() + { + var (_, peer, transport, ns) = CreatePeerFixture(39011, 39012); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.SetValue("value", version: 1); + var clock = new FakeTimeProvider(); + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.SendBroadcastResponseHandler = (_, _, cancellationToken) => + { + sendStarted.TrySetResult(cancellationToken); + return response.Task; + }; + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: clock); + + try + { + queue.Notify(peer, ns, "value"); + var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + + clock.Advance(ns.Options.StaleItemTtl); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(sendCancellation.IsCancellationRequested); + Assert.False(response.Task.IsCompleted); + ns.Options.Enabled = false; + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.False(response.Task.IsCompleted); + } + finally + { + response.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, true)] + [InlineData(32, false)] + public async Task TimedOutBroadcastReleasesLocalSlotBeforeRpcCompletes(int maxConcurrentSends, bool lateFault) + { + var (local, peer, transport, ns) = CreatePeerFixture(39013, 39014); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.SetValue("value", version: 1); + var clock = new FakeTimeProvider(); + var response = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sendCount = 0; + var sentVersions = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (_, batch, cancellationToken) => + { + var attempt = Interlocked.Increment(ref sendCount); + sentVersions.Enqueue(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + sendStarted.TrySetResult(cancellationToken); + return attempt == 1 ? response.Task : Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxConcurrentSends = maxConcurrentSends, clock); + + try + { + Assert.True(queue.Notify(peer, ns, "value")); + var sendCancellation = await sendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + clock.Advance(ns.Options.StaleItemTtl); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(sendCancellation.IsCancellationRequested); + + ns.SetValue("value", 2); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, Volatile.Read(ref sendCount)); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal(new long[] { 1, 2 }, sentVersions.ToArray()); + Assert.False(response.Task.IsCompleted); + + if (lateFault) + { + response.SetException(new InvalidOperationException("The previous attempt failed after its local deadline.")); + } + else + { + response.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 99)] }, + }); + } + + ns.SetValue("value", 3); + Assert.True(queue.Notify(peer, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(3, Volatile.Read(ref sendCount)); + Assert.Equal(new long[] { 1, 2, 3 }, sentVersions.ToArray()); + } + finally + { + ns.Options.Enabled = false; + response.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task PublishReturnsFalseAfterBroadcastQueueStops() + { + var (_, _, transport, ns) = CreatePeerFixture(39007, 39008); + var protocol = CreateProtocol(transport, ns); + await protocol.StopAsync(TestContext.Current.CancellationToken); + + Assert.False(await PublishValue(protocol, ns, ns.CreateValue("value", 1), TestContext.Current.CancellationToken)); + Assert.Empty(transport.BroadcastBatches); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BroadcastNotificationDiagnosticsRunOutsideQueueLock(bool batch) + { + var (local, peer, transport, ns) = CreatePeerFixture(39009, 39010); + ns.SetValue("value", version: 1); + var queue = CreateBroadcastQueue(transport, [ns], timeProvider: new FakeTimeProvider()); + var callbackEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseCallback = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var observer = DisseminationEvents.Listener.Subscribe( + new QueueReentryObserver(local, () => + { + callbackEntered.TrySetResult(); + releaseCallback.Task.GetAwaiter().GetResult(); + }), + static name => name == DisseminationEvents.BroadcastScheduledEventName); + var notification = Task.Run( + () => batch + ? queue.NotifyBatch(peer, ns, [new("value", 1, true)]) + : queue.Notify(peer, ns, "value"), + TestContext.Current.CancellationToken); + + try + { + await callbackEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await Task.Run(() => queue.ObservePeerVersion(peer, ns.Name, "value", 0), TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + finally + { + releaseCallback.TrySetResult(); + Assert.True(await notification); + await queue.StopAsync(TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BroadcastLocalCompletionReleasesCapacityAcrossPeerRemovalAndRecreation(bool expireBeforePrune) + { + var local = CreateSilo(39301); + var blocked = CreateSilo(39302); + var healthy = CreateSilo(39303); + var restarted = SiloAddress.New(blocked.Endpoint, blocked.Generation + 1); + var transport = new FakeTransport(local, blocked, healthy); + var ns = new FakeNamespace(local); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.SetValue("value", 1); + var clock = new FakeTimeProvider(); + var blockedResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var blockedStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sends = new ConcurrentDictionary(); + var blockedVersions = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (peer, batch, cancellation) => + { + var attempt = sends.AddOrUpdate(peer, 1, static (_, count) => count + 1); + if (peer.Equals(blocked)) + { + blockedVersions.Enqueue(Assert.Single(GetBroadcastValues(batch)).Value.ToVersion); + if (attempt == 1) + { + blockedStarted.TrySetResult(cancellation); + return blockedResponse.Task; + } + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyInterval = TimeSpan.FromHours(1); + }, clock); + var membership = new DisseminationMembership( + transport.MembershipManager, new FakeLocalSiloDetails(local), Options.Create(new DisseminationOptions())); + using var admission = new SendGateObserver(local, "broadcast"); + + try + { + Assert.True(queue.Notify(blocked, ns, "value")); + var remoteCancellation = await blockedStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var initialFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var healthyQueued = admission.Wait(healthy, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(healthy, ns, "value")); + await healthyQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var healthyFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(1, sends[blocked]); + Assert.False(sends.ContainsKey(healthy)); + Assert.Equal(1, ns.RepairRequestCount); + if (expireBeforePrune) + { + clock.Advance(ns.Options.StaleItemTtl); + await Task.WhenAll(initialFlush, healthyFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + transport.Peers.Remove(blocked); + await queue.Prune(membership.CurrentSnapshots, TestContext.Current.CancellationToken); + await Task.WhenAll(initialFlush, healthyFlush).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(remoteCancellation.IsCancellationRequested); + Assert.False(blockedResponse.Task.IsCompleted); + Assert.Equal(1, sends[healthy]); + + transport.Peers.Add(blocked); + Assert.True(queue.Notify(blocked, ns, "value")); + transport.Peers.Add(restarted); + Assert.True(queue.Notify(restarted, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, sends[blocked]); + Assert.Equal(1, sends[restarted]); + Assert.Equal(2, transport.GetTargetResolutionCount(blocked)); + Assert.Equal(4, ns.RepairRequestCount); + Assert.False(blockedResponse.Task.IsCompleted); + + blockedResponse.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 99)] }, + }); + ns.SetValue("value", 2); + Assert.True(queue.Notify(blocked, ns, "value")); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(3, sends[blocked]); + Assert.Equal(5, sends.Values.Sum()); + Assert.Equal(5, ns.RepairRequestCount); + Assert.Equal(new long[] { 1, 1, 2 }, blockedVersions.ToArray()); + } + finally + { + ns.Options.Enabled = false; + blockedResponse.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task CancelingFlushObserverPreservesQueuedBroadcastAttempt() + { + var local = CreateSilo(39331); + var first = CreateSilo(39332); + var waiting = CreateSilo(39333); + var transport = new FakeTransport(local, first, waiting); + var ns = new FakeNamespace(local); + ns.SetValue("value", 1); + var firstResponse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sent = new ConcurrentQueue(); + transport.SendBroadcastResponseHandler = (peer, batch, cancellationToken) => + { + sent.Enqueue(peer); + if (peer.Equals(first)) + { + firstStarted.TrySetResult(cancellationToken); + return firstResponse.Task; + } + + return Task.FromResult(FakeTransport.CreateAcknowledgment(batch)); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => options.MaxConcurrentSends = 1, new FakeTimeProvider()); + using var admission = new SendGateObserver(local, "broadcast"); + using var flushCancellation = new CancellationTokenSource(); + + try + { + Assert.True(queue.Notify(first, ns, "value")); + var firstFlush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + var sendCancellation = await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + var queued = admission.Wait(waiting, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(waiting, ns, "value")); + var canceledFlush = queue.FlushPendingBroadcast(flushCancellation.Token); + await queued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await flushCancellation.CancelAsync(); + var exception = await Assert.ThrowsAnyAsync( + () => canceledFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(flushCancellation.Token, exception.CancellationToken); + Assert.False(sendCancellation.IsCancellationRequested); + Assert.Equal(new[] { first }, sent.ToArray()); + Assert.Equal(1, ns.RepairRequestCount); + + firstResponse.SetResult(new DisseminationBroadcastResponse + { + Acknowledgments = new() { [ns.Name] = [new("value", 1)] }, + }); + await firstFlush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(new[] { first, waiting }, sent.ToArray()); + Assert.Equal(2, ns.RepairRequestCount); + } + finally + { + ns.Options.Enabled = false; + firstResponse.TrySetResult(new DisseminationBroadcastResponse()); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task WaitingBroadcastMaterializesLatestValueAfterAdmissionAndRotatesBehindReadyPeer() + { + var local = CreateSilo(39311); + var large = CreateSilo(39312); + var waiting = CreateSilo(39313); + var transport = new FakeTransport(local, large, waiting); + var ns = new FakeNamespace(local); + ns.SetValue("large-first", 1); + ns.SetValue("large-second", 2); + ns.SetValue("large-third", 3); + ns.SetValue("waiting", 1); + var clock = new RecordingFakeTimeProvider(); + var firstStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var waitingStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseWaiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sent = new ConcurrentQueue<(SiloAddress Peer, DisseminationKey Key, long Version)>(); + transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + var value = Assert.Single(GetBroadcastValues(batch)).Value; + sent.Enqueue((peer, value.Key, value.ToVersion)); + if (value.Key == new DisseminationKey("large-first")) + { + firstStarted.TrySetResult(); + await releaseFirst.Task.WaitAsync(cancellation); + } + else if (peer.Equals(waiting)) + { + waitingStarted.TrySetResult(); + await releaseWaiting.Task.WaitAsync(cancellation); + } + + return FakeTransport.CreateAcknowledgment(batch); + }; + var queue = CreateBroadcastQueue(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.MaxBatchItems = 1; + }, clock); + using var admission = new SendGateObserver(local, "broadcast"); + + try + { + Assert.True(queue.Notify(large, ns, "large-first")); + await firstStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(queue.Notify(large, ns, "large-second")); + Assert.True(queue.Notify(large, ns, "large-third")); + var waitingQueued = admission.Wait(waiting, "queued", TestContext.Current.CancellationToken); + Assert.True(queue.Notify(waiting, ns, "waiting")); + var flush = queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + await waitingQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(1, ns.RepairRequestCount); + Assert.Equal(0, transport.GetTargetResolutionCount(waiting)); + + ns.SetValue("waiting", 7); + var largeQueued = admission.Wait(large, "queued", TestContext.Current.CancellationToken); + releaseFirst.TrySetResult(); + await waitingStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await largeQueued.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(2, ns.RepairRequestCount); + Assert.Equal( + new[] { (large, new DisseminationKey("large-first"), 1L), (waiting, new DisseminationKey("waiting"), 7L) }, + sent.ToArray()); + releaseWaiting.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.Equal( + new[] + { + (large, new DisseminationKey("large-first"), 1L), + (waiting, new DisseminationKey("waiting"), 7L), + (large, new DisseminationKey("large-second"), 2L), + (large, new DisseminationKey("large-third"), 3L), + }, + sent.ToArray()); + Assert.Equal(4, ns.RepairRequestCount); + await queue.FlushPendingBroadcast(TestContext.Current.CancellationToken); + Assert.Equal(4, sent.Count); + } + finally + { + ns.Options.Enabled = false; + releaseFirst.TrySetResult(); + releaseWaiting.TrySetResult(); + await queue.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RepairAttemptsReleaseLocalCapacityBeforeLateCompletion(bool cancelRound) + { + var local = CreateSilo(39321); + var blocked = Enumerable.Range(39322, 3).Select(CreateSilo).ToArray(); + var healthy = CreateSilo(39325); + var transport = new FakeTransport(local, blocked); + var ns = new FakeNamespace(local); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + ns.Options.StaleItemTtl = TimeSpan.FromSeconds(1); + ns.Options.ExpectedUpdateCadence = TimeSpan.FromSeconds(1); + var clock = new FakeTimeProvider(); + var responses = blocked.ToDictionary(static peer => peer, + static _ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + var remoteCancellations = new ConcurrentDictionary(); + var attempts = new ConcurrentDictionary(); + var allStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.ExchangeAntiEntropyHandler = (peer, _, cancellation) => + { + var attempt = attempts.AddOrUpdate(peer, 1, static (_, count) => count + 1); + if (attempt == 1 && responses.TryGetValue(peer, out var response)) + { + remoteCancellations[peer] = cancellation; + if (remoteCancellations.Count == blocked.Length) + { + allStarted.TrySetResult(); + } + + return new(response.Task); + } + + return ValueTask.FromResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.CreateItem(peer, FakeNamespace.DefaultKey, peer.Equals(healthy) ? 3 : 2)), + }); + }; + var protocol = CreateProtocol(transport, ns, options => + { + options.MaxConcurrentSends = 1; + options.Overlay.AntiEntropyPeerCount = 3; + options.Overlay.FanOutFactor = static _ => 8; + }, clock); + using var admission = new SendGateObserver(local, "repair"); + using var roundCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + try + { + var released = blocked.Select(peer => admission.Wait(peer, "released", TestContext.Current.CancellationToken)).ToArray(); + var firstRound = protocol.RunAntiEntropyRound(roundCancellation.Token); + await allStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + for (var round = 0; round < 3; round++) + { + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + Assert.False(firstRound.IsCompleted); + Assert.Equal(3, transport.AntiEntropyRequests.Count); + Assert.All(blocked, peer => Assert.Single(transport.AntiEntropyRequests, request => request.Peer.Equals(peer))); + Assert.All(remoteCancellations.Values, static cancellation => Assert.False(cancellation.IsCancellationRequested)); + + Assert.True(await PublishValue( + protocol, ns, ns.CreateValue(FakeNamespace.DefaultKey, 1), TestContext.Current.CancellationToken)); + await protocol.FlushPendingBroadcast(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.Equal(blocked.Order(), transport.BroadcastBatches.Select(static batch => batch.Peer).Order()); + Assert.False(firstRound.IsCompleted); + + if (cancelRound) + { + await roundCancellation.CancelAsync(); + await Assert.ThrowsAnyAsync( + () => firstRound.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + else + { + clock.Advance(ns.Options.StaleItemTtl); + await firstRound.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + + await Task.WhenAll(released).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.All(remoteCancellations.Values, static cancellation => Assert.True(cancellation.IsCancellationRequested)); + Assert.All(responses.Values, static response => Assert.False(response.Task.IsCompleted)); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 1)); + + clock.Advance(ns.Options.StaleItemTtl); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 2)); + Assert.Equal(6, transport.AntiEntropyRequests.Count); + Assert.All(blocked, peer => Assert.Equal(2, attempts[peer])); + Assert.All(responses.Values, static response => Assert.False(response.Task.IsCompleted)); + + transport.Peers.Clear(); + transport.Peers.Add(healthy); + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(7, transport.AntiEntropyRequests.Count); + Assert.Equal(healthy, transport.AntiEntropyRequests[^1].Peer); + + foreach (var (peer, response) in responses) + { + response.SetResult(new DisseminationAntiEntropyResponse + { + Sender = peer, + Values = CreateValueGroups(ns.CreateItem(peer, FakeNamespace.DefaultKey, 99)), + }); + } + + await protocol.RunAntiEntropyRound(TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + AssertFakeState(ns, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, ns.ApplyCounts[FakeNamespace.DefaultKey]); + } + finally + { + ns.Options.Enabled = false; + await roundCancellation.CancelAsync(); + foreach (var response in responses.Values) + { + response.TrySetCanceled(TestContext.Current.CancellationToken); + } + + await protocol.StopAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + } + + [Fact] + public async Task AntiEntropyConvergesAfterPartitionWithEveryTreeBroadcastDropped() + { + var members = Enumerable.Range(39201, 4).Select(CreateSilo).ToArray(); + DisseminationKey[] keys = ["first", "second", "third", "fourth"]; + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.UnionWith(keys); + return ns; + }); + network.Partitioned = true; + + for (var i = 0; i < members.Length; i++) + { + var node = network[members[i]]; + var ns = Assert.IsType(node.Namespace); + Assert.True(await PublishValue(node.Protocol, ns, ns.CreateValue(keys[i], i + 1), network.Cancellation)); + } + + await network.WaitForBroadcastRetries(members, network.Cancellation); + Assert.Equal(12, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); + await network.RunSweep(network.Cancellation); + for (var i = 0; i < members.Length; i++) + { + AssertFakeState(Assert.IsType(network[members[i]].Namespace), (keys[i], i + 1)); + } + + // Publications have ceased. Only the repair links heal; the tree remains completely disconnected. + network.Partitioned = false; + await network.RunSweep(network.Cancellation); + await network.RunSweep(network.Cancellation); + var expected = keys.Select((key, index) => (key, (long)index + 1)).ToArray(); + foreach (var member in members) + { + AssertFakeState(Assert.IsType(network[member].Namespace), expected); + } + + var repairCount = network.Replies.Sum(static reply => GetAntiEntropyResponseValues(reply).Count()); + Assert.Equal(24, repairCount); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + Assert.Equal(4, network.Sweeps); + Assert.Equal(48, network.RequestCount); + Assert.Equal(repairCount, network.Replies.Sum(static reply => GetAntiEntropyResponseValues(reply).Count())); + Assert.Equal(12, network.Broadcasts.Count); + } + + [Fact] + public async Task LostBroadcastAcknowledgmentsAndReorderedRepairRepliesConvergeAndDrain() + { + var members = Enumerable.Range(39211, 3).Select(CreateSilo).ToArray(); + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.Add(FakeNamespace.DefaultKey); + return ns; + }); + network.DropTreeBroadcasts = false; + network.LoseAcknowledgments = true; + var source = network[members[0]]; + var older = network[members[1]]; + var receiver = network[members[2]]; + var sourceNamespace = Assert.IsType(source.Namespace); + var olderNamespace = Assert.IsType(older.Namespace); + var receiverNamespace = Assert.IsType(receiver.Namespace); + Assert.True(await PublishValue( + source.Protocol, sourceNamespace, sourceNamespace.CreateValue(FakeNamespace.DefaultKey, 1), network.Cancellation)); + await network.WaitForBroadcastRetries([source.Address], network.Cancellation); + Assert.Equal(2, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.True(send.Delivered)); + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 1)); + network.DropTreeBroadcasts = true; + + sourceNamespace.SetValue(FakeNamespace.DefaultKey, 3); + olderNamespace.SetValue(FakeNamespace.DefaultKey, 2); + var bothRepliesCaptured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var newerReplyReturned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseNewer = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseOlder = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var replies = new ConcurrentDictionary(); + var exchange = receiver.Transport.ExchangeAntiEntropyHandler; + receiver.Transport.ExchangeAntiEntropyHandler = async (peer, request, cancellation) => + { + var reply = await exchange(peer, request, cancellation); + replies[peer] = reply; + if (replies.Count == 2) + { + bothRepliesCaptured.TrySetResult(); + } + + await (peer.Equals(source.Address) ? releaseNewer.Task : releaseOlder.Task).WaitAsync(cancellation); + if (peer.Equals(source.Address)) + { + newerReplyReturned.TrySetResult(); + } + + return reply; + }; + + try + { + network.Clock.Advance(RecoveryNetwork.RepairCadence); + var round = receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + await bothRepliesCaptured.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + Assert.Equal(3, Assert.Single(GetAntiEntropyResponseValues(replies[source.Address])).Value.ToVersion); + Assert.Equal(2, Assert.Single(GetAntiEntropyResponseValues(replies[older.Address])).Value.ToVersion); + releaseNewer.TrySetResult(); + await newerReplyReturned.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + Assert.False(round.IsCompleted); + releaseOlder.TrySetResult(); + await round.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + receiver.Transport.ExchangeAntiEntropyHandler = (peer, _, _) => ValueTask.FromResult(replies[peer]); + network.Clock.Advance(RecoveryNetwork.RepairCadence); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + AssertFakeState(receiverNamespace, (FakeNamespace.DefaultKey, 3)); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + + receiver.Transport.ExchangeAntiEntropyHandler = exchange; + network.LoseAcknowledgments = false; + await network.RunSweep(network.Cancellation); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + foreach (var member in members) + { + AssertFakeState(Assert.IsType(network[member].Namespace), (FakeNamespace.DefaultKey, 3)); + } + + Assert.Equal(16, network.RequestCount); + Assert.Equal(2, network.Broadcasts.Count); + Assert.Equal(2, receiverNamespace.ApplyCounts[FakeNamespace.DefaultKey]); + } + finally + { + releaseNewer.TrySetResult(); + releaseOlder.TrySetResult(); + } + } + + [Fact] + public async Task MembershipRepairConvergesAcrossJoinDepartureAndRestartWithoutRevivingDepartedGeneration() + { + var first = CreateSilo(39221); + var second = CreateSilo(39222); + var departed = CreateSilo(39223); + var joined = CreateSilo(39224); + var restarted = SiloAddress.New(departed.Endpoint, departed.Generation + 1); + SiloAddress[] members = [first, second, departed, joined, restarted]; + var baseline = CreateMembershipSnapshot(1, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(departed, SiloStatus.Active, DateTime.UnixEpoch)); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(static silo => silo, _ => new FakeMembershipManager(baseline)); + await using var network = new RecoveryNetwork( + members, silo => CreateMembershipNamespace(managers[silo], serializer)); + network.SetTopology((first, SiloStatus.Active), (second, SiloStatus.Active), (departed, SiloStatus.Active)); + var joiningSnapshot = CreateMembershipSnapshot(2, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(departed, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(joined, SiloStatus.Joining, DateTime.UnixEpoch.AddSeconds(1))); + managers[first].CurrentSnapshot = joiningSnapshot; + network.SetTopology( + (first, SiloStatus.Active), (second, SiloStatus.Active), + (departed, SiloStatus.Active), (joined, SiloStatus.Joining)); + await network.RunSweep(network.Cancellation); + foreach (var member in new[] { first, second, departed, joined }) + { + AssertMembershipState(joiningSnapshot, managers[member].CurrentSnapshot); + } + + var delayedDepartedValue = network[departed].Namespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, null, 1024 * 1024, 1024 * 1024)).Value; + var restartedSnapshot = CreateMembershipSnapshot(3, + CreateMembershipEntry(first, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(second, SiloStatus.Active, DateTime.UnixEpoch), + CreateMembershipEntry(joined, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(1)), + CreateMembershipEntry(restarted, SiloStatus.Active, DateTime.UnixEpoch.AddSeconds(2))); + managers[first].CurrentSnapshot = restartedSnapshot; + network.SetTopology( + (first, SiloStatus.Active), (second, SiloStatus.Active), + (joined, SiloStatus.Active), (restarted, SiloStatus.Active)); + var requestsBeforeRestart = network.Nodes.ToDictionary(static node => node.Address, static node => node.Transport.AntiEntropyRequests.Count); + await network.RunSweep(network.Cancellation); + var acknowledgment = await network[second].Protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = departed, + Values = CreateValueGroups(network[second].Namespace.Name, CreateDisseminationValue(departed, delayedDepartedValue)), + }, network.Cancellation); + Assert.Equal(3, Assert.Single(acknowledgment.Acknowledgments[network[second].Namespace.Name]).Version); + await network.RunSweep(network.Cancellation); + await network.Flush(network.Cancellation); + + foreach (var member in new[] { first, second, joined, restarted }) + { + AssertMembershipState(restartedSnapshot, managers[member].CurrentSnapshot); + Assert.DoesNotContain(departed, managers[member].CurrentSnapshot.Entries.Keys); + Assert.Contains(restarted, managers[member].CurrentSnapshot.Entries.Keys); + } + + Assert.All(network.Nodes, node => Assert.DoesNotContain( + node.Transport.AntiEntropyRequests.Skip(requestsBeforeRestart[node.Address]), + request => request.Peer.Equals(departed))); + Assert.Equal(3, network.Sweeps); + Assert.Equal(36, network.RequestCount); + Assert.Empty(network.Broadcasts); + } + + [Fact] + public async Task MembershipAntiEntropyConvergesSameVersionLivenessAndTerminatesDuplicates() + { + var members = Enumerable.Range(39231, 3).Select(CreateSilo).ToArray(); + var baseline = CreateMembershipSnapshot(7, members.Select(silo => CreateMembershipEntry( + silo, SiloStatus.Active, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(1))).ToArray()); + var advanced = CreateMembershipSnapshot(7, members.Select(silo => CreateMembershipEntry( + silo, SiloStatus.Active, DateTime.UnixEpoch, + DateTime.UnixEpoch.AddSeconds(silo.Equals(members[0]) ? 2 : 1))).ToArray()); + using var services = new ServiceCollection().AddSerializer().BuildServiceProvider(); + var serializer = services.GetRequiredService(); + var managers = members.ToDictionary(static silo => silo, _ => new FakeMembershipManager(baseline)); + var applied = members.ToDictionary(static silo => silo, static _ => 0); + foreach (var member in members) + { + managers[member].ProcessGossipSnapshotHandler = (snapshot, _) => + { + applied[member]++; + managers[member].CurrentSnapshot = snapshot; + return Task.CompletedTask; + }; + } + + await using var network = new RecoveryNetwork(members, silo => CreateMembershipNamespace(managers[silo], serializer)); + await network.RunSweep(network.Cancellation); + Assert.All(network.Replies, static response => Assert.Empty(response.Values)); + var source = network[members[0]]; + var oldFingerprint = source.Namespace.Digests.Single().Fingerprint; + managers[source.Address].CurrentSnapshot = advanced; + Assert.NotEqual(oldFingerprint, source.Namespace.Digests.Single().Fingerprint); + Assert.True(await source.Protocol.Publish(source.Namespace, DisseminationKey.Default, 7, network.Cancellation)); + await network.WaitForBroadcastRetries([source.Address], network.Cancellation); + await network.RunSweep(network.Cancellation); + foreach (var member in members) + { + AssertMembershipState(advanced, managers[member].CurrentSnapshot); + } + + var repairCount = network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count()); + await network.RunSweep(network.Cancellation); + var broadcastsBeforeDuplicates = network.Broadcasts.ToArray(); + var duplicate = source.Namespace.CreateRepair(new DisseminationRepairRequest( + DisseminationKey.Default, null, 1024 * 1024, 1024 * 1024)).Value; + foreach (var member in members.Skip(1)) + { + var acknowledgment = await network[member].Protocol.ReceiveBroadcast(new DisseminationBroadcastBatch + { + Sender = source.Address, + Values = CreateValueGroups(source.Namespace.Name, CreateDisseminationValue(source.Address, duplicate)), + }, network.Cancellation); + Assert.Equal(7, Assert.Single(acknowledgment.Acknowledgments[source.Namespace.Name]).Version); + AssertMembershipState(advanced, managers[member].CurrentSnapshot); + Assert.Equal(1, applied[member]); + } + + // Flush recipients to isolate duplicate forwarding from the source's intentionally dirty retry queue. + foreach (var member in members.Skip(1)) + { + await network[member].Protocol.FlushPendingBroadcast(network.Cancellation); + } + + Assert.Equal(0, applied[source.Address]); + Assert.Equal(18, network.RequestCount); + Assert.Equal(repairCount, network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count())); + Assert.Equal(broadcastsBeforeDuplicates, network.Broadcasts.ToArray()); + Assert.Equal(2, network.Broadcasts.Count); + Assert.All(network.Broadcasts, static send => Assert.False(send.Delivered)); + } + + [Fact] + public async Task RejectedQueueKeysRecoverThroughTruncatedDigestsDespiteAdvancingFirstKey() + { + var members = Enumerable.Range(39241, 2).Select(CreateSilo).ToArray(); + DisseminationKey[] keys = ["hot", "waiting-first", "waiting-last"]; + await using var network = new RecoveryNetwork(members, silo => + { + var ns = new FakeNamespace(silo); + ns.ExpectedKeys.UnionWith(keys); + ns.Options.MaxPendingItemCount = 1; + return ns; + }, options => options.MaxBatchItems = 1); + var source = network[members[0]]; + var receiver = network[members[1]]; + var sourceNamespace = Assert.IsType(source.Namespace); + var receiverNamespace = Assert.IsType(receiver.Namespace); + var firstSendStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseFirstSend = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var send = source.Transport.SendBroadcastResponseHandler!; + source.Transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + firstSendStarted.TrySetResult(); + await releaseFirstSend.Task.WaitAsync(cancellation); + return await send(peer, batch, cancellation); + }; + + try + { + Assert.True(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[0], 1), network.Cancellation)); + await firstSendStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + var flush = source.Protocol.FlushPendingBroadcast(network.Cancellation); + Assert.False(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[1], 1), network.Cancellation)); + Assert.False(await PublishValue(source.Protocol, sourceNamespace, sourceNamespace.CreateValue(keys[2], 1), network.Cancellation)); + releaseFirstSend.TrySetResult(); + await flush.WaitAsync(TimeSpan.FromSeconds(5), network.Cancellation); + AssertFakeState(receiverNamespace); + + for (var round = 1; round <= 4; round++) + { + sourceNamespace.SetValue(keys[0], round); + network.Clock.Advance(RecoveryNetwork.RepairCadence); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + } + + Assert.Equal(1, receiverNamespace.GetVersion(keys[1])); + Assert.Equal(1, receiverNamespace.GetVersion(keys[2])); + Assert.Equal(keys, receiver.Transport.AntiEntropyRequests[0].Request.Digests[sourceNamespace.Name] + .Select(static digest => digest.Key)); + Assert.All(receiver.Transport.AntiEntropyRequests[0].Request.Digests[sourceNamespace.Name], + static digest => Assert.Equal(0, digest.Version)); + Assert.Equal(new[] { keys[0], keys[1], keys[0], keys[2] }, + network.Replies.Select(static response => Assert.Single(GetAntiEntropyResponseValues(response)).Value.Key)); + Assert.All(network.Replies, static response => Assert.True(response.Truncated)); + + for (var round = 0; round < 2; round++) + { + network.Clock.Advance(RecoveryNetwork.RepairCadence); + await receiver.Protocol.RunAntiEntropyRound(network.Cancellation); + } + + await network.Flush(network.Cancellation); + AssertFakeState(sourceNamespace, (keys[0], 4), (keys[1], 1), (keys[2], 1)); + Assert.Equal(sourceNamespace.GetState(), receiverNamespace.GetState()); + Assert.Equal(6, network.RequestCount); + Assert.Equal(5, network.Replies.Sum(static response => GetAntiEntropyResponseValues(response).Count())); + Assert.Empty(network.Replies.Last().Values); + Assert.Single(network.Broadcasts); + } + finally + { + releaseFirstSend.TrySetResult(); + } + } + + private static void AssertFakeState(FakeNamespace ns, params (DisseminationKey Key, long Version)[] expected) => + Assert.Equal( + expected.OrderBy(static entry => entry.Key.ToString(), StringComparer.Ordinal) + .Select(static entry => (entry.Key, entry.Version, entry.Version)), + ns.GetState()); + + private static void AssertMembershipState(MembershipTableSnapshot expected, MembershipTableSnapshot actual) + { + Assert.Equal(expected.Version, actual.Version); + Assert.Equal(expected.Entries.Keys.Order(), actual.Entries.Keys.Order()); + foreach (var (silo, entry) in expected.Entries) + { + var observed = actual.Entries[silo]; + Assert.Equal( + (entry.SiloAddress, entry.Status, entry.StartTime, entry.IAmAliveTime, entry.ProxyPort, entry.HostName, entry.SiloName, entry.RoleName), + (observed.SiloAddress, observed.Status, observed.StartTime, observed.IAmAliveTime, observed.ProxyPort, observed.HostName, observed.SiloName, observed.RoleName)); + Assert.Equal((entry.UpdateZone, entry.FaultZone), (observed.UpdateZone, observed.FaultZone)); + Assert.Equal(entry.SuspectTimes, observed.SuspectTimes); + } + } + + // Every delivery enters a real protocol. There is deliberately no legacy transport or direct namespace-copy path. + private sealed class RecoveryNetwork : IAsyncDisposable + { + public static readonly TimeSpan RepairCadence = TimeSpan.FromMilliseconds(1); + private readonly Dictionary _nodes = []; + private readonly CancellationTokenSource _cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + private readonly BroadcastScheduleObserver _schedules = new(); + private SiloAddress[] _participants; + + public RecoveryNetwork( + SiloAddress[] members, + Func createNamespace, + Action? configure = null) + { + _participants = members; + foreach (var member in members) + { + var ns = createNamespace(member); + ns.Options.ExpectedUpdateCadence = RepairCadence; + var transport = new FakeTransport(member, members.Where(peer => !peer.Equals(member)).ToArray()); + var protocol = CreateProtocol(transport, [ns], options => + { + options.MaxConcurrentSends = 1; + options.Overlay.FanOutFactor = static _ => 8; + options.Overlay.AntiEntropyPeerCount = members.Length; + options.Overlay.AntiEntropyInterval = TimeSpan.FromHours(1); + configure?.Invoke(options); + }, Clock); + var node = new RecoveryNode(member, ns, transport, protocol); + _nodes.Add(member, node); + transport.SendBroadcastResponseHandler = async (peer, batch, cancellation) => + { + var deliver = !DropTreeBroadcasts && !Partitioned; + Broadcasts.Enqueue((member, peer, deliver)); + if (!deliver) + { + throw new InvalidOperationException("The tree link drops every delivery in this phase."); + } + + var response = await _nodes[peer].Protocol.ReceiveBroadcast(batch, cancellation); + return LoseAcknowledgments ? new DisseminationBroadcastResponse() : response; + }; + transport.ExchangeAntiEntropyHandler = async (peer, request, cancellation) => + { + if (Partitioned) + { + throw new InvalidOperationException("The repair link is partitioned."); + } + + var response = await _nodes[peer].Protocol.ReceiveAntiEntropy(request, cancellation); + Replies.Enqueue(response); + return response; + }; + } + } + + public RecordingFakeTimeProvider Clock { get; } = new(); + public CancellationToken Cancellation => _cancellation.Token; + public IEnumerable Nodes => _nodes.Values; + public RecoveryNode this[SiloAddress silo] => _nodes[silo]; + public ConcurrentQueue<(SiloAddress Sender, SiloAddress Peer, bool Delivered)> Broadcasts { get; } = new(); + public ConcurrentQueue Replies { get; } = new(); + public bool DropTreeBroadcasts { get; set; } = true; + public bool LoseAcknowledgments { get; set; } + public bool Partitioned { get; set; } + public int Sweeps { get; private set; } + public int RequestCount => _nodes.Values.Sum(static node => node.Transport.AntiEntropyRequests.Count); + + public void SetTopology(params (SiloAddress Silo, SiloStatus Status)[] members) + { + _participants = members.Select(static member => member.Silo).ToArray(); + foreach (var node in _nodes.Values) + { + node.Transport.Peers.Clear(); + node.Transport.Peers.AddRange(_participants.Where(peer => !peer.Equals(node.Address))); + node.Transport.PeerStatuses.Clear(); + node.Transport.PeerStatuses[node.Address] = SiloStatus.Dead; + foreach (var (silo, status) in members) + { + node.Transport.PeerStatuses[silo] = status; + } + } + } + + public async Task RunSweep(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + // Repair progresses while the fixed 100 ms broadcast retry timer remains dormant. + Clock.Advance(RepairCadence); + foreach (var member in _participants) + { + await _nodes[member].Protocol.RunAntiEntropyRound(cancellationToken) + .WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + + Sweeps++; + } + + public async Task WaitForBroadcastRetries(SiloAddress[] publishers, CancellationToken cancellationToken) + { + var waits = publishers.SelectMany(publisher => + GetOriginatorTreeTargets(publisher, _nodes[publisher].Transport.Peers, fanout: 8) + .Select(peer => _schedules.WaitAsync( + scheduled => scheduled.LocalSilo.Equals(publisher) && scheduled.Peer.Equals(peer) + && scheduled.Reason == DisseminationBroadcastScheduleReason.Retry, + TimeSpan.FromSeconds(5), cancellationToken))).ToArray(); + try + { + await Task.WhenAll(waits); + } + catch (TimeoutException exception) + { + throw new TimeoutException( + $"Recovery broadcasts did not enter backoff for {string.Join(", ", publishers.Select(static silo => silo.ToString()))}; " + + $"broadcasts={Broadcasts.Count}, repair requests={RequestCount}, sweeps={Sweeps}.", exception); + } + } + + public async Task Flush(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var node in _nodes.Values) + { + await node.Protocol.FlushPendingBroadcast(cancellationToken).WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + public async ValueTask DisposeAsync() + { + _cancellation.Cancel(); + var cleanupToken = _cancellation.Token; + foreach (var node in _nodes.Values) + { + node.Namespace.Options.Enabled = false; + } + + try + { + // Teardown aborts accepted retries; scenarios assert graceful drainage before disposal. + await Task.WhenAll(_nodes.Values.Select(StopNode)) + .WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + } + finally + { + _schedules.Dispose(); + _cancellation.Dispose(); + } + + async Task StopNode(RecoveryNode node) + { + try + { + await node.Protocol.StopAsync(cleanupToken); + } + catch (OperationCanceledException exception) when (exception.CancellationToken == cleanupToken) + { + } + } + } + } + + private sealed record RecoveryNode( + SiloAddress Address, + IDisseminationNamespace Namespace, + FakeTransport Transport, + DisseminationProtocol Protocol); + + private sealed class SendGateObserver : IObserver>, IDisposable + { + private readonly SiloAddress _local; + private readonly string _kind; + private readonly object _lock = new(); + private readonly List<(SiloAddress Peer, string Stage)> _events = []; + private readonly List<(SiloAddress Peer, string Stage, TaskCompletionSource Completion)> _waiters = []; + private readonly IDisposable _subscription; + + public SendGateObserver(SiloAddress local, string kind) + { + _local = local; + _kind = kind; + _subscription = DisseminationEvents.Listener.Subscribe( + this, static name => name == DisseminationEvents.SendGateEventName); + } + + public Task Wait(SiloAddress peer, string stage, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + var index = _events.FindIndex(entry => entry.Peer.Equals(peer) && entry.Stage == stage); + if (index >= 0) + { + _events.RemoveAt(index); + return Task.CompletedTask; + } + + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Add((peer, stage, completion)); + return completion.Task.WaitAsync(cancellationToken); + } + } + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationSendGateEvent transition && transition.Kind == _kind) + { + OnTransition(transition.LocalSilo, transition.Peer, transition.Stage); + } + } + + private void OnTransition(SiloAddress local, SiloAddress peer, string stage) + { + if (!local.Equals(_local)) + { + return; + } + + TaskCompletionSource? completion = null; + lock (_lock) + { + var index = _waiters.FindIndex(waiter => waiter.Peer.Equals(peer) && waiter.Stage == stage); + if (index >= 0) + { + completion = _waiters[index].Completion; + _waiters.RemoveAt(index); + } + else + { + _events.Add((peer, stage)); + } + } + + completion?.TrySetResult(); + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + private sealed class QueueReentryObserver(SiloAddress localSilo, Action callback) : IObserver> + { + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationBroadcastScheduledEvent scheduled && Equals(scheduled.LocalSilo, localSilo)) + { + callback(); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + } + + private static DisseminationProtocol CreatePhase6Protocol( + FakeTransport transport, + IDisseminationNamespace disseminationNamespace, + Microsoft.Extensions.Logging.ILogger? logger = null, + Action? configure = null) => + CreateProtocol(transport, [disseminationNamespace], configure, logger: logger); + + private sealed class Phase6ThrowingNamespace( + SiloAddress localSilo, + DisseminationKey failingKey) : IDisseminationNamespace + { + private readonly FakeNamespace _inner = new(localSilo); + + public Func> Failure { get; init; } = + static (_, _) => throw new InvalidOperationException("A failure must be configured."); + + public List ApplyAttempts { get; } = []; + + public DisseminationNamespace Name => _inner.Name; + + public DisseminationNamespaceOptions Options => _inner.Options; + + public IEnumerable Digests => _inner.Digests; + + public DisseminationValue CreateValue(DisseminationKey key, long sequence) => + _inner.CreateValue(key, sequence); + + public DisseminationBroadcastValue CreateItem( + SiloAddress originator, + DisseminationKey key, + long sequence) => + _inner.CreateItem(originator, key, sequence); + + public long GetVersion(DisseminationKey key) => _inner.GetVersion(key); + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) => + _inner.CreateRepair(request); + + public ValueTask ApplyValueAsync( + DisseminationValue value, + CancellationToken cancellationToken) + { + ApplyAttempts.Add(value.Key); + return value.Key == failingKey + ? Failure(value, cancellationToken) + : _inner.ApplyValueAsync(value, cancellationToken); + } + } + + private sealed class Phase6ApplyObserver : IObserver>, IDisposable + { + private readonly DisseminationNamespace _namespace; + private readonly DisseminationKey _key; + private readonly SiloAddress _localSilo; + private readonly SiloAddress _sender; + private readonly IDisposable _subscription; + + public Phase6ApplyObserver( + DisseminationNamespace @namespace, + DisseminationKey key, + SiloAddress localSilo, + SiloAddress sender) + { + _namespace = @namespace; + _key = key; + _localSilo = localSilo; + _sender = sender; + _subscription = DisseminationEvents.Listener.Subscribe( + this, + static name => name == "Dissemination.ValueApply"); + } + + public List Events { get; } = []; + + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent apply + && apply.Namespace == _namespace + && apply.Key == _key + && Equals(apply.LocalSilo, _localSilo) + && Equals(apply.Peer, _sender) + && apply.Result == nameof(DisseminationApplyResult.Rejected)) + { + Events.Add(apply); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + + public void Dispose() => _subscription.Dispose(); + } + + private sealed class Phase6ProtocolLogger : Microsoft.Extensions.Logging.ILogger + { + public List Entries { get; } = []; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + + public void Log( + Microsoft.Extensions.Logging.LogLevel logLevel, + Microsoft.Extensions.Logging.EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (exception is null) + { + return; + } + + var properties = ((IEnumerable>)(object)state!) + .ToDictionary(static pair => pair.Key, static pair => pair.Value); + Entries.Add(new(logLevel, eventId, exception, properties)); + } + } + + private sealed record Phase6LogEntry( + Microsoft.Extensions.Logging.LogLevel Level, + Microsoft.Extensions.Logging.EventId EventId, + Exception Exception, + Dictionary State); + + private sealed class ThrowingApplyObserver( + DisseminationNamespace namespaceName, + DisseminationKey key, + SiloAddress localSilo, + SiloAddress sender) : IObserver> + { + public void OnNext(KeyValuePair value) + { + if (value.Value is DisseminationValueEvent apply + && apply.Namespace == namespaceName + && apply.Key == key + && Equals(apply.LocalSilo, localSilo) + && Equals(apply.Peer, sender)) + { + throw new InvalidOperationException("Diagnostic observer failure."); + } + } + + public void OnCompleted() + { + } + + public void OnError(Exception error) + { + } + } +} + +#if NET10_0_OR_GREATER +[State] +public partial class MonotonicDisseminationState +{ + public long Version { get; set; } +} + +public enum ModelApplyResult +{ + Applied, + Duplicate, + Obsolete, + Rejected, +} + +public sealed record ModelApplyResponse(ModelApplyResult Result, long Version); + +public sealed record ModelRepairResponse(bool HasValue, long Version); +#endif diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs new file mode 100644 index 00000000000..bce80ac1cd9 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/DisseminationRootBatcherTests.cs @@ -0,0 +1,1361 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Net; +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using Orleans; +using Orleans.Configuration; +using Orleans.Runtime; +using Orleans.Runtime.Dissemination; +using Xunit; +using KeyNotification = Orleans.Runtime.Dissemination.DisseminationBroadcastQueue.KeyNotification; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class DisseminationRootBatcherTests +{ + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + private static TimeSpan Period => TimeSpan.FromSeconds(1); + private static TimeSpan Milliseconds(int value) => TimeSpan.FromMilliseconds(value); + + [Fact] + public async Task SingleRootContributionSealsWithoutArmingCollectionTimer() + { + await using var rig = new TestRig(1); + Assert.Equal(new(true, Period), await Phase(rig.Publish(0), "single-silo cohort includes the local root")); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + Assert.Equal(0, rig.Clock.ScheduleCount); + } + + [Fact] + public async Task AllActiveIncarnationsIncludingRootSealEarlyAtOriginalNextPeriod() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(1); + rig.Clock.Advance(Milliseconds(10)); + var second = rig.Publish(2); + rig.Clock.Advance(Milliseconds(15)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); + Assert.Empty(rig.Attempts); + + var local = rig.Publish(0); + var receipts = await Phase(Task.WhenAll(first, second, local), "all three producer receipts"); + + Assert.All(receipts, receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal( + new KeyNotification[] { rig.Notification(1), rig.Notification(2), rig.Notification(0) }, + Assert.Single(rig.Attempts).Notifications); + Assert.Equal(1, rig.Clock.ScheduleCount); + Assert.Empty(rig.Namespace.VersionReads); + } + + [Fact] + public async Task MissingProducerTimesOutAtFirstPendingDeadlineDespiteUpdates() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(400)); + var newer = rig.Publish(0, 9); + var peer = rig.Publish(1); + rig.Clock.Advance(Milliseconds(599)); + Assert.False(first.IsCompleted); + Assert.False(newer.IsCompleted); + Assert.False(peer.IsCompleted); + Assert.Empty(rig.Attempts); + + rig.Clock.Advance(Milliseconds(1)); + var receipts = await Phase(Task.WhenAll(first, newer, peer), "partial cohort at its original deadline"); + + Assert.All(receipts, receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.Equal(new[] { rig.Notification(0, 9), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); + Assert.Equal(1, rig.Clock.ScheduleCount); + } + + [Fact] + public async Task ReceiptDelaySubtractsDispatchTimeInsteadOfStartingAnotherPeriod() + { + await using var rig = new TestRig(2); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Receipt dispatch was not released."); + return true; + }; + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(25)); + var second = rig.Publish(1); + try + { + await rig.NextAttemptAsync(); + rig.Clock.Advance(Milliseconds(75)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); + } + finally + { + release.Set(); + } + + Assert.All(await Phase(Task.WhenAll(first, second), "dispatch time deducted from receipt delay"), + receipt => Assert.Equal(new(true, Milliseconds(900)), receipt)); + } + + [Fact] + public async Task InventoryHintsAndDuplicateVersionsNeverFillMissingContributions() + { + await using var rig = new TestRig(); + await rig.NotifyAsync(rig.Members.Select(silo => new KeyNotification(silo, 10, true)).ToArray()); + var first = rig.Publish(0, 3); + var duplicate = rig.Publish(0, 3); + var older = rig.Publish(0, 1); + var second = rig.Publish(1); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(first.IsCompleted); + Assert.False(second.IsCompleted); + Assert.Empty(rig.Attempts); + + rig.Clock.Advance(Milliseconds(1)); + Assert.All(await Phase(Task.WhenAll(first, duplicate, older, second), "hints do not count"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.All(Assert.Single(rig.Attempts).Notifications, notification => + { + Assert.Equal(10, notification.Version); + Assert.True(notification.Force); + }); + Assert.Empty(rig.Namespace.VersionReads); + } + + [Fact] + public async Task HintBeforeFirstProducerAnchorsCohortDeadline() + { + await using var rig = new TestRig(); + await rig.NotifyAsync(rig.Notification(1)); + rig.Clock.Advance(Milliseconds(400)); + var first = rig.Publish(0); + rig.Clock.Advance(Milliseconds(599)); + Assert.False(first.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(first, "deadline remains anchored to first pending hint")); + Assert.Equal(1, rig.Clock.ScheduleCount); + Assert.Equal(new[] { rig.Notification(1), rig.Notification(0) }, Assert.Single(rig.Attempts).Notifications); + } + + [Fact] + public async Task SealedRetriesReuseOutcomeAndRemainingDelayWithoutAnotherCohort() + { + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Milliseconds(25)); + var peer = rig.Publish(1); + Assert.Equal(new(true, Milliseconds(975)), await Phase(first, "first early receipt")); + await Phase(peer, "second early receipt"); + rig.Clock.Advance(Milliseconds(75)); + + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 2)); + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 1)); + Assert.Equal(new(true, Milliseconds(900)), await rig.Publish(0, 2, force: true)); + rig.Clock.Advance(TimeSpan.FromHours(1)); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(0, 2)); + Assert.Single(rig.Attempts); + Assert.Equal(1, rig.Clock.ScheduleCount); + } + + [Fact] + public async Task NewCohortRequiresNewVersionsFromEveryProducer() + { + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0); + var peer = rig.Publish(1); + await Phase(Task.WhenAll(first, peer), "first complete cohort"); + rig.Clock.Advance(Period); + + var next = await rig.StartPublicationAsync(0, 2); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(1)); + rig.Clock.Advance(Milliseconds(25)); + Assert.False(next.IsCompleted); + Assert.Single(rig.Attempts); + var fresh = rig.Publish(1, 2); + Assert.All(await Phase(Task.WhenAll(next, fresh), "new versions fill the next cohort"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(2, rig.Attempts.Count); + } + + [Fact] + public async Task LateTimerDoesNotCreateEmptyOrCatchupWaves() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(TimeSpan.FromSeconds(5)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(first, "late timer receipt")); + await Phase(rig.Batcher.FlushAsync(TestToken), "late cohort reconciled"); + Assert.Equal(new(true, TimeSpan.Zero), await rig.Publish(0)); + rig.Clock.Advance(TimeSpan.FromDays(1)); + Assert.Single(rig.Attempts); + Assert.Equal(1, rig.Clock.TimerCount); + + var second = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(second.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(second, "new cohort gets its own full period")); + Assert.Equal(2, rig.Attempts.Count); + } + + [Fact] + public async Task CanceledCallerDoesNotEraseContributionOrCancelSharedReceipt() + { + await using var rig = new TestRig(2); + using var cancellation = new CancellationTokenSource(); + var scheduled = rig.Clock.WhenScheduled(); + var canceled = rig.Publish(0, token: cancellation.Token); + Assert.Equal(Period, await Phase(scheduled, "cancelable publication admitted")); + var retry = rig.Publish(0); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => canceled); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False(retry.IsCompleted); + rig.Clock.Advance(Milliseconds(25)); + + var peer = rig.Publish(1); + Assert.All(await Phase(Task.WhenAll(retry, peer), "shared receipt survives cancellation"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); + } + + [Fact] + public async Task PreCanceledPublicationDoesNotOpenCohort() + { + await using var rig = new TestRig(2); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + async () => await rig.Publish(0, token: cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(0, rig.Clock.ScheduleCount); + var peer = await rig.StartPublicationAsync(1); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(peer.IsCompleted); + rig.Clock.Advance(Milliseconds(1)); + Assert.True((await Phase(peer, "only admitted producer")).Accepted); + Assert.Equal(rig.Notification(1), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public async Task ThousandsOfProducersRetainLatestAtCapacityAndDispatchBoundedChunks(int capacity) + { + await using var rig = new TestRig(capacity + 1); + rig.Namespace.Options.MaxPendingItemCount = capacity; + rig.Options.MaxBatchItems = 128; + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < capacity; i++) + { + receipts.Add(rig.Publish(i)); + } + + Assert.Equal(new(false, Period), await rig.Publish(capacity)); + receipts.Add(rig.Publish(0, 9)); + receipts.Add(rig.Publish(capacity - 1, 7)); + rig.Clock.Advance(Period); + Assert.All(await Phase(Task.WhenAll(receipts), "bounded producer cohort"), receipt => Assert.True(receipt.Accepted)); + + var attempts = rig.Attempts.ToArray(); + Assert.Equal((capacity + 127) / 128, attempts.Length); + Assert.All(attempts.Take(attempts.Length - 1), attempt => Assert.Equal(128, attempt.Notifications.Length)); + Assert.Equal((capacity - 1) % 128 + 1, attempts[^1].Notifications.Length); + var sent = attempts.SelectMany(attempt => attempt.Notifications).ToArray(); + Assert.Equal(rig.Members.Take(capacity).Select(silo => new DisseminationKey(silo)), sent.Select(n => n.Key)); + Assert.Equal(9, sent[0].Version); + Assert.Equal(7, sent[^1].Version); + Assert.DoesNotContain(sent, n => n.Key.Equals(new DisseminationKey(rig.Members[capacity]))); + Assert.All(attempts, attempt => Assert.Equal(attempts[0].Timestamp, attempt.Timestamp)); + Assert.Empty(rig.Namespace.VersionReads); + + var later = await rig.StartPublicationAsync(capacity); + rig.Clock.Advance(Period); + Assert.True((await Phase(later, "capacity released after admission")).Accepted); + } + + [Theory] + [InlineData(2000)] + [InlineData(2048)] + public async Task FullCohortPreservesPeerItemAndByteSplitting(int count) + { + await using var rig = new TestRig(count); + rig.Options.MaxBatchItems = 128; + rig.Options.MaxBatchBytes = 64; + rig.Namespace.Options.MaxPayloadBytes = sizeof(long); + rig.Namespace.Options.MaxPendingItemCount = count; + rig.Namespace.MaterializeValues = true; + var target = Substitute.For(); + var factory = Substitute.For(); + var batches = new ConcurrentQueue(); + target.PushBroadcast(Arg.Any(), Arg.Any()) + .Returns(call => + { + var batch = call.ArgAt(0); + batches.Enqueue(batch); + return Task.FromResult(new DisseminationBroadcastResponse + { + Acknowledgments = batch.Values.ToDictionary( + pair => pair.Key, pair => pair.Value.Select(value => new DigestEntry(value.Value.Key, value.Value.ToVersion)).ToList()), + }); + }); + factory.GetSystemTarget(Constants.DisseminationSystemTargetType, rig.Members[1]).Returns(target); + var queue = new DisseminationBroadcastQueue( + rig.Clock, rig.Members[0], factory, new TestOptionsMonitor(rig.Options), + [rig.Namespace], NullLogger.Instance); + rig.OnDispatch = attempt => queue.NotifyBatch(rig.Members[1], rig.Namespace, attempt.Notifications); + try + { + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < count; i++) + { + receipts.Add(rig.Publish(i)); + } + + Assert.All(await Phase(Task.WhenAll(receipts), "all producers sealed without fake time advancement"), + receipt => Assert.Equal(new(true, Period), receipt)); + await Phase(queue.FlushPendingBroadcast(TestToken), "peer byte-limited chunks acknowledged"); + + var values = batches.SelectMany(batch => batch.Values[rig.Namespace.Name]).Select(value => value.Value).ToArray(); + Assert.Equal(count, values.Length); + Assert.Equal(count, values.Select(value => value.Key).Distinct().Count()); + Assert.Equal(rig.Members.Select(silo => new DisseminationKey(silo)).Order(), values.Select(value => value.Key).Order()); + Assert.All(batches, batch => + { + var payloads = batch.Values[rig.Namespace.Name]; + Assert.InRange(payloads.Count, 1, rig.Options.MaxBatchItems); + Assert.InRange(payloads.Sum(value => value.Value.Payload.Length), 1, rig.Options.MaxBatchBytes); + Assert.All(payloads, value => Assert.Equal(1L, BitConverter.ToInt64(value.Value.Payload.Span))); + }); + Assert.All(rig.Attempts, attempt => Assert.InRange(attempt.Notifications.Length, 1, 128)); + } + finally + { + await queue.StopAsync(TestToken); + } + } + + [Theory] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public async Task UpdatesDuringDispatchPreserveGenerationForceAndInFlightCapacity(bool firstAccepted, bool newForce) + { + await using var rig = new TestRig(2); + rig.Namespace.Options.MaxPendingItemCount = 1; + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "First dispatch was not released."); + } + + return attempt.Number > 1 || firstAccepted; + }; + var first = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task newer = null!; + try + { + await Phase(Task.Run(async () => + { + Assert.Equal(new(false, Period), await rig.Publish(1)); + newer = rig.Publish(0, 2, newForce); + Assert.False(newer.IsCompleted); + }, TestToken), "new generation admitted concurrently with the dispatch callback"); + } + finally + { + release.Set(); + } + + Assert.Equal(firstAccepted, (await Phase(first, "first generation receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "new generation deadline")); + rig.Clock.Advance(Period); + Assert.True((await Phase(newer, "new generation receipt")).Accepted); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, 2, newForce) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task SameVersionInvalidationDuringDispatchRemainsForced() + { + await using var rig = new TestRig(); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Invalidated dispatch was not released."); + } + + return true; + }; + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + try + { + await Phase(Task.Run(() => + { + Assert.True(rig.Batcher.Notify([new("a", 1, false)])); + Assert.True(rig.Batcher.Notify([new("a", 1, true)])); + }, TestToken), "same-version invalidation admitted during callback"); + } + finally + { + release.Set(); + } + + Assert.Equal(Period, await Phase(scheduled, "invalidation remains pending")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "same-version invalidation dispatched"); + Assert.Equal(new KeyNotification[] { new("a", 1, false), new("a", 1, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task OlderInFlightRetryUsesSealedSignalWhileNewerVersionWaitsForNextCohort() + { + await using var rig = new TestRig(2); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "First generation was not released."); + } + + return true; + }; + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task newer; + Task retry; + try + { + newer = rig.Publish(0, 2); + retry = rig.Publish(0); + Assert.False(first.IsCompleted); + Assert.False(newer.IsCompleted); + Assert.False(retry.IsCompleted); + } + finally + { + release.Set(); + } + + Assert.All(await Phase(Task.WhenAll(first, retry), "old callers share the sealed generation"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.False(newer.IsCompleted); + Assert.Equal(Period, await Phase(scheduled, "only the new generation waits another period")); + rig.Clock.Advance(Period); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(newer, "next generation admitted")); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(0, 2) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Theory] + [InlineData(false, false, 1L)] + [InlineData(false, true, 1L)] + [InlineData(false, true, 2L)] + [InlineData(true, false, 1L)] + public async Task RejectedDispatchCachesReceiptAndRetriesNewestHintAndForce(bool callbackThrows, bool newForce, long version) + { + await using var rig = new TestRig(2); + var failure = new InvalidOperationException("The parent dispatcher failed."); + rig.OnDispatch = attempt => attempt.Number != 1 || (callbackThrows ? throw failure : false); + var first = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + Assert.Equal(new(false, TimeSpan.Zero), await Phase(first, "partial admission receipt")); + Assert.Equal(Period, await Phase(scheduled, "bounded hint retry")); + Assert.Equal(new(false, TimeSpan.Zero), await rig.Publish(0)); + if (newForce) + { + Assert.True(rig.Batcher.Notify([rig.Notification(0, version, true)])); + } + + rig.Clock.Advance(Milliseconds(999)); + Assert.Single(rig.Attempts); + rig.Clock.Advance(Milliseconds(1)); + await Phase(rig.Batcher.FlushAsync(TestToken), "retained hint admitted on retry"); + Assert.Equal(2, rig.Attempts.Count); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, version, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + // A later hint retry is not another publication receipt or another producer contribution. + Assert.Equal(new(false, TimeSpan.Zero), await rig.Publish(0)); + if (callbackThrows) + { + var entry = Assert.Single(rig.Logger.Entries); + Assert.Equal(LogLevel.Warning, entry.Level); + Assert.Same(failure, entry.Exception); + } + else + { + Assert.Empty(rig.Logger.Entries); + } + } + + [Fact] + public async Task PartialChunkAdmissionResolvesOnlyItsOwnProducerReceipts() + { + await using var rig = new TestRig(3); + rig.Options.MaxBatchItems = 1; + rig.OnDispatch = attempt => attempt.Number != 2; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var retryScheduled = rig.Clock.WhenScheduled(); + var third = rig.Publish(2); + var receipts = await Phase(Task.WhenAll(first, second, third), "per-chunk receipts"); + Assert.Equal(new[] { true, false, true }, receipts.Select(receipt => receipt.Accepted)); + Assert.All(receipts, receipt => Assert.Equal(Period, receipt.NextPublicationDelay)); + Assert.Equal(Period, await Phase(retryScheduled, "only rejected chunk retained")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "one rejected chunk retried"); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1), rig.Notification(2), rig.Notification(1) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task CallbackLoggingFailureIsVisibleToReceiptAdmissionAndDrain() + { + await using var rig = new TestRig(); + var dispatchFailure = new InvalidOperationException("The parent dispatcher failed."); + rig.Logger.ThrowOnLog = true; + rig.OnDispatch = _ => throw dispatchFailure; + var publication = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Period); + var failure = await Phase(rig.Logger.WorkerFailed.Task, "dispatch logging failure recorded"); + Assert.Same(dispatchFailure, Assert.IsType(failure).InnerExceptions[0]); + var receipt = await Assert.ThrowsAsync(() => publication); + Assert.Same(failure, receipt.InnerException); + var admission = Assert.Throws(() => rig.Batcher.Notify([new("b", 1, true)])); + Assert.Same(failure, admission.InnerException); + var drain = await Assert.ThrowsAsync(() => rig.Batcher.FlushAsync(TestToken)); + Assert.Same(failure, drain.InnerException); + } + + [Fact] + public async Task WorkerFailureIsObservedByActiveAndLaterFlushNotifyPublishAndStop() + { + await using var rig = new TestRig(); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.Logger.ThrowOnLog = true; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Failing dispatch was not released."); + return false; + }; + await rig.NotifyAsync(new KeyNotification("a", 1, true)); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task activeFlush; + try + { + activeFlush = rig.Batcher.FlushAsync(TestToken); + Assert.False(activeFlush.IsCompleted); + rig.Clock.FailNextSchedule = true; + } + finally + { + release.Set(); + } + + var failure = await Phase(rig.Logger.WorkerFailed.Task, "terminal timer failure logged"); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => Phase(activeFlush, "active flush observes terminal failure"))).InnerException); + Assert.Same(failure, Assert.Throws(() => rig.Batcher.Notify([new("b", 2, false)])).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + async () => await rig.Publish(0))).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => rig.Batcher.FlushAsync(TestToken))).InnerException); + Assert.Same(failure, (await Assert.ThrowsAsync( + () => rig.Batcher.StopAsync(TestToken))).InnerException); + Assert.True(rig.Clock.TimerDisposed); + Assert.Single(rig.Attempts); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisablingRejectsReceiptsAndReenableDoesNotReplay(bool global) + { + await using var rig = new TestRig(); + var publication = await rig.StartPublicationAsync(0); + if (global) + { + rig.Options.Enabled = false; + } + else + { + rig.Namespace.Options.Enabled = false; + } + + await Phase(rig.Batcher.FlushAsync(TestToken), "disabled work retired"); + Assert.False((await publication).Accepted); + Assert.False(rig.Batcher.Notify([new("disabled", 2, true)])); + Assert.False((await rig.Publish(1)).Accepted); + Assert.Empty(rig.Attempts); + rig.Options.Enabled = true; + rig.Namespace.Options.Enabled = true; + Assert.False((await rig.Publish(0)).Accepted); + var newer = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Period); + Assert.True((await Phase(newer, "new work after reenabling")).Accepted); + Assert.Equal(rig.Notification(0, 2), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task MembershipSnapshotIsStableAndRetiredIncarnationCannotContribute() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var retired = rig.Members[2]; + var replacement = SiloAddress.New(retired.Endpoint, retired.Generation + 1); + rig.SetMembership([rig.Members[0], rig.Members[1], replacement]); + Assert.False((await rig.Batcher.PublishAsync(new(retired, 9, true), TestToken)).Accepted); + Assert.False(rig.Batcher.Notify([new(retired, 9, true)])); + var other = rig.Publish(1); + var joined = rig.Batcher.PublishAsync(new(replacement, 1, false), TestToken).AsTask(); + rig.Clock.Advance(Milliseconds(999)); + Assert.False(first.IsCompleted); + Assert.False(other.IsCompleted); + Assert.False(joined.IsCompleted); + Assert.Empty(rig.Attempts); + rig.Clock.Advance(Milliseconds(1)); + Assert.All(await Phase(Task.WhenAll(first, other, joined), "stable cohort tolerates incarnation replacement"), + receipt => Assert.Equal(new(true, TimeSpan.Zero), receipt)); + Assert.DoesNotContain(Assert.Single(rig.Attempts).Notifications, n => n.Key.Equals(new DisseminationKey(retired))); + } + + [Fact] + public async Task MembershipChangeRetiresPendingReceiptsAndFreesCapacity() + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 1; + var departed = await rig.StartPublicationAsync(2); + rig.SetMembership([rig.Members[0], rig.Members[1]]); + var scheduled = rig.Clock.WhenScheduled(); + var active = rig.Publish(0); + Assert.False((await Phase(departed, "departed incarnation rejected")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "replacement pending capacity")); + rig.Clock.Advance(Period); + Assert.True((await Phase(active, "active contribution accepted")).Accepted); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task EmptyMembershipReleasesPendingReceiptsWithoutDispatch() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + rig.SetMembership([]); + rig.Batcher.WakeForMembershipChange(); + Assert.All(await Phase(Task.WhenAll(first, second), "root and peers no longer eligible"), + receipt => Assert.False(receipt.Accepted)); + await Phase(rig.Batcher.StopAsync(TestToken), "root loss drains without destinations"); + Assert.Empty(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task RacingMembershipReadCannotReadmitAnAlreadyRetiredIncarnation() + { + await using var rig = new TestRig(); + var first = await rig.StartPublicationAsync(0); + var oldMembership = rig.Membership; + using var release = new ManualResetEventSlim(); + var token = TestToken; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + rig.OnMembershipRead = () => + { + if (Interlocked.Increment(ref reads) == 1) + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Stale membership read was not released."); + return oldMembership; + } + + return rig.Membership; + }; + var stale = Task.Run(() => rig.Publish(1), TestToken); + try + { + await Phase(entered.Task, "producer captured old membership"); + rig.SetMembership([rig.Members[0]]); + await Phase(Task.Run(() => Assert.False(rig.Batcher.Notify([rig.Notification(2)])), TestToken), + "new membership observed while older callback is blocked"); + } + finally + { + release.Set(); + } + + Assert.False((await Phase(stale, "stale membership cannot reverse retirement")).Accepted); + rig.Clock.Advance(Period); + Assert.True((await Phase(first, "remaining root contribution")).Accepted); + Assert.Equal(rig.Notification(0), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task MembershipWakeHandsAllPendingChunksToNewRootWithoutWaiting() + { + await using var rig = new TestRig(5); + rig.Options.MaxBatchItems = 2; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var third = rig.Publish(2); + Assert.True(rig.Batcher.Notify([rig.Notification(3)])); + rig.Clock.Advance(Milliseconds(25)); + rig.SetMembership([rig.Members[1], rig.Members[0], rig.Members[2], rig.Members[3]]); + var routedToNewRoot = new ConcurrentQueue(); + rig.OnDispatch = _ => + { + routedToNewRoot.Enqueue(!rig.Membership.IsAggregationRoot); + return true; + }; + rig.Batcher.WakeForMembershipChange(); + Assert.All(await Phase(Task.WhenAll(first, second, third), "new root handoff receipts"), + receipt => Assert.Equal(new(true, Milliseconds(975)), receipt)); + Assert.Equal(new[] { 2, 2 }, rig.Attempts.Select(attempt => attempt.Notifications.Length)); + Assert.All(routedToNewRoot, routed => Assert.True(routed)); + Assert.False((await rig.Publish(0, 2)).Accepted); + await Phase(rig.Batcher.StopAsync(TestToken), "former root stopped"); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task MembershipChangeDuringDispatchSkipsRetiredChunkAndRejectsItsReceipt() + { + await using var rig = new TestRig(4); + rig.Options.MaxBatchItems = 1; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var retired = rig.Publish(2); + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + rig.SetMembership([rig.Members[0], rig.Members[1]]); + } + + return true; + }; + + await Phase(rig.Batcher.FlushAsync(TestToken), "membership revalidated between chunks"); + var receipts = await Task.WhenAll(first, second, retired); + Assert.Equal(new[] { true, true, false }, receipts.Select(receipt => receipt.Accepted)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task RootLossBypassesOldRetryFloorOnceButFailedHandoffRemainsBounded() + { + await using var rig = new TestRig(2); + rig.OnDispatch = _ => false; + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + Assert.False((await Phase(first, "first admission rejected")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "old root retry floor")); + rig.Clock.Advance(Milliseconds(25)); + rig.SetMembership([rig.Members[1], rig.Members[0]]); + var handoffScheduled = rig.Clock.WhenScheduled(); + rig.Batcher.WakeForMembershipChange(); + Assert.Equal(Period, await Phase(handoffScheduled, "failed new-root handoff uses one period retry")); + Assert.Equal(2, rig.Attempts.Count); + rig.Clock.Advance(Milliseconds(999)); + Assert.Equal(2, rig.Attempts.Count); + rig.OnDispatch = _ => true; + rig.Clock.Advance(Milliseconds(1)); + await Phase(rig.Batcher.FlushAsync(TestToken), "handoff retry admitted"); + Assert.Equal(3, rig.Attempts.Count); + Assert.Equal(Milliseconds(25), rig.Clock.GetElapsedTime(rig.Attempts.First().Timestamp, rig.Attempts.ElementAt(1).Timestamp)); + } + + [Fact] + public async Task PruneRemovesRetiredKeysAndPreservesNewerOwnerInventory() + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 3; + await rig.NotifyAsync(new("retired", 1, true), new("retained", 2, false)); + var inventory = new HashSet { "retained" }; + rig.Namespace.GetVersionHandler = key => key.Equals(new DisseminationKey("new")) ? 3 : 0; + Assert.True(rig.Batcher.Notify([new("new", 3, false)])); + var scheduled = rig.Clock.WhenScheduled(); + rig.Batcher.Prune(inventory); + Assert.Equal(Period, await Phase(scheduled, "inventory rechecked outside lock")); + Assert.True(rig.Batcher.Notify([new("extra", 4, false)])); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "pruned inventory dispatched"); + Assert.Equal(new KeyNotification[] { new("retained", 2, false), new("new", 3, false), new("extra", 4, false) }, + Assert.Single(rig.Attempts).Notifications); + Assert.Equal(new DisseminationKey[] { "retired", "new" }, rig.Namespace.VersionReads); + } + + [Theory] + [InlineData(1L)] + [InlineData(2L)] + public async Task PruneVersionReadRunsOutsideLockAndPreservesRacingNotifications(long version) + { + await using var rig = new TestRig(); + await rig.NotifyAsync(new KeyNotification("a", 1, false)); + using var release = new ManualResetEventSlim(); + var versionRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Namespace.GetVersionHandler = key => + { + Assert.Equal(new DisseminationKey("a"), key); + versionRead.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), TestToken), "Owner version read was not released."); + return 0; + }; + var scheduled = rig.Clock.WhenScheduled(); + var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken); + try + { + await Phase(versionRead.Task, "pruning captured the old generation"); + await Phase(Task.Run(() => Assert.True(rig.Batcher.Notify([new("a", version, false)])), TestToken), + "notification admitted while namespace version read is blocked"); + } + finally + { + release.Set(); + await Phase(prune, "pruning reconciled admission generations"); + } + + Assert.Equal(Period, await Phase(scheduled, "racing notification preserved")); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "racing notification dispatched"); + Assert.Equal(new KeyNotification("a", version, false), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task PruneDuringDispatchDoesNotEraseReadmittedIdentity() + { + await using var rig = new TestRig(); + rig.Namespace.Options.MaxPendingItemCount = 1; + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.OnDispatch = attempt => + { + if (attempt.Number == 1) + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Pruned dispatch was not released."); + } + + return true; + }; + var first = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + rig.Clock.Advance(Period); + await rig.NextAttemptAsync(); + Task next; + try + { + rig.Batcher.Prune(new HashSet()); + Assert.False((await Phase(first, "pruned in-flight receipt rejected")).Accepted); + next = rig.Publish(0, 7, force: true); + } + finally + { + release.Set(); + } + + Assert.Equal(Period, await Phase(scheduled, "readmitted identity gets its own deadline")); + rig.Clock.Advance(Period); + Assert.True((await Phase(next, "readmitted identity accepted")).Accepted); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(0, 7, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + [Fact] + public async Task PublicationRetryProtectsAdmissionFromRacingPruneWithoutContributingTwice() + { + await using var rig = new TestRig(2); + var first = await rig.StartPublicationAsync(0); + using var release = new ManualResetEventSlim(); + var token = TestToken; + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + rig.Namespace.GetVersionHandler = _ => + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Prune version read was not released."); + return 0; + }; + var scheduled = rig.Clock.WhenScheduled(); + var prune = Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken); + Task retry; + try + { + await Phase(entered.Task, "prune captured pre-retry admission"); + retry = rig.Publish(0); + Assert.False(first.IsCompleted); + Assert.False(retry.IsCompleted); + Assert.Empty(rig.Attempts); + } + finally + { + release.Set(); + await Phase(prune, "prune reconciled retry"); + } + + Assert.Equal(Period, await Phase(scheduled, "retry retained without changing deadline")); + var second = rig.Publish(1); + Assert.All(await Phase(Task.WhenAll(first, retry, second), "original contribution survives prune"), + receipt => Assert.Equal(new(true, Period), receipt)); + Assert.Equal(new[] { rig.Notification(0), rig.Notification(1) }, Assert.Single(rig.Attempts).Notifications); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NamespaceSettingsCallbacksRunOutsideBatcherLock(bool period) + { + await using var rig = new TestRig(); + using var release = new ManualResetEventSlim(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var calls = 0; + Action callback = () => + { + if (Interlocked.Increment(ref calls) == 1) + { + entered.TrySetResult(); + Assert.True(release.Wait(TimeSpan.FromSeconds(10), TestToken), "Namespace settings callback was not released."); + } + }; + if (period) + { + rig.Namespace.OnPeriodRead = callback; + } + else + { + rig.Namespace.OnOptionsRead = callback; + } + + var admission = Task.Run(() => rig.Batcher.Notify([new("a", 1, true)]), TestToken); + try + { + await Phase(entered.Task, "namespace callback entered"); + await Phase(Task.Run(() => rig.Batcher.Prune(new HashSet()), TestToken), + "another thread enters batcher while namespace callback blocks"); + } + finally + { + release.Set(); + } + + Assert.True(await Phase(admission, "namespace callback returned")); + await Phase(rig.Batcher.FlushAsync(TestToken), "namespace reentrancy work drained"); + Assert.Equal(new KeyNotification("a", 1, true), Assert.Single(Assert.Single(rig.Attempts).Notifications)); + } + + [Fact] + public async Task StopSealsPartialCohortDrainsBoundedChunksAndRejectsFurtherAdmission() + { + await using var rig = new TestRig(6); + rig.Options.MaxBatchItems = 2; + var receipts = new List> { await rig.StartPublicationAsync(0) }; + for (var i = 1; i < 5; i++) + { + receipts.Add(rig.Publish(i)); + } + + await Phase(rig.Batcher.StopAsync(TestToken), "stop seals partial cohort"); + Assert.All(await Task.WhenAll(receipts), receipt => Assert.Equal(new(true, Period), receipt)); + Assert.Equal(new[] { 2, 2, 1 }, rig.Attempts.Select(attempt => attempt.Notifications.Length)); + Assert.False(rig.Batcher.Notify([new("late", 6, true)])); + Assert.False((await rig.Publish(5)).Accepted); + Assert.True(rig.Clock.TimerDisposed); + await rig.Batcher.StopAsync(TestToken); + } + + [Fact] + public async Task FlushSealsPartialCohortWithoutStoppingLaterAdmission() + { + await using var rig = new TestRig(3); + var first = await rig.StartPublicationAsync(0); + rig.Clock.Advance(Milliseconds(25)); + await Phase(rig.Batcher.FlushAsync(TestToken), "flush seals partial cohort"); + Assert.Equal(new(true, Milliseconds(975)), await first); + Assert.False(rig.Clock.TimerDisposed); + rig.Clock.Advance(Milliseconds(975)); + var second = await rig.StartPublicationAsync(0, 2); + rig.Clock.Advance(Period); + Assert.Equal(new(true, TimeSpan.Zero), await Phase(second, "admission after flush")); + Assert.Equal(2, rig.Attempts.Count); + } + + [Fact] + public async Task StopCancellationEndsRetryBudgetAndPreservesCallerToken() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + rig.OnDispatch = _ => false; + var publication = await rig.StartPublicationAsync(0); + var scheduled = rig.Clock.WhenScheduled(); + var stop = rig.Batcher.StopAsync(cancellation.Token); + Assert.False((await Phase(publication, "stop's failed attempt rejects receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "stop retry bounded")); + Assert.False(stop.IsCompleted); + Assert.False((await rig.Publish(1)).Accepted); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop budget exhausted")); + Assert.Equal(cancellation.Token, exception.CancellationToken); + var later = await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(CancellationToken.None)); + Assert.Equal(cancellation.Token, later.CancellationToken); + Assert.True(rig.Clock.TimerDisposed); + rig.Clock.Advance(TimeSpan.FromDays(1)); + Assert.Single(rig.Attempts); + } + + [Fact] + public async Task StopCancellationDuringDispatchRejectsWaitersAndDoesNotClaimNextChunk() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + using var release = new ManualResetEventSlim(); + var token = TestToken; + rig.Options.MaxBatchItems = 1; + rig.OnDispatch = _ => + { + Assert.True(release.Wait(TimeSpan.FromSeconds(10), token), "Stop's in-flight callback was not released."); + return true; + }; + var first = await rig.StartPublicationAsync(0); + var second = rig.Publish(1); + var stop = rig.Batcher.StopAsync(cancellation.Token); + await rig.NextAttemptAsync(); + try + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => Phase(stop, "stop callback budget")); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.All(await Phase(Task.WhenAll(first, second), "all in-flight and unclaimed receipts rejected"), + receipt => Assert.False(receipt.Accepted)); + } + finally + { + release.Set(); + } + + await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(CancellationToken.None)); + Assert.Single(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task PreCanceledStopRejectsReceiptsWithoutDispatch() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + var publication = await rig.StartPublicationAsync(0); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => rig.Batcher.StopAsync(cancellation.Token)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.False((await Phase(publication, "abandoned producer released")).Accepted); + Assert.False(rig.Batcher.Notify([new("late", 2, true)])); + Assert.Empty(rig.Attempts); + Assert.True(rig.Clock.TimerDisposed); + } + + [Fact] + public async Task FlushCancellationLeavesAdmittedWorkForPeriodBoundedRetry() + { + await using var rig = new TestRig(); + using var cancellation = new CancellationTokenSource(); + rig.OnDispatch = attempt => attempt.Number != 1; + var publication = await rig.StartPublicationAsync(0, force: true); + var scheduled = rig.Clock.WhenScheduled(); + var flush = rig.Batcher.FlushAsync(cancellation.Token); + Assert.False((await Phase(publication, "flush partial admission receipt")).Accepted); + Assert.Equal(Period, await Phase(scheduled, "flush retry bounded")); + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync(() => flush); + Assert.Equal(cancellation.Token, exception.CancellationToken); + rig.Clock.Advance(Period); + await Phase(rig.Batcher.FlushAsync(TestToken), "admitted hint survives flush cancellation"); + Assert.Equal(new[] { rig.Notification(0, 1, true), rig.Notification(0, 1, true) }, + rig.Attempts.SelectMany(attempt => attempt.Notifications)); + } + + private static async Task Phase(Task task, string phase) + { + try + { + await task.WaitAsync(TimeSpan.FromSeconds(10), TestToken); + } + catch (TimeoutException exception) + { + throw new TimeoutException($"Root batcher phase did not complete: {phase}.", exception); + } + } + + private static async Task Phase(Task task, string phase) + { + await Phase((Task)task, phase); + return await task; + } + + private sealed record Attempt(int Number, long Timestamp, KeyNotification[] Notifications); + + private sealed class TestRig : IAsyncDisposable + { + private readonly Channel _attempts = Channel.CreateUnbounded(); + private int _attemptNumber; + + public TestRig(int members = 3) + { + Members = Enumerable.Range(0, members).Select(index => SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 18000 + index), 1)).ToImmutableArray(); + SetMembership(Members); + Batcher = new(Clock, Namespace, new TestOptionsMonitor(Options), () => OnMembershipRead?.Invoke() ?? Membership, Dispatch, Logger); + } + + public TestClock Clock { get; } = new(); + public TestNamespace Namespace { get; } = new(); + public DisseminationOptions Options { get; } = new() { Enabled = true }; + public TestLogger Logger { get; } = new(); + public ImmutableArray Members { get; } + public DisseminationMembershipSnapshot Membership { get; private set; } = null!; + public DisseminationRootBatcher Batcher { get; } + public ConcurrentQueue Attempts { get; } = new(); + public Func? OnDispatch { get; set; } + public Func? OnMembershipRead { get; set; } + + public void SetMembership(ImmutableArray members) => + Membership = new(new MembershipVersion((Membership?.MembershipVersion.Value ?? 0) + 1), + Members[0], members, Options.Overlay); + + public KeyNotification Notification(int producer, long version = 1, bool force = false) => + new(Members[producer], version, force); + + public Task Publish(int producer, long version = 1, bool force = false, CancellationToken? token = null) => + Batcher.PublishAsync(Notification(producer, version, force), token ?? TestToken).AsTask(); + + public async Task> StartPublicationAsync(int producer, long version = 1, bool force = false) + { + var scheduled = Clock.WhenScheduled(); + var publication = Publish(producer, version, force); + Assert.Equal(Period, await Phase(scheduled, "first contribution timer armed")); + return publication; + } + + public async Task NotifyAsync(params KeyNotification[] notifications) + { + var scheduled = Clock.WhenScheduled(); + Assert.True(Batcher.Notify(notifications)); + Assert.Equal(Period, await Phase(scheduled, "hint collection timer armed")); + } + + public Task NextAttemptAsync() => + Phase(_attempts.Reader.ReadAsync(TestToken).AsTask(), "dispatch callback entered"); + + private bool Dispatch(ReadOnlySpan notifications) + { + var attempt = new Attempt(Interlocked.Increment(ref _attemptNumber), Clock.GetTimestamp(), notifications.ToArray()); + Attempts.Enqueue(attempt); + Assert.True(_attempts.Writer.TryWrite(attempt)); + return OnDispatch?.Invoke(attempt) ?? true; + } + + public async ValueTask DisposeAsync() + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try + { + await Batcher.StopAsync(cleanup.Token); + } + catch (OperationCanceledException) + { + } + catch (InvalidOperationException) when (Logger.WorkerFailed.Task.IsCompletedSuccessfully) + { + } + } + } + + private sealed class TestNamespace : IDisseminationNamespace + { + private readonly DisseminationNamespaceOptions _options = new() + { + Enabled = true, + MaxPendingItemCount = 8192, + }; + + public DisseminationNamespace Name => new("root-batcher-test"); + public ConcurrentQueue VersionReads { get; } = new(); + public Func? GetVersionHandler { get; set; } + public Action? OnOptionsRead { get; set; } + public Action? OnPeriodRead { get; set; } + public bool MaterializeValues { get; set; } + public DisseminationNamespaceOptions Options + { + get + { + OnOptionsRead?.Invoke(); + return _options; + } + } + + public TimeSpan AggregationPeriod + { + get + { + OnPeriodRead?.Invoke(); + return Period; + } + } + + public IEnumerable Digests => throw new InvalidOperationException("Root cohorts must not inspect namespace values."); + public long GetVersion(DisseminationKey key) + { + VersionReads.Enqueue(key); + return GetVersionHandler?.Invoke(key) ?? (MaterializeValues ? 1 : 0); + } + + public DisseminationRepairResult CreateRepair(in DisseminationRepairRequest request) + { + if (!MaterializeValues) + { + throw new InvalidOperationException("Only peer queues may materialize payloads."); + } + + if (request.FromVersion >= 1) + { + return DisseminationRepairResult.Current(1); + } + + if (request.MaxBatchBytes < sizeof(long) || request.MaxPayloadBytes < sizeof(long)) + { + return DisseminationRepairResult.InsufficientCapacity(1); + } + + return DisseminationRepairResult.Produced(new(request.Key, 0, 1, BitConverter.GetBytes(1L))); + } + + public ValueTask ApplyValueAsync(DisseminationValue value, CancellationToken cancellationToken) => + throw new InvalidOperationException("Root cohorts must not apply payloads."); + } + + private sealed class TestOptionsMonitor(DisseminationOptions options) : IOptionsMonitor + { + public DisseminationOptions CurrentValue => options; + public DisseminationOptions Get(string? name) => options; + public IDisposable? OnChange(Action listener) => null; + } + + // Exactly one fake-time driver advances time after a barrier proves the real timer was armed. + private sealed class TestClock : TimeProvider + { + private readonly FakeTimeProvider _inner = new(); + private TaskCompletionSource? _nextSchedule; + private int _failNextSchedule; + private int _timerDisposed; + private int _timerCount; + private int _scheduleCount; + + public int TimerCount => Volatile.Read(ref _timerCount); + public int ScheduleCount => Volatile.Read(ref _scheduleCount); + public bool TimerDisposed => Volatile.Read(ref _timerDisposed) != 0; + public bool FailNextSchedule { set => Volatile.Write(ref _failNextSchedule, value ? 1 : 0); } + public override DateTimeOffset GetUtcNow() => _inner.GetUtcNow(); + public override long GetTimestamp() => _inner.GetTimestamp(); + public override long TimestampFrequency => _inner.TimestampFrequency; + public void Advance(TimeSpan elapsed) => _inner.Advance(elapsed); + + public Task WhenScheduled() + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Assert.Null(Interlocked.CompareExchange(ref _nextSchedule, completion, null)); + return completion.Task; + } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + Interlocked.Increment(ref _timerCount); + return new TestTimer(this, _inner.CreateTimer(callback, state, dueTime, period)); + } + + private sealed class TestTimer(TestClock owner, ITimer inner) : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) + { + if (dueTime > TimeSpan.Zero && Interlocked.Exchange(ref owner._failNextSchedule, 0) != 0) + { + throw new InvalidOperationException("The root timer cannot schedule another cohort."); + } + + var changed = inner.Change(dueTime, period); + if (dueTime > TimeSpan.Zero) + { + Interlocked.Increment(ref owner._scheduleCount); + Interlocked.Exchange(ref owner._nextSchedule, null)?.TrySetResult(dueTime); + } + + return changed; + } + + public void Dispose() + { + inner.Dispose(); + Volatile.Write(ref owner._timerDisposed, 1); + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } + + private sealed class TestLogger : ILogger + { + public ConcurrentQueue<(LogLevel Level, Exception? Exception)> Entries { get; } = new(); + public TaskCompletionSource WorkerFailed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public bool ThrowOnLog { get; set; } + public bool IsEnabled(LogLevel logLevel) => true; + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Entries.Enqueue((logLevel, exception)); + if (logLevel == LogLevel.Error && exception is not null) + { + WorkerFailed.TrySetResult(exception); + } + + if (ThrowOnLog) + { + throw new InvalidOperationException("The test logger throws."); + } + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs new file mode 100644 index 00000000000..cba2cc6cd52 --- /dev/null +++ b/test/Orleans.Runtime.Internal.Tests/Dissemination/WakeTimerTests.cs @@ -0,0 +1,276 @@ +#nullable enable + +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.Dissemination; +using Xunit; + +namespace UnitTests.Dissemination; + +[TestCategory("BVT"), TestCategory("Dissemination")] +[TestSuite("BVT")] +[TestProvider("None")] +[TestArea("Dissemination")] +public class WakeTimerTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ChangeCompletesWaitAtLatestDueTime(bool rearm) + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Change(TimeSpan.FromSeconds(1)); + if (rearm) + { + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timer.Change(TimeSpan.FromSeconds(1)); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(999)); + Assert.False(wait.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + + Assert.True(await wait); + } + + [Fact] + public async Task WakeBeforeWaitCompletesNextWait() + { + using var timer = new WakeTimer(TimeProvider.System); + + timer.Wake(); + + Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ResetConsumesRedundantWakeBeforeRetryIsArmed() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var first = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + timer.Wake(); + timer.Wake(); + Assert.True(await first); + + timer.Reset(); + timer.Change(TimeSpan.FromMilliseconds(100)); + var retry = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.False(retry.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + Assert.False(retry.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + Assert.True(await retry.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ResetDisarmsOldDeadlineAndPreservesActiveWaiter(bool cancelWait) + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + using var cancellation = new CancellationTokenSource(); + var wait = timer.WaitAsync(cancellation.Token).AsTask(); + timer.Change(TimeSpan.FromMilliseconds(100)); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timer.Reset(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + Assert.False(wait.IsCompleted); + + if (cancelWait) + { + cancellation.Cancel(); + var exception = await Assert.ThrowsAnyAsync( + () => wait.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.Equal(cancellation.Token, exception.CancellationToken); + timer.Wake(); + Assert.True(await timer.WaitAsync(TestContext.Current.CancellationToken)); + } + else + { + timer.Change(TimeSpan.FromMilliseconds(100)); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + Assert.False(wait.IsCompleted); + timeProvider.Advance(TimeSpan.FromMilliseconds(1)); + Assert.True(await wait.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + } + } + + [Fact] + public async Task StaleCallbackDoesNotWakeRearmedTimer() + { + var timeProvider = new ControllableTimeProvider(); + using var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timer.Change(TimeSpan.FromSeconds(1)); + timeProvider.FireTimer(); + + Assert.False(wait.IsCompleted); + + timeProvider.Advance(TimeSpan.FromSeconds(1)); + timeProvider.FireTimer(); + + Assert.True(await wait); + } + + [Fact] + public async Task CancellingWaitDoesNotDisarmTimer() + { + var timeProvider = new FakeTimeProvider(); + using var timer = new WakeTimer(timeProvider); + using var cancellation = new CancellationTokenSource(); + timer.Change(TimeSpan.FromSeconds(1)); + var canceledWait = timer.WaitAsync(cancellation.Token).AsTask(); + + cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledWait); + + var nextWait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + timeProvider.Advance(TimeSpan.FromSeconds(1)); + + Assert.True(await nextWait); + } + + [Fact] + public async Task WakeRacingCancelledWaitIsObservedByNextWaiter() + { + using var timer = new WakeTimer(TimeProvider.System); + using var cancellation = new CancellationTokenSource(); + var synchronizationContext = new QueuedSynchronizationContext(); + Task canceledWait; + var previousSynchronizationContext = SynchronizationContext.Current; + try + { + SynchronizationContext.SetSynchronizationContext(synchronizationContext); + canceledWait = timer.WaitAsync(cancellation.Token).AsTask(); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousSynchronizationContext); + } + + cancellation.Cancel(); + timer.Wake(); + synchronizationContext.RunAll(); + + await Assert.ThrowsAnyAsync(() => canceledWait); + var nextWait = timer.WaitAsync(TestContext.Current.CancellationToken); + Assert.True(nextWait.IsCompletedSuccessfully); + Assert.True(await nextWait); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeCompletesCurrentAndFutureWaitsWithoutChangingTimer(bool changeThrows) + { + TimeProvider timeProvider = changeThrows ? new ThrowingChangeTimeProvider() : TimeProvider.System; + var timer = new WakeTimer(timeProvider); + var wait = timer.WaitAsync(TestContext.Current.CancellationToken).AsTask(); + + timer.Dispose(); + + Assert.False(await wait); + Assert.False(await timer.WaitAsync(TestContext.Current.CancellationToken)); + if (timeProvider is ThrowingChangeTimeProvider throwingProvider) + { + Assert.True(throwingProvider.Timer.IsDisposed); + } + } + + private sealed class QueuedSynchronizationContext : SynchronizationContext + { + private readonly Queue<(SendOrPostCallback Callback, object? State)> _workItems = []; + + public override void Post(SendOrPostCallback callback, object? state) => _workItems.Enqueue((callback, state)); + + public void RunAll() + { + while (_workItems.TryDequeue(out var workItem)) + { + workItem.Callback(workItem.State); + } + } + } + + private sealed class ControllableTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + private long _timestamp; + private ControllableTimer? _timer; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public override long GetTimestamp() => _timestamp; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + _timer = new ControllableTimer(callback, state); + _timer.Change(dueTime, period); + return _timer; + } + + public void Advance(TimeSpan duration) + { + _utcNow += duration; + _timestamp += duration.Ticks; + } + + public void FireTimer() => _timer!.Fire(); + + private sealed class ControllableTimer(TimerCallback callback, object? state) : ITimer + { + private bool _disposed; + + public bool Change(TimeSpan dueTime, TimeSpan period) => !_disposed; + + public void Dispose() => _disposed = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + + public void Fire() + { + if (!_disposed) + { + callback(state); + } + } + } + } + + private sealed class ThrowingChangeTimeProvider : TimeProvider + { + public ThrowingChangeTimer Timer { get; } = new(); + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => + Timer; + } + + private sealed class ThrowingChangeTimer : ITimer + { + public bool IsDisposed { get; private set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) => + throw new InvalidOperationException("Changing this timer is not supported."); + + public void Dispose() => IsDisposed = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj b/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj index 51393eff87c..83cd3c21549 100644 --- a/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj +++ b/test/Orleans.Runtime.Internal.Tests/Orleans.Runtime.Internal.Tests.csproj @@ -8,6 +8,7 @@ +